ceph: expose node,pod,ns names to ceph daemons

Ceph expects to be able to reason about physical hardware / nodes.
Before this patch the NODE_NAME that was available referred to the pod
name. This patch exposes the underlying node name, pod, and pod
namespace for the ceph daemons to consume.

fixes: #2078

Signed-off-by: Noah Watkins <nwatkins@redhat.com>
This commit is contained in:
Noah Watkins committed 2018-09-24 13:26:24 -07:00
1 parent 896ff184b5
commit e015a21cbd
11 files changed
+49 -22

No files matched your search

+1
View File
@@ -131,6 +131,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
Protocol: v1.ProtocolTCP,
},
},
Env: k8sutil.ClusterDaemonEnvVars(),
Resources: c.resources,
}
}
+1 -1
View File
@@ -108,7 +108,7 @@ func TestPodSpec(t *testing.T) {
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
daemonImage := "rook/rook:myversion"
daemonEnvs := 0
daemonEnvs := len(k8sutil.ClusterDaemonEnvVars())
daemonContainerDefinition := cephtest.ContainerTestDefinition{
Image: &daemonImage,
Command: []string{
+1
View File
@@ -254,6 +254,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
Protocol: v1.ProtocolTCP,
},
},
Env: k8sutil.ClusterDaemonEnvVars(),
Resources: c.resources,
}
}
+2 -1
View File
@@ -169,6 +169,7 @@ func testPodSpec(t *testing.T, dataDir string) {
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
// main mon daemon container
monDaemonEnvs := len(k8sutil.ClusterDaemonEnvVars())
monDaemonContDev := test_opceph.ContainerTestDefinition{
Image: &cephImage,
Command: []string{
@@ -178,7 +179,7 @@ func testPodSpec(t *testing.T, dataDir string) {
[]string{"--foreground"},
[]string{"--public-addr", "2.4.6.1:6790"}),
VolumeMountNames: cephVolumeMountNames,
EnvCount: &cephEnvs,
EnvCount: &monDaemonEnvs,
Ports: []v1.ContainerPort{
{ContainerPort: config.Port,
Protocol: v1.ProtocolTCP}},
+1
View File
@@ -129,6 +129,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
tiniEnvVar,
}
envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...)
configEnvVars := append(c.getConfigEnvVars(storeConfig, dataDir, location), []v1.EnvVar{
tiniEnvVar,
{Name: "ROOK_OSD_ID", Value: osdID},
@@ -23,6 +23,7 @@ import (
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
"github.com/rook/rook/pkg/operator/k8sutil"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -195,6 +196,8 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath)
assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath)
assert.Equal(t, (4 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env))
require.Equal(t, 1, len(podSpec.InitContainers))
initCont = podSpec.InitContainers[0]
require.Equal(t, 2, len(initCont.VolumeMounts))
+14 -11
View File
@@ -143,6 +143,19 @@ func makeDeployment(clientset kubernetes.Interface, fs cephv1beta1.Filesystem, f
func mdsContainer(fs cephv1beta1.Filesystem, filesystemID, version string) v1.Container {
envVars := []v1.EnvVar{
{Name: "ROOK_POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
{Name: "ROOK_FILESYSTEM_ID", Value: filesystemID},
{Name: "ROOK_ACTIVE_STANDBY", Value: strconv.FormatBool(fs.Spec.MetadataServer.ActiveStandby)},
opmon.ClusterNameEnvVar(fs.Namespace),
opmon.EndpointEnvVar(),
opmon.AdminSecretEnvVar(),
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
k8sutil.ConfigOverrideEnvVar(),
}
envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...)
return v1.Container{
Args: []string{
"ceph",
@@ -155,17 +168,7 @@ func mdsContainer(fs cephv1beta1.Filesystem, filesystemID, version string) v1.Co
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
},
Env: []v1.EnvVar{
{Name: "ROOK_POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
{Name: "ROOK_FILESYSTEM_ID", Value: filesystemID},
{Name: "ROOK_ACTIVE_STANDBY", Value: strconv.FormatBool(fs.Spec.MetadataServer.ActiveStandby)},
opmon.ClusterNameEnvVar(fs.Namespace),
opmon.EndpointEnvVar(),
opmon.AdminSecretEnvVar(),
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
k8sutil.ConfigOverrideEnvVar(),
},
Env: envVars,
Resources: fs.Spec.MetadataServer.Resources,
}
}
+3
View File
@@ -26,6 +26,7 @@ import (
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
"github.com/rook/rook/pkg/clusterd"
cephtest "github.com/rook/rook/pkg/daemon/ceph/test"
"github.com/rook/rook/pkg/operator/k8sutil"
testop "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
@@ -162,6 +163,8 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, "mds", cont.Args[1])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[2])
assert.Equal(t, (9 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env))
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
}
+12 -9
View File
@@ -326,6 +326,17 @@ func startDaemonset(context *clusterd.Context, store cephv1beta1.ObjectStore, ve
func rgwContainer(store cephv1beta1.ObjectStore, version string) v1.Container {
envVars := []v1.EnvVar{
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: instanceName(store)}, Key: keyringName}}},
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
opmon.ClusterNameEnvVar(store.Namespace),
opmon.EndpointEnvVar(),
opmon.SecretEnvVar(),
k8sutil.ConfigOverrideEnvVar(),
}
envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...)
container := v1.Container{
Args: []string{
"ceph",
@@ -341,15 +352,7 @@ func rgwContainer(store cephv1beta1.ObjectStore, version string) v1.Container {
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
},
Env: []v1.EnvVar{
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: instanceName(store)}, Key: keyringName}}},
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
opmon.ClusterNameEnvVar(store.Namespace),
opmon.EndpointEnvVar(),
opmon.SecretEnvVar(),
k8sutil.ConfigOverrideEnvVar(),
},
Env: envVars,
Resources: store.Spec.Gateway.Resources,
}
+2
View File
@@ -130,6 +130,8 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, fmt.Sprintf("--rgw-port=%d", 123), cont.Args[4])
assert.Equal(t, fmt.Sprintf("--rgw-secure-port=%d", 0), cont.Args[5])
assert.Equal(t, (7 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env))
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
}
+9
View File
@@ -262,3 +262,12 @@ func deleteResourceAndWait(namespace, name, resourceType string,
return fmt.Errorf("gave up waiting for %s pods to be terminated", name)
}
// Environment variables used by storage cluster daemons
func ClusterDaemonEnvVars() []v1.EnvVar {
return []v1.EnvVar{
{Name: "POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
{Name: "POD_NAMESPACE", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}},
{Name: "NODE_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}},
}
}