From e015a21cbdcf8241bf23bbb7210baf37b31faa83 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Wed, 12 Sep 2018 15:18:54 -0700 Subject: [PATCH] ceph: expose node,pod,ns names to ceph daemons Ceph expects to be able to reason about physical hardware / nodes. Before this patch the NODE_NAME that was available referred to the pod name. This patch exposes the underlying node name, pod, and pod namespace for the ceph daemons to consume. fixes: #2078 Signed-off-by: Noah Watkins --- pkg/operator/ceph/cluster/mgr/spec.go | 1 + pkg/operator/ceph/cluster/mgr/spec_test.go | 2 +- pkg/operator/ceph/cluster/mon/spec.go | 1 + pkg/operator/ceph/cluster/mon/spec_test.go | 3 ++- pkg/operator/ceph/cluster/osd/pod.go | 1 + pkg/operator/ceph/cluster/osd/pod_test.go | 3 +++ pkg/operator/ceph/file/mds.go | 25 ++++++++++++---------- pkg/operator/ceph/file/mds_test.go | 3 +++ pkg/operator/ceph/object/rgw.go | 21 ++++++++++-------- pkg/operator/ceph/object/rgw_test.go | 2 ++ pkg/operator/k8sutil/pod.go | 9 ++++++++ 11 files changed, 49 insertions(+), 22 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 8a99d15ad..5080c02f0 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -131,6 +131,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { Protocol: v1.ProtocolTCP, }, }, + Env: k8sutil.ClusterDaemonEnvVars(), Resources: c.resources, } } diff --git a/pkg/operator/ceph/cluster/mgr/spec_test.go b/pkg/operator/ceph/cluster/mgr/spec_test.go index 7ba517f09..2219e7808 100644 --- a/pkg/operator/ceph/cluster/mgr/spec_test.go +++ b/pkg/operator/ceph/cluster/mgr/spec_test.go @@ -108,7 +108,7 @@ func TestPodSpec(t *testing.T) { assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) daemonImage := "rook/rook:myversion" - daemonEnvs := 0 + daemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) daemonContainerDefinition := cephtest.ContainerTestDefinition{ Image: &daemonImage, Command: []string{ diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index b7f853eaf..616f6c2de 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -254,6 +254,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { Protocol: v1.ProtocolTCP, }, }, + Env: k8sutil.ClusterDaemonEnvVars(), Resources: c.resources, } } diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index 15884a607..9b297ea71 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -169,6 +169,7 @@ func testPodSpec(t *testing.T, dataDir string) { assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) // main mon daemon container + monDaemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) monDaemonContDev := test_opceph.ContainerTestDefinition{ Image: &cephImage, Command: []string{ @@ -178,7 +179,7 @@ func testPodSpec(t *testing.T, dataDir string) { []string{"--foreground"}, []string{"--public-addr", "2.4.6.1:6790"}), VolumeMountNames: cephVolumeMountNames, - EnvCount: &cephEnvs, + EnvCount: &monDaemonEnvs, Ports: []v1.ContainerPort{ {ContainerPort: config.Port, Protocol: v1.ProtocolTCP}}, diff --git a/pkg/operator/ceph/cluster/osd/pod.go b/pkg/operator/ceph/cluster/osd/pod.go index 88e58f981..b314c02f2 100644 --- a/pkg/operator/ceph/cluster/osd/pod.go +++ b/pkg/operator/ceph/cluster/osd/pod.go @@ -129,6 +129,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), tiniEnvVar, } + envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...) configEnvVars := append(c.getConfigEnvVars(storeConfig, dataDir, location), []v1.EnvVar{ tiniEnvVar, {Name: "ROOK_OSD_ID", Value: osdID}, diff --git a/pkg/operator/ceph/cluster/osd/pod_test.go b/pkg/operator/ceph/cluster/osd/pod_test.go index 3c1ebf723..45db4af29 100644 --- a/pkg/operator/ceph/cluster/osd/pod_test.go +++ b/pkg/operator/ceph/cluster/osd/pod_test.go @@ -23,6 +23,7 @@ import ( rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" + "github.com/rook/rook/pkg/operator/k8sutil" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -195,6 +196,8 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath) assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath) + assert.Equal(t, (4 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env)) + require.Equal(t, 1, len(podSpec.InitContainers)) initCont = podSpec.InitContainers[0] require.Equal(t, 2, len(initCont.VolumeMounts)) diff --git a/pkg/operator/ceph/file/mds.go b/pkg/operator/ceph/file/mds.go index b9a64179b..77e2b984f 100644 --- a/pkg/operator/ceph/file/mds.go +++ b/pkg/operator/ceph/file/mds.go @@ -143,6 +143,19 @@ func makeDeployment(clientset kubernetes.Interface, fs cephv1beta1.Filesystem, f func mdsContainer(fs cephv1beta1.Filesystem, filesystemID, version string) v1.Container { + envVars := []v1.EnvVar{ + {Name: "ROOK_POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + {Name: "ROOK_FILESYSTEM_ID", Value: filesystemID}, + {Name: "ROOK_ACTIVE_STANDBY", Value: strconv.FormatBool(fs.Spec.MetadataServer.ActiveStandby)}, + opmon.ClusterNameEnvVar(fs.Namespace), + opmon.EndpointEnvVar(), + opmon.AdminSecretEnvVar(), + k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), + k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), + k8sutil.ConfigOverrideEnvVar(), + } + envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...) + return v1.Container{ Args: []string{ "ceph", @@ -155,17 +168,7 @@ func mdsContainer(fs cephv1beta1.Filesystem, filesystemID, version string) v1.Co {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, k8sutil.ConfigOverrideMount(), }, - Env: []v1.EnvVar{ - {Name: "ROOK_POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, - {Name: "ROOK_FILESYSTEM_ID", Value: filesystemID}, - {Name: "ROOK_ACTIVE_STANDBY", Value: strconv.FormatBool(fs.Spec.MetadataServer.ActiveStandby)}, - opmon.ClusterNameEnvVar(fs.Namespace), - opmon.EndpointEnvVar(), - opmon.AdminSecretEnvVar(), - k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), - k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), - k8sutil.ConfigOverrideEnvVar(), - }, + Env: envVars, Resources: fs.Spec.MetadataServer.Resources, } } diff --git a/pkg/operator/ceph/file/mds_test.go b/pkg/operator/ceph/file/mds_test.go index 1a654c1b4..7bc98e734 100644 --- a/pkg/operator/ceph/file/mds_test.go +++ b/pkg/operator/ceph/file/mds_test.go @@ -26,6 +26,7 @@ import ( cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" "github.com/rook/rook/pkg/clusterd" cephtest "github.com/rook/rook/pkg/daemon/ceph/test" + "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -162,6 +163,8 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, "mds", cont.Args[1]) assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[2]) + assert.Equal(t, (9 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env)) + assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) } diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index e1449b6d7..687f0d6b0 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -326,6 +326,17 @@ func startDaemonset(context *clusterd.Context, store cephv1beta1.ObjectStore, ve func rgwContainer(store cephv1beta1.ObjectStore, version string) v1.Container { + envVars := []v1.EnvVar{ + {Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: instanceName(store)}, Key: keyringName}}}, + k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), + k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), + opmon.ClusterNameEnvVar(store.Namespace), + opmon.EndpointEnvVar(), + opmon.SecretEnvVar(), + k8sutil.ConfigOverrideEnvVar(), + } + envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...) + container := v1.Container{ Args: []string{ "ceph", @@ -341,15 +352,7 @@ func rgwContainer(store cephv1beta1.ObjectStore, version string) v1.Container { {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, k8sutil.ConfigOverrideMount(), }, - Env: []v1.EnvVar{ - {Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: instanceName(store)}, Key: keyringName}}}, - k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), - k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), - opmon.ClusterNameEnvVar(store.Namespace), - opmon.EndpointEnvVar(), - opmon.SecretEnvVar(), - k8sutil.ConfigOverrideEnvVar(), - }, + Env: envVars, Resources: store.Spec.Gateway.Resources, } diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 8b45ec740..c51f301fc 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -130,6 +130,8 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, fmt.Sprintf("--rgw-port=%d", 123), cont.Args[4]) assert.Equal(t, fmt.Sprintf("--rgw-secure-port=%d", 0), cont.Args[5]) + assert.Equal(t, (7 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env)) + assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) } diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index 174c5c88b..b5bb12da1 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -262,3 +262,12 @@ func deleteResourceAndWait(namespace, name, resourceType string, return fmt.Errorf("gave up waiting for %s pods to be terminated", name) } + +// Environment variables used by storage cluster daemons +func ClusterDaemonEnvVars() []v1.EnvVar { + return []v1.EnvVar{ + {Name: "POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + {Name: "POD_NAMESPACE", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, + {Name: "NODE_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}}, + } +}