ceph: expose node,pod,ns names to ceph daemons
Ceph expects to be able to reason about physical hardware / nodes. Before this patch the NODE_NAME that was available referred to the pod name. This patch exposes the underlying node name, pod, and pod namespace for the ceph daemons to consume. fixes: #2078 Signed-off-by: Noah Watkins <nwatkins@redhat.com>
This commit is contained in:
1 parent
896ff184b5
commit
e015a21cbd
11 files changed
+49
-22
No files matched your search
@@ -131,6 +131,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
Env: k8sutil.ClusterDaemonEnvVars(),
|
||||
Resources: c.resources,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,7 +108,7 @@ func TestPodSpec(t *testing.T) {
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
|
||||
daemonImage := "rook/rook:myversion"
|
||||
daemonEnvs := 0
|
||||
daemonEnvs := len(k8sutil.ClusterDaemonEnvVars())
|
||||
daemonContainerDefinition := cephtest.ContainerTestDefinition{
|
||||
Image: &daemonImage,
|
||||
Command: []string{
|
||||
|
||||
@@ -254,6 +254,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
Env: k8sutil.ClusterDaemonEnvVars(),
|
||||
Resources: c.resources,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +169,7 @@ func testPodSpec(t *testing.T, dataDir string) {
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
|
||||
// main mon daemon container
|
||||
monDaemonEnvs := len(k8sutil.ClusterDaemonEnvVars())
|
||||
monDaemonContDev := test_opceph.ContainerTestDefinition{
|
||||
Image: &cephImage,
|
||||
Command: []string{
|
||||
@@ -178,7 +179,7 @@ func testPodSpec(t *testing.T, dataDir string) {
|
||||
[]string{"--foreground"},
|
||||
[]string{"--public-addr", "2.4.6.1:6790"}),
|
||||
VolumeMountNames: cephVolumeMountNames,
|
||||
EnvCount: &cephEnvs,
|
||||
EnvCount: &monDaemonEnvs,
|
||||
Ports: []v1.ContainerPort{
|
||||
{ContainerPort: config.Port,
|
||||
Protocol: v1.ProtocolTCP}},
|
||||
|
||||
@@ -129,6 +129,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
tiniEnvVar,
|
||||
}
|
||||
envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...)
|
||||
configEnvVars := append(c.getConfigEnvVars(storeConfig, dataDir, location), []v1.EnvVar{
|
||||
tiniEnvVar,
|
||||
{Name: "ROOK_OSD_ID", Value: osdID},
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -195,6 +196,8 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
|
||||
assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath)
|
||||
assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath)
|
||||
|
||||
assert.Equal(t, (4 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env))
|
||||
|
||||
require.Equal(t, 1, len(podSpec.InitContainers))
|
||||
initCont = podSpec.InitContainers[0]
|
||||
require.Equal(t, 2, len(initCont.VolumeMounts))
|
||||
|
||||
@@ -143,6 +143,19 @@ func makeDeployment(clientset kubernetes.Interface, fs cephv1beta1.Filesystem, f
|
||||
|
||||
func mdsContainer(fs cephv1beta1.Filesystem, filesystemID, version string) v1.Container {
|
||||
|
||||
envVars := []v1.EnvVar{
|
||||
{Name: "ROOK_POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
|
||||
{Name: "ROOK_FILESYSTEM_ID", Value: filesystemID},
|
||||
{Name: "ROOK_ACTIVE_STANDBY", Value: strconv.FormatBool(fs.Spec.MetadataServer.ActiveStandby)},
|
||||
opmon.ClusterNameEnvVar(fs.Namespace),
|
||||
opmon.EndpointEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
k8sutil.ConfigOverrideEnvVar(),
|
||||
}
|
||||
envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...)
|
||||
|
||||
return v1.Container{
|
||||
Args: []string{
|
||||
"ceph",
|
||||
@@ -155,17 +168,7 @@ func mdsContainer(fs cephv1beta1.Filesystem, filesystemID, version string) v1.Co
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
k8sutil.ConfigOverrideMount(),
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
{Name: "ROOK_POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
|
||||
{Name: "ROOK_FILESYSTEM_ID", Value: filesystemID},
|
||||
{Name: "ROOK_ACTIVE_STANDBY", Value: strconv.FormatBool(fs.Spec.MetadataServer.ActiveStandby)},
|
||||
opmon.ClusterNameEnvVar(fs.Namespace),
|
||||
opmon.EndpointEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
k8sutil.ConfigOverrideEnvVar(),
|
||||
},
|
||||
Env: envVars,
|
||||
Resources: fs.Spec.MetadataServer.Resources,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephtest "github.com/rook/rook/pkg/daemon/ceph/test"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -162,6 +163,8 @@ func TestPodSpecs(t *testing.T) {
|
||||
assert.Equal(t, "mds", cont.Args[1])
|
||||
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[2])
|
||||
|
||||
assert.Equal(t, (9 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env))
|
||||
|
||||
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
}
|
||||
|
||||
@@ -326,6 +326,17 @@ func startDaemonset(context *clusterd.Context, store cephv1beta1.ObjectStore, ve
|
||||
|
||||
func rgwContainer(store cephv1beta1.ObjectStore, version string) v1.Container {
|
||||
|
||||
envVars := []v1.EnvVar{
|
||||
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: instanceName(store)}, Key: keyringName}}},
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
opmon.ClusterNameEnvVar(store.Namespace),
|
||||
opmon.EndpointEnvVar(),
|
||||
opmon.SecretEnvVar(),
|
||||
k8sutil.ConfigOverrideEnvVar(),
|
||||
}
|
||||
envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...)
|
||||
|
||||
container := v1.Container{
|
||||
Args: []string{
|
||||
"ceph",
|
||||
@@ -341,15 +352,7 @@ func rgwContainer(store cephv1beta1.ObjectStore, version string) v1.Container {
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
k8sutil.ConfigOverrideMount(),
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: instanceName(store)}, Key: keyringName}}},
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
opmon.ClusterNameEnvVar(store.Namespace),
|
||||
opmon.EndpointEnvVar(),
|
||||
opmon.SecretEnvVar(),
|
||||
k8sutil.ConfigOverrideEnvVar(),
|
||||
},
|
||||
Env: envVars,
|
||||
Resources: store.Spec.Gateway.Resources,
|
||||
}
|
||||
|
||||
|
||||
@@ -130,6 +130,8 @@ func TestPodSpecs(t *testing.T) {
|
||||
assert.Equal(t, fmt.Sprintf("--rgw-port=%d", 123), cont.Args[4])
|
||||
assert.Equal(t, fmt.Sprintf("--rgw-secure-port=%d", 0), cont.Args[5])
|
||||
|
||||
assert.Equal(t, (7 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env))
|
||||
|
||||
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
}
|
||||
|
||||
@@ -262,3 +262,12 @@ func deleteResourceAndWait(namespace, name, resourceType string,
|
||||
|
||||
return fmt.Errorf("gave up waiting for %s pods to be terminated", name)
|
||||
}
|
||||
|
||||
// Environment variables used by storage cluster daemons
|
||||
func ClusterDaemonEnvVars() []v1.EnvVar {
|
||||
return []v1.EnvVar{
|
||||
{Name: "POD_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
|
||||
{Name: "POD_NAMESPACE", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}},
|
||||
{Name: "NODE_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}},
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user