From 64fecc75643d13d29c32a95dd8a63bff6fe88dae Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Fri, 25 Aug 2023 19:58:46 +0530 Subject: [PATCH] core: restart ceph daemons when network updated We need to restart all the ceph daemons whenever cephCluster network settings are modified like requiremsgr2, encryption and compression. This required for Ceph to consider the new settings it require new ceph daemons all over. Signed-off-by: subhamkrai (cherry picked from commit 29d2b6a0715881a19edcda381a68a4ca6895db87) --- pkg/operator/ceph/cluster/mgr/spec.go | 4 +- pkg/operator/ceph/cluster/mon/spec.go | 4 +- pkg/operator/ceph/cluster/nodedaemon/crash.go | 5 +- .../ceph/cluster/nodedaemon/exporter.go | 2 +- .../ceph/cluster/nodedaemon/pruner.go | 5 +- pkg/operator/ceph/cluster/osd/spec.go | 1 + pkg/operator/ceph/cluster/rbd/spec.go | 2 +- pkg/operator/ceph/controller/spec.go | 32 ++++++- pkg/operator/ceph/controller/spec_test.go | 94 +++++++++++++++++++ pkg/operator/ceph/file/mds/spec.go | 2 +- pkg/operator/ceph/file/mirror/spec.go | 2 +- pkg/operator/ceph/nfs/spec.go | 2 +- pkg/operator/ceph/object/spec.go | 2 +- 13 files changed, 139 insertions(+), 18 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index bf616e5ee..d6f8d24f3 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -184,7 +184,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { }, }, Env: append( - controller.DaemonEnvVars(c.spec.CephVersion.Image), + controller.DaemonEnvVars(&c.spec), c.cephMgrOrchestratorModuleEnvs()..., ), Resources: cephv1.GetMgrResources(c.spec.Resources), @@ -246,7 +246,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe Image: c.spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), VolumeMounts: append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount), - Env: append(controller.DaemonEnvVars(c.spec.CephVersion.Image), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}), + Env: append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}), Resources: cephv1.GetMgrResources(c.spec.Resources), SecurityContext: controller.PodSecurityContext(), } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 8d4c8ab81..b146914f4 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -269,7 +269,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), SecurityContext: controller.PodSecurityContext(), // filesystem creation does not require ports to be exposed - Env: controller.DaemonEnvVars(c.spec.CephVersion.Image), + Env: controller.DaemonEnvVars(&c.spec), Resources: cephv1.GetMonResources(c.spec.Resources), } } @@ -308,7 +308,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container }, }, Env: append( - controller.DaemonEnvVars(c.spec.CephVersion.Image), + controller.DaemonEnvVars(&c.spec), k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: cephv1.GetMonResources(c.spec.Resources), diff --git a/pkg/operator/ceph/cluster/nodedaemon/crash.go b/pkg/operator/ceph/cluster/nodedaemon/crash.go index 0e40b1912..83757853e 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/crash.go +++ b/pkg/operator/ceph/cluster/nodedaemon/crash.go @@ -167,10 +167,9 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container } func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container { - cephImage := cephCluster.Spec.CephVersion.Image dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath) crashEnvVar := generateCrashEnvVar() - envVars := append(controller.DaemonEnvVars(cephImage), crashEnvVar) + envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), crashEnvVar) volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath) volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector()) @@ -179,7 +178,7 @@ func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver Command: []string{ "ceph-crash", }, - Image: cephImage, + Image: cephCluster.Spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), Env: envVars, VolumeMounts: volumeMounts, diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index e8d8573db..7f6134b41 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -171,7 +171,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin()) envVars := append( - controller.DaemonEnvVars(cephCluster.Spec.CephVersion.Image), + controller.DaemonEnvVars(&cephCluster.Spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}) container := corev1.Container{ diff --git a/pkg/operator/ceph/cluster/nodedaemon/pruner.go b/pkg/operator/ceph/cluster/nodedaemon/pruner.go index cb033130e..d03b525b2 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/pruner.go +++ b/pkg/operator/ceph/cluster/nodedaemon/pruner.go @@ -170,8 +170,7 @@ func (r *ReconcileNode) deletev1betaJob(objectMeta metav1.ObjectMeta) { } func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container { - cephImage := cephCluster.Spec.CephVersion.Image - envVars := append(controller.DaemonEnvVars(cephImage), generateCrashEnvVar()) + envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), generateCrashEnvVar()) dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath) volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath) volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector()) @@ -188,7 +187,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver. Args: []string{ fmt.Sprintf("%d", cephCluster.Spec.CrashCollector.DaysToRetain), }, - Image: cephImage, + Image: cephCluster.Spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), Env: envVars, VolumeMounts: volumeMounts, diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 3f3f9fbe0..0cf055427 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -583,6 +583,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC "", )) + envVars = append(envVars, controller.ApplyNetworkEnv(&c.spec)...) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: AppName, diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index 72d1eaf59..6abd32712 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -122,7 +122,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem Image: r.cephClusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: rbdMirror.Spec.Resources, SecurityContext: controller.PodSecurityContext(), WorkingDir: config.VarLogCephDir, diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 7e919e7b6..573c89628 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -394,13 +394,41 @@ func ContainerEnvVarReference(envVarName string) string { } // DaemonEnvVars returns the container environment variables used by all Ceph daemons. -func DaemonEnvVars(image string) []v1.EnvVar { +func DaemonEnvVars(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar { + networkEnv := ApplyNetworkEnv(cephClusterSpec) + cephDaemonsEnvVars := append(k8sutil.ClusterDaemonEnvVars(cephClusterSpec.CephVersion.Image), networkEnv...) + return append( - k8sutil.ClusterDaemonEnvVars(image), + cephDaemonsEnvVars, config.StoredMonHostEnvVars()..., ) } +func ApplyNetworkEnv(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar { + if cephClusterSpec.Network.Connections != nil { + msgr2Required := false + encryptionEnabled := false + compressionEnabled := false + if cephClusterSpec.Network.Connections.RequireMsgr2 { + msgr2Required = true + } + if cephClusterSpec.Network.Connections.Encryption != nil && cephClusterSpec.Network.Connections.Encryption.Enabled { + encryptionEnabled = true + } + if cephClusterSpec.Network.Connections.Compression != nil && cephClusterSpec.Network.Connections.Compression.Enabled { + compressionEnabled = true + } + envVarValue := fmt.Sprintf("msgr2_%t_encryption_%t_compression_%t", msgr2Required, encryptionEnabled, compressionEnabled) + + rookMsgr2Env := []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: envVarValue, + }} + return rookMsgr2Env + } + return []v1.EnvVar{} +} + // AppLabels returns labels common for all Rook-Ceph applications which may be useful for admins. // App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc. func AppLabels(appName, namespace string) map[string]string { diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index a2697daaa..fa82bd7d8 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -463,3 +463,97 @@ func TestGetContainerImagePullPolicy(t *testing.T) { assert.Equal(t, exepctedImagePullPolicy, imagePullPolicy) }) } + +func TestDaemonEnvVars(t *testing.T) { + // No network settings specified + want := []v1.EnvVar{} + + clusterSpec := &cephv1.ClusterSpec{} + got := ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Encryption is enabled + connections := &cephv1.ConnectionsSpec{ + Encryption: &cephv1.EncryptionSpec{Enabled: true}, + } + + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_false_encryption_true_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Compression is enabled + connections = &cephv1.ConnectionsSpec{ + Compression: &cephv1.CompressionSpec{ + Enabled: true, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_false_encryption_false_compression_true", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2 is enabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_false_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2, Compression, Encryption are enabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + Encryption: &cephv1.EncryptionSpec{ + Enabled: true, + }, + Compression: &cephv1.CompressionSpec{ + Enabled: true, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_true_compression_true", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2 is enabled but Compression, Encryption are disabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + Encryption: &cephv1.EncryptionSpec{ + Enabled: false, + }, + Compression: &cephv1.CompressionSpec{ + Enabled: false, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_false_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) +} diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 2f4edfe02..2136fe46d 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -147,7 +147,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { Image: c.clusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath), - Env: append(controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), k8sutil.PodIPEnvVar(podIPEnvVar)), + Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)), Resources: c.fs.Spec.MetadataServer.Resources, SecurityContext: controller.PodSecurityContext(), StartupProbe: controller.GenerateStartupProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID), diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go index e750d02d8..72212573b 100644 --- a/pkg/operator/ceph/file/mirror/spec.go +++ b/pkg/operator/ceph/file/mirror/spec.go @@ -119,7 +119,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig Image: r.cephClusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: fsMirror.Spec.Resources, SecurityContext: controller.PodSecurityContext(), // TODO: diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 068b82ed3..dd03819c8 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -243,7 +243,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig nfsConfigMount, dbusMount, }, - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, SecurityContext: controller.PodSecurityContext(), } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index ac33b7171..b85e72333 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -359,7 +359,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), c.mimeTypesVolumeMount(), ), - Env: controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(c.clusterSpec), Resources: c.store.Spec.Gateway.Resources, StartupProbe: startupProbe, LivenessProbe: noLivenessProbe(),