From 7f3e6bd7fae4785a33e047dc22752bc6daf7f558 Mon Sep 17 00:00:00 2001 From: Patryk Rostkowski Date: Sun, 11 May 2025 10:07:37 +0200 Subject: [PATCH] mon: allow running mon pods as root This change addresses a permission issue where mon pods crashloop on some Kubernetes setups with SELinux enabled, even when ROOK_HOSTPATH_REQUIRES_PRIVILEGED is set. To resolve this, a new function makeMonSecurityContext() was introduced to explicitly set runAsUser: 0 at the pod level for mon pods when the environment variable ROOK_CEPH_MON_RUN_AS_ROOT is set to true. Additionally, the function previously named PodSecurityContext() was renamed to DefaultContainerSecurityContext() to avoid confusion between container-level and pod-level security context configuration. All container SecurityContext usages across Ceph daemons were updated to reflect this change. This ensures the root user configuration is applied automatically and consistently in environments where it is required for mon pod startup, while preserving clear separation of container and pod-level security logic. Signed-off-by: Patryk Rostkowski --- deploy/examples/operator.yaml | 5 +++++ pkg/operator/ceph/cluster/mgr/spec.go | 8 ++++---- pkg/operator/ceph/cluster/mon/spec.go | 18 ++++++++++++++---- pkg/operator/ceph/cluster/mon/spec_test.go | 17 +++++++++++++++++ pkg/operator/ceph/cluster/nodedaemon/crash.go | 4 ++-- .../ceph/cluster/nodedaemon/exporter.go | 4 ++-- pkg/operator/ceph/cluster/nodedaemon/pruner.go | 2 +- pkg/operator/ceph/cluster/rbd/spec.go | 4 ++-- pkg/operator/ceph/controller/spec.go | 14 +++++++++----- pkg/operator/ceph/file/mds/spec.go | 4 ++-- pkg/operator/ceph/file/mirror/spec.go | 4 ++-- pkg/operator/ceph/nfs/spec.go | 4 ++-- pkg/operator/ceph/object/spec.go | 8 ++++---- 13 files changed, 66 insertions(+), 30 deletions(-) diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 93e068442..486bb6354 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -638,6 +638,11 @@ spec: # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED value: "false" + # Whether to run Ceph mon pods as root (runAsUser: 0) at the pod level. + # Set this to true if mon pods are CrashLooping on SELinux-enabled systems despite other settings. + # For more context, see https://github.com/rook/rook/issues/15564 + - name: ROOK_CEPH_MON_RUN_AS_ROOT + value: "false" # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". # In case of more than one regex, use comma to separate between them. # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 01e747320..5d309166a 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -159,7 +159,7 @@ func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container { controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), cephv1.GetMgrResources(c.spec.Resources), - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -203,7 +203,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { c.cephMgrOrchestratorModuleEnvs()..., ), Resources: cephv1.GetMgrResources(c.spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), WorkingDir: config.VarLogCephDir, @@ -259,7 +259,7 @@ func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container { ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), Env: envVars, Resources: cephv1.GetMgrSidecarResources(c.spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), VolumeMounts: volumeMounts, } } @@ -275,7 +275,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe VolumeMounts: append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount), Env: append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}), Resources: cephv1.GetMgrResources(c.spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } return container diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 09ad8bc85..a99cc8f24 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -32,6 +32,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) const ( @@ -176,6 +177,15 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*corev1.Persiste return pvc, nil } +func makeMonSecurityContext() *corev1.PodSecurityContext { + if controller.CephMonRunAsRoot() { + return &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(0)), + } + } + return &corev1.PodSecurityContext{} +} + func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, error) { logger.Debugf("monConfig: %+v", monConfig) podSpec := corev1.PodSpec{ @@ -192,7 +202,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, er Volumes: controller.DaemonVolumesBase(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), HostNetwork: monConfig.UseHostNetwork, PriorityClassName: cephv1.GetMonPriorityClassName(c.spec.PriorityClassNames), - SecurityContext: &corev1.PodSecurityContext{}, + SecurityContext: makeMonSecurityContext(), ServiceAccountName: k8sutil.DefaultServiceAccount, } @@ -257,7 +267,7 @@ func (c *Cluster) makeChownInitContainer(monConfig *monConfig) corev1.Container controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), cephv1.GetMonResources(c.spec.Resources), - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -278,7 +288,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container Image: c.spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), // filesystem creation does not require ports to be exposed Env: controller.DaemonEnvVars(&c.spec), Resources: cephv1.GetMonResources(c.spec.Resources), @@ -310,7 +320,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container Image: c.spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), Ports: []corev1.ContainerPort{ { Name: DefaultMsgr2PortName, diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index 1384d90de..abd0d0b36 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -322,3 +322,20 @@ func TestGetFailureDomainLabel(t *testing.T) { }) } } + +func TestMakeMonSecurityContext(t *testing.T) { + t.Run("returns root context when env is true", func(t *testing.T) { + t.Setenv("ROOK_CEPH_MON_RUN_AS_ROOT", "true") + sc := makeMonSecurityContext() + assert.NotNil(t, sc) + assert.NotNil(t, sc.RunAsUser) + assert.Equal(t, int64(0), *sc.RunAsUser) + }) + + t.Run("returns default context when env is not true", func(t *testing.T) { + t.Setenv("ROOK_CEPH_MON_RUN_AS_ROOT", "false") + sc := makeMonSecurityContext() + assert.NotNil(t, sc) + assert.Nil(t, sc.RunAsUser) + }) +} diff --git a/pkg/operator/ceph/cluster/nodedaemon/crash.go b/pkg/operator/ceph/cluster/nodedaemon/crash.go index 86c4eeb85..ddceb2c78 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/crash.go +++ b/pkg/operator/ceph/cluster/nodedaemon/crash.go @@ -147,7 +147,7 @@ func getCrashDirInitContainer(cephCluster cephv1.CephCluster) corev1.Container { }, Image: cephCluster.Spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources), VolumeMounts: controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath), } @@ -163,7 +163,7 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath), cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources), - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index 987a49d43..198c6e4e5 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -164,7 +164,7 @@ func getCephExporterChownInitContainer(cephCluster cephv1.CephCluster) corev1.Co controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), mounts, cephv1.GetCephExporterResources(cephCluster.Spec.Resources), - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -211,7 +211,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion Ports: []corev1.ContainerPort{containerPort}, VolumeMounts: volumeMounts, Resources: cephv1.GetCephExporterResources(cephCluster.Spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } return container diff --git a/pkg/operator/ceph/cluster/nodedaemon/pruner.go b/pkg/operator/ceph/cluster/nodedaemon/pruner.go index f072a3136..19f8714bf 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/pruner.go +++ b/pkg/operator/ceph/cluster/nodedaemon/pruner.go @@ -144,7 +144,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster) corev1.Container { Env: envVars, VolumeMounts: volumeMounts, Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } return container diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index cb78011da..f080e2775 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -107,7 +107,7 @@ func (r *ReconcileCephRBDMirror) makeChownInitContainer(daemonConfig *daemonConf controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), rbdMirror.Spec.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -128,7 +128,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: rbdMirror.Spec.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), WorkingDir: config.VarLogCephDir, // TODO: // Not implemented at this point since the socket name is '/run/ceph/ceph-client.rbd-mirror.a.1.94362516231272.asok' diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 699d68d1c..07916d56b 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -726,8 +726,12 @@ func HostPathRequiresPrivileged() bool { return os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true" } -// PodSecurityContext detects if the pod needs privileges to run -func PodSecurityContext() *v1.SecurityContext { +func CephMonRunAsRoot() bool { + return os.Getenv("ROOK_CEPH_MON_RUN_AS_ROOT") == "true" +} + +// DefaultContainerSecurityContext detects if the container needs privileges to run +func DefaultContainerSecurityContext() *v1.SecurityContext { privileged := HostPathRequiresPrivileged() return &v1.SecurityContext{ @@ -743,7 +747,7 @@ func PodSecurityContext() *v1.SecurityContext { // PodSecurityContext detects if the pod needs privileges to run func CephSecurityContext() *v1.SecurityContext { - context := PodSecurityContext() + context := DefaultContainerSecurityContext() cephUserID := CephUserID context.RunAsUser = &cephUserID context.RunAsGroup = &cephUserID @@ -824,7 +828,7 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, env []v1.E Image: c.CephVersion.Image, ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy), VolumeMounts: DaemonVolumeMounts(opconfig.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath), - SecurityContext: PodSecurityContext(), + SecurityContext: DefaultContainerSecurityContext(), Resources: cephv1.GetLogCollectorResources(c.Resources), // We need a TTY for the bash job control (enabled by -m) TTY: true, @@ -845,7 +849,7 @@ func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, env Image: c.CephVersion.Image, ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy), VolumeMounts: DaemonVolumeMounts(opconfig.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath), - SecurityContext: PodSecurityContext(), + SecurityContext: DefaultContainerSecurityContext(), Resources: Resources, // We need a TTY for the bash job control (enabled by -m) TTY: true, diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index ac54eba58..febbf98ed 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -126,7 +126,7 @@ func (c *Cluster) makeChownInitContainer(mdsConfig *mdsConfig) v1.Container { controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath), c.fs.Spec.MetadataServer.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -153,7 +153,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig, fsName string) v1 VolumeMounts: controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath), Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)), Resources: c.fs.Spec.MetadataServer.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), // StartupProbe time for MDS is covered liveness probe LivenessProbe: generateMDSLivenessProbeExecDaemon(mdsConfig.DaemonID, fsName, keyring.VolumeMount().KeyringFilePath()), WorkingDir: cephconfig.VarLogCephDir, diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go index 45fdc9a38..4aef52949 100644 --- a/pkg/operator/ceph/file/mirror/spec.go +++ b/pkg/operator/ceph/file/mirror/spec.go @@ -104,7 +104,7 @@ func (r *ReconcileFilesystemMirror) makeChownInitContainer(daemonConfig *daemonC controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), fsMirror.Spec.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -125,7 +125,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: fsMirror.Spec.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), // TODO: // LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.fsMirrorType, daemonConfig.DaemonID), } diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index e5d9fb1a0..b761e9eea 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -216,7 +216,7 @@ func (r *ReconcileCephNFS) connectionConfigInitContainer(nfs *cephv1.CephNFS, na keyring.VolumeMount().Resource(instanceName(nfs, name)), }, nfs.Spec.Server.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), ) } @@ -250,7 +250,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig }, Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), LivenessProbe: r.defaultGaneshaLivenessProbe(nfs), } return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe) diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 454e2d16c..16fefa340 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -330,7 +330,7 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig) ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: volumeMounts, Resources: c.store.Spec.Gateway.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } } @@ -367,7 +367,7 @@ func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig, kmsEnabled VolumeMounts: append( controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), vaultVolMounts...), Resources: c.store.Spec.Gateway.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } } @@ -378,7 +378,7 @@ func (c *clusterConfig) makeChownInitContainer(rgwConfig *rgwConfig) v1.Containe controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), c.store.Spec.Gateway.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -420,7 +420,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, StartupProbe: startupProbe, LivenessProbe: noLivenessProbe(), ReadinessProbe: readinessProbe, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), WorkingDir: cephconfig.VarLogCephDir, }