mon: allow running mon pods as root

This change addresses a permission issue where mon pods crashloop
on some Kubernetes setups with SELinux enabled, even when
ROOK_HOSTPATH_REQUIRES_PRIVILEGED is set.

To resolve this, a new function makeMonSecurityContext() was introduced
to explicitly set runAsUser: 0 at the pod level for mon pods
when the environment variable ROOK_CEPH_MON_RUN_AS_ROOT is set to true.

Additionally, the function previously named PodSecurityContext() was renamed
to DefaultContainerSecurityContext() to avoid confusion between
container-level and pod-level security context configuration. All container
SecurityContext usages across Ceph daemons were updated to reflect this change.

This ensures the root user configuration is applied
automatically and consistently in environments where it is required
for mon pod startup, while preserving clear separation of container
and pod-level security logic.

Signed-off-by: Patryk Rostkowski <patrostkowski@gmail.com>
This commit is contained in:
Patryk Rostkowski committed 2025-06-16 18:29:31 +02:00
1 parent 3b5f7c4ca5
commit 7f3e6bd7fa
13 files changed
+66 -30

No files matched your search

+5
View File
@@ -638,6 +638,11 @@ spec:
# For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
- name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
value: "false"
# Whether to run Ceph mon pods as root (runAsUser: 0) at the pod level.
# Set this to true if mon pods are CrashLooping on SELinux-enabled systems despite other settings.
# For more context, see https://github.com/rook/rook/issues/15564
- name: ROOK_CEPH_MON_RUN_AS_ROOT
value: "false"
# Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+".
# In case of more than one regex, use comma to separate between them.
# Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
+4 -4
View File
@@ -159,7 +159,7 @@ func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container {
controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath),
cephv1.GetMgrResources(c.spec.Resources),
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
"",
)
}
@@ -203,7 +203,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
c.cephMgrOrchestratorModuleEnvs()...,
),
Resources: cephv1.GetMgrResources(c.spec.Resources),
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
WorkingDir: config.VarLogCephDir,
@@ -259,7 +259,7 @@ func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container {
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
Env: envVars,
Resources: cephv1.GetMgrSidecarResources(c.spec.Resources),
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
VolumeMounts: volumeMounts,
}
}
@@ -275,7 +275,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe
VolumeMounts: append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount),
Env: append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}),
Resources: cephv1.GetMgrResources(c.spec.Resources),
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
}
return container
+14 -4
View File
@@ -32,6 +32,7 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
)
const (
@@ -176,6 +177,15 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*corev1.Persiste
return pvc, nil
}
func makeMonSecurityContext() *corev1.PodSecurityContext {
if controller.CephMonRunAsRoot() {
return &corev1.PodSecurityContext{
RunAsUser: ptr.To(int64(0)),
}
}
return &corev1.PodSecurityContext{}
}
func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, error) {
logger.Debugf("monConfig: %+v", monConfig)
podSpec := corev1.PodSpec{
@@ -192,7 +202,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, er
Volumes: controller.DaemonVolumesBase(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
HostNetwork: monConfig.UseHostNetwork,
PriorityClassName: cephv1.GetMonPriorityClassName(c.spec.PriorityClassNames),
SecurityContext: &corev1.PodSecurityContext{},
SecurityContext: makeMonSecurityContext(),
ServiceAccountName: k8sutil.DefaultServiceAccount,
}
@@ -257,7 +267,7 @@ func (c *Cluster) makeChownInitContainer(monConfig *monConfig) corev1.Container
controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
cephv1.GetMonResources(c.spec.Resources),
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
"",
)
}
@@ -278,7 +288,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container
Image: c.spec.CephVersion.Image,
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
// filesystem creation does not require ports to be exposed
Env: controller.DaemonEnvVars(&c.spec),
Resources: cephv1.GetMonResources(c.spec.Resources),
@@ -310,7 +320,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
Image: c.spec.CephVersion.Image,
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
Ports: []corev1.ContainerPort{
{
Name: DefaultMsgr2PortName,
@@ -322,3 +322,20 @@ func TestGetFailureDomainLabel(t *testing.T) {
})
}
}
func TestMakeMonSecurityContext(t *testing.T) {
t.Run("returns root context when env is true", func(t *testing.T) {
t.Setenv("ROOK_CEPH_MON_RUN_AS_ROOT", "true")
sc := makeMonSecurityContext()
assert.NotNil(t, sc)
assert.NotNil(t, sc.RunAsUser)
assert.Equal(t, int64(0), *sc.RunAsUser)
})
t.Run("returns default context when env is not true", func(t *testing.T) {
t.Setenv("ROOK_CEPH_MON_RUN_AS_ROOT", "false")
sc := makeMonSecurityContext()
assert.NotNil(t, sc)
assert.Nil(t, sc.RunAsUser)
})
}
@@ -147,7 +147,7 @@ func getCrashDirInitContainer(cephCluster cephv1.CephCluster) corev1.Container {
},
Image: cephCluster.Spec.CephVersion.Image,
ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources),
VolumeMounts: controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath),
}
@@ -163,7 +163,7 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container
controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath),
cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources),
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
"",
)
}
@@ -164,7 +164,7 @@ func getCephExporterChownInitContainer(cephCluster cephv1.CephCluster) corev1.Co
controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
mounts,
cephv1.GetCephExporterResources(cephCluster.Spec.Resources),
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
"",
)
}
@@ -211,7 +211,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
Ports: []corev1.ContainerPort{containerPort},
VolumeMounts: volumeMounts,
Resources: cephv1.GetCephExporterResources(cephCluster.Spec.Resources),
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
}
return container
@@ -144,7 +144,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster) corev1.Container {
Env: envVars,
VolumeMounts: volumeMounts,
Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources),
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
}
return container
+2 -2
View File
@@ -107,7 +107,7 @@ func (r *ReconcileCephRBDMirror) makeChownInitContainer(daemonConfig *daemonConf
controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy),
controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
rbdMirror.Spec.Resources,
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
"",
)
}
@@ -128,7 +128,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem
VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
Env: controller.DaemonEnvVars(r.cephClusterSpec),
Resources: rbdMirror.Spec.Resources,
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
WorkingDir: config.VarLogCephDir,
// TODO:
// Not implemented at this point since the socket name is '/run/ceph/ceph-client.rbd-mirror.a.1.94362516231272.asok'
+9 -5
View File
@@ -726,8 +726,12 @@ func HostPathRequiresPrivileged() bool {
return os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true"
}
// PodSecurityContext detects if the pod needs privileges to run
func PodSecurityContext() *v1.SecurityContext {
func CephMonRunAsRoot() bool {
return os.Getenv("ROOK_CEPH_MON_RUN_AS_ROOT") == "true"
}
// DefaultContainerSecurityContext detects if the container needs privileges to run
func DefaultContainerSecurityContext() *v1.SecurityContext {
privileged := HostPathRequiresPrivileged()
return &v1.SecurityContext{
@@ -743,7 +747,7 @@ func PodSecurityContext() *v1.SecurityContext {
// PodSecurityContext detects if the pod needs privileges to run
func CephSecurityContext() *v1.SecurityContext {
context := PodSecurityContext()
context := DefaultContainerSecurityContext()
cephUserID := CephUserID
context.RunAsUser = &cephUserID
context.RunAsGroup = &cephUserID
@@ -824,7 +828,7 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, env []v1.E
Image: c.CephVersion.Image,
ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy),
VolumeMounts: DaemonVolumeMounts(opconfig.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath),
SecurityContext: PodSecurityContext(),
SecurityContext: DefaultContainerSecurityContext(),
Resources: cephv1.GetLogCollectorResources(c.Resources),
// We need a TTY for the bash job control (enabled by -m)
TTY: true,
@@ -845,7 +849,7 @@ func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, env
Image: c.CephVersion.Image,
ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy),
VolumeMounts: DaemonVolumeMounts(opconfig.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath),
SecurityContext: PodSecurityContext(),
SecurityContext: DefaultContainerSecurityContext(),
Resources: Resources,
// We need a TTY for the bash job control (enabled by -m)
TTY: true,
+2 -2
View File
@@ -126,7 +126,7 @@ func (c *Cluster) makeChownInitContainer(mdsConfig *mdsConfig) v1.Container {
controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath),
c.fs.Spec.MetadataServer.Resources,
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
"",
)
}
@@ -153,7 +153,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig, fsName string) v1
VolumeMounts: controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath),
Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)),
Resources: c.fs.Spec.MetadataServer.Resources,
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
// StartupProbe time for MDS is covered liveness probe
LivenessProbe: generateMDSLivenessProbeExecDaemon(mdsConfig.DaemonID, fsName, keyring.VolumeMount().KeyringFilePath()),
WorkingDir: cephconfig.VarLogCephDir,
+2 -2
View File
@@ -104,7 +104,7 @@ func (r *ReconcileFilesystemMirror) makeChownInitContainer(daemonConfig *daemonC
controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy),
controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
fsMirror.Spec.Resources,
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
"",
)
}
@@ -125,7 +125,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig
VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
Env: controller.DaemonEnvVars(r.cephClusterSpec),
Resources: fsMirror.Spec.Resources,
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
// TODO:
// LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.fsMirrorType, daemonConfig.DaemonID),
}
+2 -2
View File
@@ -216,7 +216,7 @@ func (r *ReconcileCephNFS) connectionConfigInitContainer(nfs *cephv1.CephNFS, na
keyring.VolumeMount().Resource(instanceName(nfs, name)),
},
nfs.Spec.Server.Resources,
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
)
}
@@ -250,7 +250,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
},
Env: controller.DaemonEnvVars(r.cephClusterSpec),
Resources: nfs.Spec.Server.Resources,
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
LivenessProbe: r.defaultGaneshaLivenessProbe(nfs),
}
return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe)
+4 -4
View File
@@ -330,7 +330,7 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig)
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
VolumeMounts: volumeMounts,
Resources: c.store.Spec.Gateway.Resources,
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
}
}
@@ -367,7 +367,7 @@ func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig, kmsEnabled
VolumeMounts: append(
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), vaultVolMounts...),
Resources: c.store.Spec.Gateway.Resources,
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
}
}
@@ -378,7 +378,7 @@ func (c *clusterConfig) makeChownInitContainer(rgwConfig *rgwConfig) v1.Containe
controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath),
c.store.Spec.Gateway.Resources,
controller.PodSecurityContext(),
controller.DefaultContainerSecurityContext(),
"",
)
}
@@ -420,7 +420,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
StartupProbe: startupProbe,
LivenessProbe: noLivenessProbe(),
ReadinessProbe: readinessProbe,
SecurityContext: controller.PodSecurityContext(),
SecurityContext: controller.DefaultContainerSecurityContext(),
WorkingDir: cephconfig.VarLogCephDir,
}