mon: allow running mon pods as root
This change addresses a permission issue where mon pods crashloop on some Kubernetes setups with SELinux enabled, even when ROOK_HOSTPATH_REQUIRES_PRIVILEGED is set. To resolve this, a new function makeMonSecurityContext() was introduced to explicitly set runAsUser: 0 at the pod level for mon pods when the environment variable ROOK_CEPH_MON_RUN_AS_ROOT is set to true. Additionally, the function previously named PodSecurityContext() was renamed to DefaultContainerSecurityContext() to avoid confusion between container-level and pod-level security context configuration. All container SecurityContext usages across Ceph daemons were updated to reflect this change. This ensures the root user configuration is applied automatically and consistently in environments where it is required for mon pod startup, while preserving clear separation of container and pod-level security logic. Signed-off-by: Patryk Rostkowski <patrostkowski@gmail.com>
This commit is contained in:
13 files changed
+66
-30
No files matched your search
@@ -638,6 +638,11 @@ spec:
|
||||
# For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
|
||||
- name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
|
||||
value: "false"
|
||||
# Whether to run Ceph mon pods as root (runAsUser: 0) at the pod level.
|
||||
# Set this to true if mon pods are CrashLooping on SELinux-enabled systems despite other settings.
|
||||
# For more context, see https://github.com/rook/rook/issues/15564
|
||||
- name: ROOK_CEPH_MON_RUN_AS_ROOT
|
||||
value: "false"
|
||||
# Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+".
|
||||
# In case of more than one regex, use comma to separate between them.
|
||||
# Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
|
||||
|
||||
@@ -159,7 +159,7 @@ func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
|
||||
controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath),
|
||||
cephv1.GetMgrResources(c.spec.Resources),
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
"",
|
||||
)
|
||||
}
|
||||
@@ -203,7 +203,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
c.cephMgrOrchestratorModuleEnvs()...,
|
||||
),
|
||||
Resources: cephv1.GetMgrResources(c.spec.Resources),
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
|
||||
LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
|
||||
WorkingDir: config.VarLogCephDir,
|
||||
@@ -259,7 +259,7 @@ func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
|
||||
Env: envVars,
|
||||
Resources: cephv1.GetMgrSidecarResources(c.spec.Resources),
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
VolumeMounts: volumeMounts,
|
||||
}
|
||||
}
|
||||
@@ -275,7 +275,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe
|
||||
VolumeMounts: append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount),
|
||||
Env: append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}),
|
||||
Resources: cephv1.GetMgrResources(c.spec.Resources),
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
}
|
||||
|
||||
return container
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/utils/ptr"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -176,6 +177,15 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*corev1.Persiste
|
||||
return pvc, nil
|
||||
}
|
||||
|
||||
func makeMonSecurityContext() *corev1.PodSecurityContext {
|
||||
if controller.CephMonRunAsRoot() {
|
||||
return &corev1.PodSecurityContext{
|
||||
RunAsUser: ptr.To(int64(0)),
|
||||
}
|
||||
}
|
||||
return &corev1.PodSecurityContext{}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, error) {
|
||||
logger.Debugf("monConfig: %+v", monConfig)
|
||||
podSpec := corev1.PodSpec{
|
||||
@@ -192,7 +202,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, er
|
||||
Volumes: controller.DaemonVolumesBase(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
|
||||
HostNetwork: monConfig.UseHostNetwork,
|
||||
PriorityClassName: cephv1.GetMonPriorityClassName(c.spec.PriorityClassNames),
|
||||
SecurityContext: &corev1.PodSecurityContext{},
|
||||
SecurityContext: makeMonSecurityContext(),
|
||||
ServiceAccountName: k8sutil.DefaultServiceAccount,
|
||||
}
|
||||
|
||||
@@ -257,7 +267,7 @@ func (c *Cluster) makeChownInitContainer(monConfig *monConfig) corev1.Container
|
||||
controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
|
||||
controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
|
||||
cephv1.GetMonResources(c.spec.Resources),
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
"",
|
||||
)
|
||||
}
|
||||
@@ -278,7 +288,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container
|
||||
Image: c.spec.CephVersion.Image,
|
||||
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
|
||||
VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
// filesystem creation does not require ports to be exposed
|
||||
Env: controller.DaemonEnvVars(&c.spec),
|
||||
Resources: cephv1.GetMonResources(c.spec.Resources),
|
||||
@@ -310,7 +320,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
|
||||
Image: c.spec.CephVersion.Image,
|
||||
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
|
||||
VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
Ports: []corev1.ContainerPort{
|
||||
{
|
||||
Name: DefaultMsgr2PortName,
|
||||
|
||||
@@ -322,3 +322,20 @@ func TestGetFailureDomainLabel(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeMonSecurityContext(t *testing.T) {
|
||||
t.Run("returns root context when env is true", func(t *testing.T) {
|
||||
t.Setenv("ROOK_CEPH_MON_RUN_AS_ROOT", "true")
|
||||
sc := makeMonSecurityContext()
|
||||
assert.NotNil(t, sc)
|
||||
assert.NotNil(t, sc.RunAsUser)
|
||||
assert.Equal(t, int64(0), *sc.RunAsUser)
|
||||
})
|
||||
|
||||
t.Run("returns default context when env is not true", func(t *testing.T) {
|
||||
t.Setenv("ROOK_CEPH_MON_RUN_AS_ROOT", "false")
|
||||
sc := makeMonSecurityContext()
|
||||
assert.NotNil(t, sc)
|
||||
assert.Nil(t, sc.RunAsUser)
|
||||
})
|
||||
}
|
||||
@@ -147,7 +147,7 @@ func getCrashDirInitContainer(cephCluster cephv1.CephCluster) corev1.Container {
|
||||
},
|
||||
Image: cephCluster.Spec.CephVersion.Image,
|
||||
ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources),
|
||||
VolumeMounts: controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath),
|
||||
}
|
||||
@@ -163,7 +163,7 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container
|
||||
controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
|
||||
controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath),
|
||||
cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources),
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
"",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -164,7 +164,7 @@ func getCephExporterChownInitContainer(cephCluster cephv1.CephCluster) corev1.Co
|
||||
controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
|
||||
mounts,
|
||||
cephv1.GetCephExporterResources(cephCluster.Spec.Resources),
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
"",
|
||||
)
|
||||
}
|
||||
@@ -211,7 +211,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
|
||||
Ports: []corev1.ContainerPort{containerPort},
|
||||
VolumeMounts: volumeMounts,
|
||||
Resources: cephv1.GetCephExporterResources(cephCluster.Spec.Resources),
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
}
|
||||
|
||||
return container
|
||||
|
||||
@@ -144,7 +144,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster) corev1.Container {
|
||||
Env: envVars,
|
||||
VolumeMounts: volumeMounts,
|
||||
Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources),
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
}
|
||||
|
||||
return container
|
||||
|
||||
@@ -107,7 +107,7 @@ func (r *ReconcileCephRBDMirror) makeChownInitContainer(daemonConfig *daemonConf
|
||||
controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy),
|
||||
controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
|
||||
rbdMirror.Spec.Resources,
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
"",
|
||||
)
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem
|
||||
VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
|
||||
Env: controller.DaemonEnvVars(r.cephClusterSpec),
|
||||
Resources: rbdMirror.Spec.Resources,
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
WorkingDir: config.VarLogCephDir,
|
||||
// TODO:
|
||||
// Not implemented at this point since the socket name is '/run/ceph/ceph-client.rbd-mirror.a.1.94362516231272.asok'
|
||||
|
||||
@@ -726,8 +726,12 @@ func HostPathRequiresPrivileged() bool {
|
||||
return os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true"
|
||||
}
|
||||
|
||||
// PodSecurityContext detects if the pod needs privileges to run
|
||||
func PodSecurityContext() *v1.SecurityContext {
|
||||
func CephMonRunAsRoot() bool {
|
||||
return os.Getenv("ROOK_CEPH_MON_RUN_AS_ROOT") == "true"
|
||||
}
|
||||
|
||||
// DefaultContainerSecurityContext detects if the container needs privileges to run
|
||||
func DefaultContainerSecurityContext() *v1.SecurityContext {
|
||||
privileged := HostPathRequiresPrivileged()
|
||||
|
||||
return &v1.SecurityContext{
|
||||
@@ -743,7 +747,7 @@ func PodSecurityContext() *v1.SecurityContext {
|
||||
|
||||
// PodSecurityContext detects if the pod needs privileges to run
|
||||
func CephSecurityContext() *v1.SecurityContext {
|
||||
context := PodSecurityContext()
|
||||
context := DefaultContainerSecurityContext()
|
||||
cephUserID := CephUserID
|
||||
context.RunAsUser = &cephUserID
|
||||
context.RunAsGroup = &cephUserID
|
||||
@@ -824,7 +828,7 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, env []v1.E
|
||||
Image: c.CephVersion.Image,
|
||||
ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy),
|
||||
VolumeMounts: DaemonVolumeMounts(opconfig.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath),
|
||||
SecurityContext: PodSecurityContext(),
|
||||
SecurityContext: DefaultContainerSecurityContext(),
|
||||
Resources: cephv1.GetLogCollectorResources(c.Resources),
|
||||
// We need a TTY for the bash job control (enabled by -m)
|
||||
TTY: true,
|
||||
@@ -845,7 +849,7 @@ func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, env
|
||||
Image: c.CephVersion.Image,
|
||||
ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy),
|
||||
VolumeMounts: DaemonVolumeMounts(opconfig.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath),
|
||||
SecurityContext: PodSecurityContext(),
|
||||
SecurityContext: DefaultContainerSecurityContext(),
|
||||
Resources: Resources,
|
||||
// We need a TTY for the bash job control (enabled by -m)
|
||||
TTY: true,
|
||||
|
||||
@@ -126,7 +126,7 @@ func (c *Cluster) makeChownInitContainer(mdsConfig *mdsConfig) v1.Container {
|
||||
controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
|
||||
controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath),
|
||||
c.fs.Spec.MetadataServer.Resources,
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
"",
|
||||
)
|
||||
}
|
||||
@@ -153,7 +153,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig, fsName string) v1
|
||||
VolumeMounts: controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath),
|
||||
Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)),
|
||||
Resources: c.fs.Spec.MetadataServer.Resources,
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
// StartupProbe time for MDS is covered liveness probe
|
||||
LivenessProbe: generateMDSLivenessProbeExecDaemon(mdsConfig.DaemonID, fsName, keyring.VolumeMount().KeyringFilePath()),
|
||||
WorkingDir: cephconfig.VarLogCephDir,
|
||||
|
||||
@@ -104,7 +104,7 @@ func (r *ReconcileFilesystemMirror) makeChownInitContainer(daemonConfig *daemonC
|
||||
controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy),
|
||||
controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
|
||||
fsMirror.Spec.Resources,
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
"",
|
||||
)
|
||||
}
|
||||
@@ -125,7 +125,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig
|
||||
VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
|
||||
Env: controller.DaemonEnvVars(r.cephClusterSpec),
|
||||
Resources: fsMirror.Spec.Resources,
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
// TODO:
|
||||
// LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.fsMirrorType, daemonConfig.DaemonID),
|
||||
}
|
||||
|
||||
@@ -216,7 +216,7 @@ func (r *ReconcileCephNFS) connectionConfigInitContainer(nfs *cephv1.CephNFS, na
|
||||
keyring.VolumeMount().Resource(instanceName(nfs, name)),
|
||||
},
|
||||
nfs.Spec.Server.Resources,
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -250,7 +250,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
|
||||
},
|
||||
Env: controller.DaemonEnvVars(r.cephClusterSpec),
|
||||
Resources: nfs.Spec.Server.Resources,
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
LivenessProbe: r.defaultGaneshaLivenessProbe(nfs),
|
||||
}
|
||||
return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe)
|
||||
|
||||
@@ -330,7 +330,7 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig)
|
||||
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
|
||||
VolumeMounts: volumeMounts,
|
||||
Resources: c.store.Spec.Gateway.Resources,
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -367,7 +367,7 @@ func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig, kmsEnabled
|
||||
VolumeMounts: append(
|
||||
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), vaultVolMounts...),
|
||||
Resources: c.store.Spec.Gateway.Resources,
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -378,7 +378,7 @@ func (c *clusterConfig) makeChownInitContainer(rgwConfig *rgwConfig) v1.Containe
|
||||
controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
|
||||
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath),
|
||||
c.store.Spec.Gateway.Resources,
|
||||
controller.PodSecurityContext(),
|
||||
controller.DefaultContainerSecurityContext(),
|
||||
"",
|
||||
)
|
||||
}
|
||||
@@ -420,7 +420,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
|
||||
StartupProbe: startupProbe,
|
||||
LivenessProbe: noLivenessProbe(),
|
||||
ReadinessProbe: readinessProbe,
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
SecurityContext: controller.DefaultContainerSecurityContext(),
|
||||
WorkingDir: cephconfig.VarLogCephDir,
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user