diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 93e068442..486bb6354 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -638,6 +638,11 @@ spec: # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED value: "false" + # Whether to run Ceph mon pods as root (runAsUser: 0) at the pod level. + # Set this to true if mon pods are CrashLooping on SELinux-enabled systems despite other settings. + # For more context, see https://github.com/rook/rook/issues/15564 + - name: ROOK_CEPH_MON_RUN_AS_ROOT + value: "false" # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". # In case of more than one regex, use comma to separate between them. # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 01e747320..5d309166a 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -159,7 +159,7 @@ func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container { controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), cephv1.GetMgrResources(c.spec.Resources), - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -203,7 +203,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { c.cephMgrOrchestratorModuleEnvs()..., ), Resources: cephv1.GetMgrResources(c.spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), WorkingDir: config.VarLogCephDir, @@ -259,7 +259,7 @@ func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container { ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), Env: envVars, Resources: cephv1.GetMgrSidecarResources(c.spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), VolumeMounts: volumeMounts, } } @@ -275,7 +275,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe VolumeMounts: append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount), Env: append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}), Resources: cephv1.GetMgrResources(c.spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } return container diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 09ad8bc85..a99cc8f24 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -32,6 +32,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) const ( @@ -176,6 +177,15 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*corev1.Persiste return pvc, nil } +func makeMonSecurityContext() *corev1.PodSecurityContext { + if controller.CephMonRunAsRoot() { + return &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(0)), + } + } + return &corev1.PodSecurityContext{} +} + func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, error) { logger.Debugf("monConfig: %+v", monConfig) podSpec := corev1.PodSpec{ @@ -192,7 +202,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, er Volumes: controller.DaemonVolumesBase(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), HostNetwork: monConfig.UseHostNetwork, PriorityClassName: cephv1.GetMonPriorityClassName(c.spec.PriorityClassNames), - SecurityContext: &corev1.PodSecurityContext{}, + SecurityContext: makeMonSecurityContext(), ServiceAccountName: k8sutil.DefaultServiceAccount, } @@ -257,7 +267,7 @@ func (c *Cluster) makeChownInitContainer(monConfig *monConfig) corev1.Container controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), cephv1.GetMonResources(c.spec.Resources), - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -278,7 +288,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container Image: c.spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), // filesystem creation does not require ports to be exposed Env: controller.DaemonEnvVars(&c.spec), Resources: cephv1.GetMonResources(c.spec.Resources), @@ -310,7 +320,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container Image: c.spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), Ports: []corev1.ContainerPort{ { Name: DefaultMsgr2PortName, diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index 1384d90de..abd0d0b36 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -322,3 +322,20 @@ func TestGetFailureDomainLabel(t *testing.T) { }) } } + +func TestMakeMonSecurityContext(t *testing.T) { + t.Run("returns root context when env is true", func(t *testing.T) { + t.Setenv("ROOK_CEPH_MON_RUN_AS_ROOT", "true") + sc := makeMonSecurityContext() + assert.NotNil(t, sc) + assert.NotNil(t, sc.RunAsUser) + assert.Equal(t, int64(0), *sc.RunAsUser) + }) + + t.Run("returns default context when env is not true", func(t *testing.T) { + t.Setenv("ROOK_CEPH_MON_RUN_AS_ROOT", "false") + sc := makeMonSecurityContext() + assert.NotNil(t, sc) + assert.Nil(t, sc.RunAsUser) + }) +} diff --git a/pkg/operator/ceph/cluster/nodedaemon/crash.go b/pkg/operator/ceph/cluster/nodedaemon/crash.go index 86c4eeb85..ddceb2c78 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/crash.go +++ b/pkg/operator/ceph/cluster/nodedaemon/crash.go @@ -147,7 +147,7 @@ func getCrashDirInitContainer(cephCluster cephv1.CephCluster) corev1.Container { }, Image: cephCluster.Spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources), VolumeMounts: controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath), } @@ -163,7 +163,7 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath), cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources), - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index 987a49d43..198c6e4e5 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -164,7 +164,7 @@ func getCephExporterChownInitContainer(cephCluster cephv1.CephCluster) corev1.Co controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), mounts, cephv1.GetCephExporterResources(cephCluster.Spec.Resources), - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -211,7 +211,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion Ports: []corev1.ContainerPort{containerPort}, VolumeMounts: volumeMounts, Resources: cephv1.GetCephExporterResources(cephCluster.Spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } return container diff --git a/pkg/operator/ceph/cluster/nodedaemon/pruner.go b/pkg/operator/ceph/cluster/nodedaemon/pruner.go index f072a3136..19f8714bf 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/pruner.go +++ b/pkg/operator/ceph/cluster/nodedaemon/pruner.go @@ -144,7 +144,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster) corev1.Container { Env: envVars, VolumeMounts: volumeMounts, Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources), - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } return container diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index cb78011da..f080e2775 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -107,7 +107,7 @@ func (r *ReconcileCephRBDMirror) makeChownInitContainer(daemonConfig *daemonConf controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), rbdMirror.Spec.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -128,7 +128,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: rbdMirror.Spec.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), WorkingDir: config.VarLogCephDir, // TODO: // Not implemented at this point since the socket name is '/run/ceph/ceph-client.rbd-mirror.a.1.94362516231272.asok' diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 699d68d1c..07916d56b 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -726,8 +726,12 @@ func HostPathRequiresPrivileged() bool { return os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true" } -// PodSecurityContext detects if the pod needs privileges to run -func PodSecurityContext() *v1.SecurityContext { +func CephMonRunAsRoot() bool { + return os.Getenv("ROOK_CEPH_MON_RUN_AS_ROOT") == "true" +} + +// DefaultContainerSecurityContext detects if the container needs privileges to run +func DefaultContainerSecurityContext() *v1.SecurityContext { privileged := HostPathRequiresPrivileged() return &v1.SecurityContext{ @@ -743,7 +747,7 @@ func PodSecurityContext() *v1.SecurityContext { // PodSecurityContext detects if the pod needs privileges to run func CephSecurityContext() *v1.SecurityContext { - context := PodSecurityContext() + context := DefaultContainerSecurityContext() cephUserID := CephUserID context.RunAsUser = &cephUserID context.RunAsGroup = &cephUserID @@ -824,7 +828,7 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, env []v1.E Image: c.CephVersion.Image, ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy), VolumeMounts: DaemonVolumeMounts(opconfig.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath), - SecurityContext: PodSecurityContext(), + SecurityContext: DefaultContainerSecurityContext(), Resources: cephv1.GetLogCollectorResources(c.Resources), // We need a TTY for the bash job control (enabled by -m) TTY: true, @@ -845,7 +849,7 @@ func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, env Image: c.CephVersion.Image, ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy), VolumeMounts: DaemonVolumeMounts(opconfig.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath), - SecurityContext: PodSecurityContext(), + SecurityContext: DefaultContainerSecurityContext(), Resources: Resources, // We need a TTY for the bash job control (enabled by -m) TTY: true, diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index ac54eba58..febbf98ed 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -126,7 +126,7 @@ func (c *Cluster) makeChownInitContainer(mdsConfig *mdsConfig) v1.Container { controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath), c.fs.Spec.MetadataServer.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -153,7 +153,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig, fsName string) v1 VolumeMounts: controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath), Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)), Resources: c.fs.Spec.MetadataServer.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), // StartupProbe time for MDS is covered liveness probe LivenessProbe: generateMDSLivenessProbeExecDaemon(mdsConfig.DaemonID, fsName, keyring.VolumeMount().KeyringFilePath()), WorkingDir: cephconfig.VarLogCephDir, diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go index 45fdc9a38..4aef52949 100644 --- a/pkg/operator/ceph/file/mirror/spec.go +++ b/pkg/operator/ceph/file/mirror/spec.go @@ -104,7 +104,7 @@ func (r *ReconcileFilesystemMirror) makeChownInitContainer(daemonConfig *daemonC controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), fsMirror.Spec.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -125,7 +125,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: fsMirror.Spec.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), // TODO: // LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.fsMirrorType, daemonConfig.DaemonID), } diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index e5d9fb1a0..b761e9eea 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -216,7 +216,7 @@ func (r *ReconcileCephNFS) connectionConfigInitContainer(nfs *cephv1.CephNFS, na keyring.VolumeMount().Resource(instanceName(nfs, name)), }, nfs.Spec.Server.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), ) } @@ -250,7 +250,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig }, Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), LivenessProbe: r.defaultGaneshaLivenessProbe(nfs), } return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe) diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 454e2d16c..16fefa340 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -330,7 +330,7 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig) ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: volumeMounts, Resources: c.store.Spec.Gateway.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } } @@ -367,7 +367,7 @@ func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig, kmsEnabled VolumeMounts: append( controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), vaultVolMounts...), Resources: c.store.Spec.Gateway.Resources, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), } } @@ -378,7 +378,7 @@ func (c *clusterConfig) makeChownInitContainer(rgwConfig *rgwConfig) v1.Containe controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), c.store.Spec.Gateway.Resources, - controller.PodSecurityContext(), + controller.DefaultContainerSecurityContext(), "", ) } @@ -420,7 +420,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, StartupProbe: startupProbe, LivenessProbe: noLivenessProbe(), ReadinessProbe: readinessProbe, - SecurityContext: controller.PodSecurityContext(), + SecurityContext: controller.DefaultContainerSecurityContext(), WorkingDir: cephconfig.VarLogCephDir, }