diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index c1de0778b..1dbf733df 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -31,6 +31,7 @@ an example usage created. Previously, it was ignored when a cluster was first installed. - Upgrades have drastically improved, Rook intelligently checks for each daemon state before and after upgrading. To learn more about the upgrade workflow see [Ceph Upgrades](Documentation/ceph-upgrade.md) - Rook Operator now supports 2 new environmental variables: `AGENT_TOLERATIONS` and `DISCOVER_TOLERATIONS`. Each accept list of tolerations for agent and discover pods accordingly. +- Ceph daemons now run under 'ceph' user and not 'root' anymore (monitor or osd store already owned by 'root' will keep running under 'root') ## Breaking Changes diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 1eac50a51..e72a7d253 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -274,6 +274,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { }, InitialDelaySeconds: 60, }, + Lifecycle: opspec.PodLifeCycle(""), } return container } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index a3f2dba0a..a31bee0ce 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -19,6 +19,7 @@ package mon import ( "fmt" "os" + "path" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -90,6 +91,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { logger.Debug("monConfig: %+v", monConfig) podSpec := v1.PodSpec{ InitContainers: []v1.Container{ + c.makeChownDataDirInitContainer(monConfig), c.makeMonFSInitContainer(monConfig), }, Containers: []v1.Container{ @@ -134,7 +136,33 @@ func podSecurityContext() *v1.SecurityContext { if os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true" { privileged = true } - return &v1.SecurityContext{Privileged: &privileged} + + return &v1.SecurityContext{ + Privileged: &privileged, + } +} + +func (c *Cluster) makeChownDataDirInitContainer(monConfig *monConfig) v1.Container { + // Before makeMonFSInitContainer starts we must apply the right ownership to the mon data dir + // so the mkfs can succeed, otherwise it'll fail since it's owned by root + // Unfortunately, we can't use: + // Lifecycle: &v1.Lifecycle{PostStart: &v1.Handler{Exec: &v1.ExecAction{ + // On an InitContainer :-( + container := v1.Container{ + Name: "chown-container-data-dir", + Command: []string{ + "chown", + }, + Args: []string{ + "--verbose", + "ceph:ceph", + monConfig.DataPathMap.ContainerDataDir, + }, + Image: c.spec.CephVersion.Image, + VolumeMounts: opspec.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName), + Resources: cephv1.GetMonResources(c.spec.Resources), + } + return container } func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container { @@ -182,6 +210,13 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { // If the mon is already in the monmap, when the port is left off of --public-addr, // it will still advertise on the previous port b/c monmap is saved to mon database. config.NewFlag("public-addr", publicAddr), + // Set '--setuser-match-path' so that existing directory owned by root won't affect the daemon startup. + // For existing data store owned by root, the daemon will continue to run as root + // + // We use 'store.db' here because during an upgrade the init container will set 'ceph:ceph' to monConfig.DataPathMap.ContainerDataDir + // but inside the permissions will be 'root:root' AND we don't want to chown recursively on the mon data directory + // We want to avoid potential startup time issue if the store is big + config.NewFlag("setuser-match-path", path.Join(monConfig.DataPathMap.ContainerDataDir, "store.db")), ), Image: c.spec.CephVersion.Image, VolumeMounts: opspec.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName), @@ -198,6 +233,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: cephv1.GetMonResources(c.spec.Resources), + Lifecycle: opspec.PodLifeCycle(""), } // If host networking is enabled, we don't need a bind addr that is different from the public addr diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 6fa9d016f..95dc16c09 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -210,6 +210,11 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, "--osd-uuid", osd.UUID, "--conf", osd.Config, "--cluster", "ceph", + "--setuser", "ceph", + "--setgroup", "ceph", + // Set '--setuser-match-path' so that existing directory owned by root won't affect the daemon startup. + // For existing data store owned by root, the daemon will continue to run as root + "--setuser-match-path", osd.DataPath, } // Set osd memory target to the best appropriate value @@ -260,6 +265,7 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, if c.HostNetwork { DNSPolicy = v1.DNSClusterFirstWithHostNet } + deployment := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(osdAppNameFmt, osd.ID), @@ -319,6 +325,7 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, Env: envVars, Resources: resources, SecurityContext: securityContext, + Lifecycle: opspec.PodLifeCycle(osd.DataPath), }, }, Volumes: volumes, diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index c5ec8100c..2abad7a88 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -83,6 +83,7 @@ func (m *Mirroring) makeMirroringDaemonContainer(daemonConfig *daemonConfig) v1. VolumeMounts: opspec.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName), Env: opspec.DaemonEnvVars(m.cephVersion.Image), Resources: m.resources, + Lifecycle: opspec.PodLifeCycle(""), } return container } diff --git a/pkg/operator/ceph/config/config.go b/pkg/operator/ceph/config/config.go index c2c0bafe7..4fc0d5207 100644 --- a/pkg/operator/ceph/config/config.go +++ b/pkg/operator/ceph/config/config.go @@ -52,6 +52,12 @@ const ( // RbdMirrorType defines the rbd-mirror DaemonType RbdMirrorType DaemonType = "rbd-mirror" + + // CephUser is the Linux Ceph username + CephUser = "ceph" + + // CephGroup is the Linux Ceph groupname + CephGroup = "ceph" ) var ( @@ -66,6 +72,13 @@ var ( // VarLogCephDir defines Ceph logging directory. It is made overwriteable only for unit tests where it // may be needed to send data intended for /var/log/ceph to a temporary test dir. VarLogCephDir = "/var/log/ceph" + + // chownUserGroup represents ceph:ceph + chownUserGroup = CephUser + ":" + CephGroup + + // ContainerPostStartCmd is the command we run before starting any Ceph daemon + // It makes sure Ceph directories are owned by 'ceph' + ContainerPostStartCmd = []string{"chown", "--recursive", "--verbose", chownUserGroup, VarLogCephDir} ) // Config is a Ceph config struct containing zero or more Ceph config sections. diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index aaf2a1021..fd4f73226 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -220,8 +220,17 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephconfig.ClusterInfo // StoredFileVolume returns a pod volume sourced from the stored config file. func StoredFileVolume() v1.Volume { - // TL;DR: mount the configmap's "ceph.conf" to a file called "ceph.conf" with 0400 permissions - mode := int32(0400) // security: only allow the owner to read and no one to write + // TL;DR: mount the configmap's "ceph.conf" to a file called "ceph.conf" with 0444 permissions + // security: allow to be read by everyone since now ceph processes run as 'ceph' and not 'root' user + // Further investigation needs to be done to copy the ceph.conf and change its ownership + // since configuring a owner of a ConfigMap secret is currently impossible + // This also works around the following issue: https://tracker.ceph.com/issues/38606 + // + // This design choice avoids the crash/restart situation in Rook + // If we don't set 0444 to the ceph.conf configuration file during its respawn (with exec) the ceph-mgr + // won't be able to read the ceph.conf and the container will die, the "restart" count will increase in k8s + // This will mislead users thinking something won't wrong but that a false positive + mode := int32(0444) return v1.Volume{ Name: configVolumeName, VolumeSource: v1.VolumeSource{ diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 110ab1384..8cecb1e2e 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -117,6 +117,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { opspec.DaemonEnvVars(c.cephVersion.Image), ), Resources: c.fs.Spec.MetadataServer.Resources, + Lifecycle: opspec.PodLifeCycle(""), } return container diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 99c91d836..5fc899a17 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -102,7 +102,6 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec } func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { - // start the rgw daemon in the foreground container := v1.Container{ Name: "rgw", @@ -134,6 +133,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { }, InitialDelaySeconds: 10, }, + Lifecycle: opspec.PodLifeCycle(""), } if c.store.Spec.Gateway.SSLCertificateRef != "" { diff --git a/pkg/operator/ceph/spec/spec.go b/pkg/operator/ceph/spec/spec.go index 97a78c4ae..2f029629c 100644 --- a/pkg/operator/ceph/spec/spec.go +++ b/pkg/operator/ceph/spec/spec.go @@ -109,13 +109,23 @@ func DaemonFlags(cluster *cephconfig.ClusterInfo, daemonID string) []string { return append( config.DefaultFlags(cluster.FSID, keyring.VolumeMount().KeyringFilePath(), cluster.CephVersion), config.NewFlag("id", daemonID), + // Ceph daemons in Rook will run as 'ceph' instead of 'root' + // If we run on a version of Ceph does not these flags it will simply ignore them + //run ceph daemon process under the 'ceph' user + config.NewFlag("setuser", "ceph"), + // run ceph daemon process under the 'ceph' group + config.NewFlag("setgroup", "ceph"), ) } // AdminFlags returns the command line flags used for Ceph commands requiring admin authentication. func AdminFlags(cluster *cephconfig.ClusterInfo) []string { - return config.DefaultFlags(cluster.FSID, keyring.VolumeMount().AdminKeyringFilePath(), cluster.CephVersion) + return append( + config.DefaultFlags(cluster.FSID, keyring.VolumeMount().AdminKeyringFilePath(), cluster.CephVersion), + config.NewFlag("setuser", "ceph"), + config.NewFlag("setgroup", "ceph"), + ) } // ContainerEnvVarReference returns a reference to a Kubernetes container env var of the given name @@ -202,3 +212,26 @@ func StoredLogVolumeMount() v1.VolumeMount { MountPath: config.VarLogCephDir, } } + +func generateLifeCycleCmd(dataDirHostPath string) []string { + cmd := config.ContainerPostStartCmd + + if dataDirHostPath != "" { + cmd = append(cmd, dataDirHostPath) + } + + return cmd +} + +// PodLifeCycle returns a pod lifecycle resource to execute actions before a pod starts +func PodLifeCycle(dataDirHostPath string) *v1.Lifecycle { + cmd := generateLifeCycleCmd(dataDirHostPath) + + return &v1.Lifecycle{ + PostStart: &v1.Handler{ + Exec: &v1.ExecAction{ + Command: cmd, + }, + }, + } +} diff --git a/pkg/operator/ceph/spec/spec_test.go b/pkg/operator/ceph/spec/spec_test.go index bf575541c..5a519593f 100644 --- a/pkg/operator/ceph/spec/spec_test.go +++ b/pkg/operator/ceph/spec/spec_test.go @@ -19,8 +19,10 @@ package spec import ( "testing" + "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" ) func TestPodVolumes(t *testing.T) { @@ -42,3 +44,11 @@ func TestMountsMatchVolumes(t *testing.T) { } volsMountsTestDef.TestMountsMatchVolumes(t) } + +func TestGenerateLifeCycleCmd(t *testing.T) { + cmd := generateLifeCycleCmd("") + assert.Equal(t, config.ContainerPostStartCmd, cmd) + + cmd = generateLifeCycleCmd("foo") + assert.Equal(t, append(config.ContainerPostStartCmd, "foo"), cmd) +}