From 900abbc967e108ad622648b740a7c57f1268209f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 12 Feb 2019 17:51:17 +0100 Subject: [PATCH 1/2] ceph: run ceph processes with the 'ceph' user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ceph daemons will run as the 'ceph' user instead of the 'root' user. Existing data store (osd and mon) owned by root will continue to run as 'root'. ALl CLI calls also inherate from this and will use them when fetching a key for instance. If we run on a version of ceph that does not know these flags they will simply get ignored. Consider the following: [leseb@tarox~/go/src/github.com/rook/rook][use-ceph-user !] kubectl get pods -n rook-ceph NAME READY STATUS RESTARTS AGE rook-ceph-mds-myfs-a-67cff7df69-fq4l5 1/1 Running 0 3m rook-ceph-mds-myfs-b-85bd5f59f5-d8hdn 1/1 Running 0 3m rook-ceph-mgr-a-7d88b9457b-jzln6 1/1 Running 2 4m rook-ceph-mon-a-64bf697496-r94tc 1/1 Running 0 4m rook-ceph-osd-0-c5fdcd74c-9kvb4 1/1 Running 0 3m rook-ceph-osd-prepare-minikube-shz9l 0/2 Completed 0 3m rook-ceph-rbd-mirror-a-5cbff8469-xv64c 1/1 Running 0 3m rook-ceph-rgw-my-store-f7645b44-vlbp4 1/1 Running 0 3m MDS daemon: ceph 1 0.2 0.1 449508 29200 ? Ssl 15:10 0:00 ceph-mds --fsid=bf359e2c-a1df-4a80-9137-8134b41a5459 --keyring=/etc/ceph/keyring-store/keyring --log-to-stderr=true --err-to-stderr=true --mon-cluster-log-to-stderr=true --log-stderr-prefix=debug --mon-host=10.110.204.228:6789 --mon-initial-members=a --name=mds.myfs-a --setuser=ceph --setgroup=ceph --foreground --mds-standby-for-fscid=1 --mds-standby-replay=true MGR daemon: ceph 1 0.8 0.6 867176 104732 ? Ssl 15:10 0:02 ceph-mgr --fsid=bf359e2c-a1df-4a80-9137-8134b41a5459 --keyring=/etc/ceph/keyring-store/keyring --log-to-stderr=true --err-to-stderr=true --mon-cluster-log-to-stderr=true --log-stderr-prefix=debug --mon-host=10.110.204.228:6789 --mon-initial-members=a --name=mgr.a --setuser=ceph --setgroup=ceph --foreground MON daemon: ceph 1 0 0 15:10 ? 00:00:02 ceph-mon --fsid=bf359e2c-a1df-4a80-9137-8134b41a5459 --keyring=/etc/ceph/keyring-store/keyring --log-to-stderr=true --err-to-stderr=true --mon-cluster-log-to-stderr=true --log-stderr-prefix=debug --mon-host=10.110.204.228:6789 --mon-initial-members=a --name=mon.a --setuser=ceph --setgroup=ceph --foreground --public-addr=10.110.204.228 --public-bind-addr=172.17.0.6 --setuser-match-path=/var/lib/ceph/mon/ceph-a OSD daemon: ceph 30985 0.9 0.5 885736 81236 ? Ssl 15:10 0:04 | \_ ceph-osd --foreground --id 0 --conf /var/lib/rook/osd0/rook-ceph.config --osd-data /var/lib/rook/osd0 --setuser-match-path /var/lib/rook/osd0 --setuser ceph --setgro up ceph --keyring /var/lib/rook/osd0/keyring --cluster rook-ceph --osd-uuid 98b4527d-f7a8-44f0-ac47-f8b55ff5ee05 --osd-journal=/var/lib/rook/osd0/journal RBD mirror daemon: ceph 1 0.0 0.1 914500 21848 ? Ssl 15:10 0:00 rbd-mirror --setuser ceph --setgroup ceph --foreground -n client.rbd-mirror.a --conf /etc/ceph/ceph.conf --keyring /etc/ceph/keyring RGW daemon: ceph 1 0.2 0.3 5179452 52244 ? Ssl 15:11 0:01 radosgw --setuser ceph --setgroup ceph --foreground --name=client.radosgw.gateway --rgw-mime-types-file=/var/lib/rook/rgw/mime.types Even though Ceph processes are running as Ceph, the container itself runs as root. We haven't reached the situation where we can runAsUser != "root" and enable SecurityContext to RunAsNonRoot, but that's our ultimate goal. Resolves: https://github.com/rook/rook/issues/2664 Signed-off-by: Sébastien Han --- PendingReleaseNotes.md | 1 + pkg/operator/ceph/cluster/mon/spec.go | 37 ++++++++++++++++++++++++++- pkg/operator/ceph/cluster/osd/spec.go | 5 ++++ pkg/operator/ceph/config/store.go | 13 ++++++++-- pkg/operator/ceph/spec/spec.go | 12 ++++++++- 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index c1de0778b..1dbf733df 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -31,6 +31,7 @@ an example usage created. Previously, it was ignored when a cluster was first installed. - Upgrades have drastically improved, Rook intelligently checks for each daemon state before and after upgrading. To learn more about the upgrade workflow see [Ceph Upgrades](Documentation/ceph-upgrade.md) - Rook Operator now supports 2 new environmental variables: `AGENT_TOLERATIONS` and `DISCOVER_TOLERATIONS`. Each accept list of tolerations for agent and discover pods accordingly. +- Ceph daemons now run under 'ceph' user and not 'root' anymore (monitor or osd store already owned by 'root' will keep running under 'root') ## Breaking Changes diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index a3f2dba0a..816ad2c40 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -19,6 +19,7 @@ package mon import ( "fmt" "os" + "path" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -90,6 +91,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { logger.Debug("monConfig: %+v", monConfig) podSpec := v1.PodSpec{ InitContainers: []v1.Container{ + c.makeChownDataDirInitContainer(monConfig), c.makeMonFSInitContainer(monConfig), }, Containers: []v1.Container{ @@ -134,7 +136,33 @@ func podSecurityContext() *v1.SecurityContext { if os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true" { privileged = true } - return &v1.SecurityContext{Privileged: &privileged} + + return &v1.SecurityContext{ + Privileged: &privileged, + } +} + +func (c *Cluster) makeChownDataDirInitContainer(monConfig *monConfig) v1.Container { + // Before makeMonFSInitContainer starts we must apply the right ownership to the mon data dir + // so the mkfs can succeed, otherwise it'll fail since it's owned by root + // Unfortunately, we can't use: + // Lifecycle: &v1.Lifecycle{PostStart: &v1.Handler{Exec: &v1.ExecAction{ + // On an InitContainer :-( + container := v1.Container{ + Name: "chown-container-data-dir", + Command: []string{ + "chown", + }, + Args: []string{ + "--verbose", + "ceph:ceph", + monConfig.DataPathMap.ContainerDataDir, + }, + Image: c.spec.CephVersion.Image, + VolumeMounts: opspec.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName), + Resources: cephv1.GetMonResources(c.spec.Resources), + } + return container } func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container { @@ -182,6 +210,13 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { // If the mon is already in the monmap, when the port is left off of --public-addr, // it will still advertise on the previous port b/c monmap is saved to mon database. config.NewFlag("public-addr", publicAddr), + // Set '--setuser-match-path' so that existing directory owned by root won't affect the daemon startup. + // For existing data store owned by root, the daemon will continue to run as root + // + // We use 'store.db' here because during an upgrade the init container will set 'ceph:ceph' to monConfig.DataPathMap.ContainerDataDir + // but inside the permissions will be 'root:root' AND we don't want to chown recursively on the mon data directory + // We want to avoid potential startup time issue if the store is big + config.NewFlag("setuser-match-path", path.Join(monConfig.DataPathMap.ContainerDataDir, "store.db")), ), Image: c.spec.CephVersion.Image, VolumeMounts: opspec.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName), diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 6fa9d016f..c1697d963 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -210,6 +210,11 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, "--osd-uuid", osd.UUID, "--conf", osd.Config, "--cluster", "ceph", + "--setuser", "ceph", + "--setgroup", "ceph", + // Set '--setuser-match-path' so that existing directory owned by root won't affect the daemon startup. + // For existing data store owned by root, the daemon will continue to run as root + "--setuser-match-path", osd.DataPath, } // Set osd memory target to the best appropriate value diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index aaf2a1021..fd4f73226 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -220,8 +220,17 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephconfig.ClusterInfo // StoredFileVolume returns a pod volume sourced from the stored config file. func StoredFileVolume() v1.Volume { - // TL;DR: mount the configmap's "ceph.conf" to a file called "ceph.conf" with 0400 permissions - mode := int32(0400) // security: only allow the owner to read and no one to write + // TL;DR: mount the configmap's "ceph.conf" to a file called "ceph.conf" with 0444 permissions + // security: allow to be read by everyone since now ceph processes run as 'ceph' and not 'root' user + // Further investigation needs to be done to copy the ceph.conf and change its ownership + // since configuring a owner of a ConfigMap secret is currently impossible + // This also works around the following issue: https://tracker.ceph.com/issues/38606 + // + // This design choice avoids the crash/restart situation in Rook + // If we don't set 0444 to the ceph.conf configuration file during its respawn (with exec) the ceph-mgr + // won't be able to read the ceph.conf and the container will die, the "restart" count will increase in k8s + // This will mislead users thinking something won't wrong but that a false positive + mode := int32(0444) return v1.Volume{ Name: configVolumeName, VolumeSource: v1.VolumeSource{ diff --git a/pkg/operator/ceph/spec/spec.go b/pkg/operator/ceph/spec/spec.go index 97a78c4ae..a6805ad7a 100644 --- a/pkg/operator/ceph/spec/spec.go +++ b/pkg/operator/ceph/spec/spec.go @@ -109,13 +109,23 @@ func DaemonFlags(cluster *cephconfig.ClusterInfo, daemonID string) []string { return append( config.DefaultFlags(cluster.FSID, keyring.VolumeMount().KeyringFilePath(), cluster.CephVersion), config.NewFlag("id", daemonID), + // Ceph daemons in Rook will run as 'ceph' instead of 'root' + // If we run on a version of Ceph does not these flags it will simply ignore them + //run ceph daemon process under the 'ceph' user + config.NewFlag("setuser", "ceph"), + // run ceph daemon process under the 'ceph' group + config.NewFlag("setgroup", "ceph"), ) } // AdminFlags returns the command line flags used for Ceph commands requiring admin authentication. func AdminFlags(cluster *cephconfig.ClusterInfo) []string { - return config.DefaultFlags(cluster.FSID, keyring.VolumeMount().AdminKeyringFilePath(), cluster.CephVersion) + return append( + config.DefaultFlags(cluster.FSID, keyring.VolumeMount().AdminKeyringFilePath(), cluster.CephVersion), + config.NewFlag("setuser", "ceph"), + config.NewFlag("setgroup", "ceph"), + ) } // ContainerEnvVarReference returns a reference to a Kubernetes container env var of the given name From 9120791e24c2f911dc5935f2a78fdcbe7f8ea014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 18 Jul 2019 12:59:12 +0200 Subject: [PATCH 2/2] ceph: chown ceph directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the beginning of every container startup we now change the ownership of every known ceph directories to ceph:ceph. This will allow daemons to start properly as well as allowing daemon to writes their logs on files. This is needed because ceph daemon now run as 'ceph' instead of 'root' Signed-off-by: Sébastien Han --- pkg/operator/ceph/cluster/mgr/spec.go | 1 + pkg/operator/ceph/cluster/mon/spec.go | 1 + pkg/operator/ceph/cluster/osd/spec.go | 2 ++ pkg/operator/ceph/cluster/rbd/spec.go | 1 + pkg/operator/ceph/config/config.go | 13 +++++++++++++ pkg/operator/ceph/file/mds/spec.go | 1 + pkg/operator/ceph/object/spec.go | 2 +- pkg/operator/ceph/spec/spec.go | 23 +++++++++++++++++++++++ pkg/operator/ceph/spec/spec_test.go | 10 ++++++++++ 9 files changed, 53 insertions(+), 1 deletion(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 1eac50a51..e72a7d253 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -274,6 +274,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { }, InitialDelaySeconds: 60, }, + Lifecycle: opspec.PodLifeCycle(""), } return container } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 816ad2c40..a31bee0ce 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -233,6 +233,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: cephv1.GetMonResources(c.spec.Resources), + Lifecycle: opspec.PodLifeCycle(""), } // If host networking is enabled, we don't need a bind addr that is different from the public addr diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index c1697d963..95dc16c09 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -265,6 +265,7 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, if c.HostNetwork { DNSPolicy = v1.DNSClusterFirstWithHostNet } + deployment := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(osdAppNameFmt, osd.ID), @@ -324,6 +325,7 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, Env: envVars, Resources: resources, SecurityContext: securityContext, + Lifecycle: opspec.PodLifeCycle(osd.DataPath), }, }, Volumes: volumes, diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index c5ec8100c..2abad7a88 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -83,6 +83,7 @@ func (m *Mirroring) makeMirroringDaemonContainer(daemonConfig *daemonConfig) v1. VolumeMounts: opspec.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName), Env: opspec.DaemonEnvVars(m.cephVersion.Image), Resources: m.resources, + Lifecycle: opspec.PodLifeCycle(""), } return container } diff --git a/pkg/operator/ceph/config/config.go b/pkg/operator/ceph/config/config.go index c2c0bafe7..4fc0d5207 100644 --- a/pkg/operator/ceph/config/config.go +++ b/pkg/operator/ceph/config/config.go @@ -52,6 +52,12 @@ const ( // RbdMirrorType defines the rbd-mirror DaemonType RbdMirrorType DaemonType = "rbd-mirror" + + // CephUser is the Linux Ceph username + CephUser = "ceph" + + // CephGroup is the Linux Ceph groupname + CephGroup = "ceph" ) var ( @@ -66,6 +72,13 @@ var ( // VarLogCephDir defines Ceph logging directory. It is made overwriteable only for unit tests where it // may be needed to send data intended for /var/log/ceph to a temporary test dir. VarLogCephDir = "/var/log/ceph" + + // chownUserGroup represents ceph:ceph + chownUserGroup = CephUser + ":" + CephGroup + + // ContainerPostStartCmd is the command we run before starting any Ceph daemon + // It makes sure Ceph directories are owned by 'ceph' + ContainerPostStartCmd = []string{"chown", "--recursive", "--verbose", chownUserGroup, VarLogCephDir} ) // Config is a Ceph config struct containing zero or more Ceph config sections. diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 110ab1384..8cecb1e2e 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -117,6 +117,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { opspec.DaemonEnvVars(c.cephVersion.Image), ), Resources: c.fs.Spec.MetadataServer.Resources, + Lifecycle: opspec.PodLifeCycle(""), } return container diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 99c91d836..5fc899a17 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -102,7 +102,6 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec } func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { - // start the rgw daemon in the foreground container := v1.Container{ Name: "rgw", @@ -134,6 +133,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { }, InitialDelaySeconds: 10, }, + Lifecycle: opspec.PodLifeCycle(""), } if c.store.Spec.Gateway.SSLCertificateRef != "" { diff --git a/pkg/operator/ceph/spec/spec.go b/pkg/operator/ceph/spec/spec.go index a6805ad7a..2f029629c 100644 --- a/pkg/operator/ceph/spec/spec.go +++ b/pkg/operator/ceph/spec/spec.go @@ -212,3 +212,26 @@ func StoredLogVolumeMount() v1.VolumeMount { MountPath: config.VarLogCephDir, } } + +func generateLifeCycleCmd(dataDirHostPath string) []string { + cmd := config.ContainerPostStartCmd + + if dataDirHostPath != "" { + cmd = append(cmd, dataDirHostPath) + } + + return cmd +} + +// PodLifeCycle returns a pod lifecycle resource to execute actions before a pod starts +func PodLifeCycle(dataDirHostPath string) *v1.Lifecycle { + cmd := generateLifeCycleCmd(dataDirHostPath) + + return &v1.Lifecycle{ + PostStart: &v1.Handler{ + Exec: &v1.ExecAction{ + Command: cmd, + }, + }, + } +} diff --git a/pkg/operator/ceph/spec/spec_test.go b/pkg/operator/ceph/spec/spec_test.go index bf575541c..5a519593f 100644 --- a/pkg/operator/ceph/spec/spec_test.go +++ b/pkg/operator/ceph/spec/spec_test.go @@ -19,8 +19,10 @@ package spec import ( "testing" + "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" ) func TestPodVolumes(t *testing.T) { @@ -42,3 +44,11 @@ func TestMountsMatchVolumes(t *testing.T) { } volsMountsTestDef.TestMountsMatchVolumes(t) } + +func TestGenerateLifeCycleCmd(t *testing.T) { + cmd := generateLifeCycleCmd("") + assert.Equal(t, config.ContainerPostStartCmd, cmd) + + cmd = generateLifeCycleCmd("foo") + assert.Equal(t, append(config.ContainerPostStartCmd, "foo"), cmd) +}