Merge pull request #2778 from leseb/use-ceph-user

ceph: run ceph processes with the 'ceph' user

PendingReleaseNotes.md

@@ -31,6 +31,7 @@ an example usage
   created. Previously, it was ignored when a cluster was first installed.
 - Upgrades have drastically improved, Rook intelligently checks for each daemon state before and after upgrading. To learn more about the upgrade workflow see [Ceph Upgrades](Documentation/ceph-upgrade.md)
 - Rook Operator now supports 2 new environmental variables: `AGENT_TOLERATIONS` and `DISCOVER_TOLERATIONS`. Each accept list of tolerations for agent and discover pods accordingly.
+- Ceph daemons now run under 'ceph' user and not 'root' anymore (monitor or osd store already owned by 'root' will keep running under 'root')
 
 ## Breaking Changes
 

pkg/operator/ceph/cluster/mgr/spec.go

@@ -274,6 +274,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
 			},
 			InitialDelaySeconds: 60,
 		},
+		Lifecycle: opspec.PodLifeCycle(""),
 	}
 	return container
 }

pkg/operator/ceph/cluster/mon/spec.go

@@ -19,6 +19,7 @@ package mon
 import (
 	"fmt"
 	"os"
+	"path"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
@@ -90,6 +91,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod {
 	logger.Debug("monConfig: %+v", monConfig)
 	podSpec := v1.PodSpec{
 		InitContainers: []v1.Container{
+			c.makeChownDataDirInitContainer(monConfig),
 			c.makeMonFSInitContainer(monConfig),
 		},
 		Containers: []v1.Container{
@@ -134,7 +136,33 @@ func podSecurityContext() *v1.SecurityContext {
 	if os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true" {
 		privileged = true
 	}
-	return &v1.SecurityContext{Privileged: &privileged}
+
+	return &v1.SecurityContext{
+		Privileged: &privileged,
+	}
+}
+
+func (c *Cluster) makeChownDataDirInitContainer(monConfig *monConfig) v1.Container {
+	// Before makeMonFSInitContainer starts we must apply the right ownership to the mon data dir
+	// so the mkfs can succeed, otherwise it'll fail since it's owned by root
+	// Unfortunately, we can't use:
+	// Lifecycle: &v1.Lifecycle{PostStart: &v1.Handler{Exec: &v1.ExecAction{
+	// On an InitContainer :-(
+	container := v1.Container{
+		Name: "chown-container-data-dir",
+		Command: []string{
+			"chown",
+		},
+		Args: []string{
+			"--verbose",
+			"ceph:ceph",
+			monConfig.DataPathMap.ContainerDataDir,
+		},
+		Image:        c.spec.CephVersion.Image,
+		VolumeMounts: opspec.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName),
+		Resources:    cephv1.GetMonResources(c.spec.Resources),
+	}
+	return container
 }
 
 func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container {
@@ -182,6 +210,13 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
 			// If the mon is already in the monmap, when the port is left off of --public-addr,
 			// it will still advertise on the previous port b/c monmap is saved to mon database.
 			config.NewFlag("public-addr", publicAddr),
+			// Set '--setuser-match-path' so that existing directory owned by root won't affect the daemon startup.
+			// For existing data store owned by root, the daemon will continue to run as root
+			//
+			// We use 'store.db' here because during an upgrade the init container will set 'ceph:ceph' to monConfig.DataPathMap.ContainerDataDir
+			// but inside the permissions will be 'root:root' AND we don't want to chown recursively on the mon data directory
+			// We want to avoid potential startup time issue if the store is big
+			config.NewFlag("setuser-match-path", path.Join(monConfig.DataPathMap.ContainerDataDir, "store.db")),
 		),
 		Image:           c.spec.CephVersion.Image,
 		VolumeMounts:    opspec.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName),
@@ -198,6 +233,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
 			k8sutil.PodIPEnvVar(podIPEnvVar),
 		),
 		Resources: cephv1.GetMonResources(c.spec.Resources),
+		Lifecycle: opspec.PodLifeCycle(""),
 	}
 
 	// If host networking is enabled, we don't need a bind addr that is different from the public addr

pkg/operator/ceph/cluster/osd/spec.go

@@ -210,6 +210,11 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection,
 			"--osd-uuid", osd.UUID,
 			"--conf", osd.Config,
 			"--cluster", "ceph",
+			"--setuser", "ceph",
+			"--setgroup", "ceph",
+			// Set '--setuser-match-path' so that existing directory owned by root won't affect the daemon startup.
+			// For existing data store owned by root, the daemon will continue to run as root
+			"--setuser-match-path", osd.DataPath,
 		}
 
 		// Set osd memory target to the best appropriate value
@@ -260,6 +265,7 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection,
 	if c.HostNetwork {
 		DNSPolicy = v1.DNSClusterFirstWithHostNet
 	}
+
 	deployment := &apps.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf(osdAppNameFmt, osd.ID),
@@ -319,6 +325,7 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection,
 							Env:             envVars,
 							Resources:       resources,
 							SecurityContext: securityContext,
+							Lifecycle:       opspec.PodLifeCycle(osd.DataPath),
 						},
 					},
 					Volumes: volumes,

pkg/operator/ceph/cluster/rbd/spec.go

@@ -83,6 +83,7 @@ func (m *Mirroring) makeMirroringDaemonContainer(daemonConfig *daemonConfig) v1.
 		VolumeMounts: opspec.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName),
 		Env:          opspec.DaemonEnvVars(m.cephVersion.Image),
 		Resources:    m.resources,
+		Lifecycle:    opspec.PodLifeCycle(""),
 	}
 	return container
 }

pkg/operator/ceph/config/config.go

@@ -52,6 +52,12 @@ const (
 
 	// RbdMirrorType defines the rbd-mirror DaemonType
 	RbdMirrorType DaemonType = "rbd-mirror"
+
+	// CephUser is the Linux Ceph username
+	CephUser = "ceph"
+
+	// CephGroup is the Linux Ceph groupname
+	CephGroup = "ceph"
 )
 
 var (
@@ -66,6 +72,13 @@ var (
 	// VarLogCephDir defines Ceph logging directory. It is made overwriteable only for unit tests where it
 	// may be needed to send data intended for /var/log/ceph to a temporary test dir.
 	VarLogCephDir = "/var/log/ceph"
+
+	// chownUserGroup represents ceph:ceph
+	chownUserGroup = CephUser + ":" + CephGroup
+
+	// ContainerPostStartCmd is the command we run before starting any Ceph daemon
+	// It makes sure Ceph directories are owned by 'ceph'
+	ContainerPostStartCmd = []string{"chown", "--recursive", "--verbose", chownUserGroup, VarLogCephDir}
 )
 
 // Config is a Ceph config struct containing zero or more Ceph config sections.

pkg/operator/ceph/config/store.go

@@ -220,8 +220,17 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephconfig.ClusterInfo
 
 // StoredFileVolume returns a pod volume sourced from the stored config file.
 func StoredFileVolume() v1.Volume {
-	// TL;DR: mount the configmap's "ceph.conf" to a file called "ceph.conf" with 0400 permissions
-	mode := int32(0400) // security: only allow the owner to read and no one to write
+	// TL;DR: mount the configmap's "ceph.conf" to a file called "ceph.conf" with 0444 permissions
+	// security: allow to be read by everyone since now ceph processes run as 'ceph' and not 'root' user
+	// Further investigation needs to be done to copy the ceph.conf and change its ownership
+	// since configuring a owner of a ConfigMap secret is currently impossible
+	// This also works around the following issue: https://tracker.ceph.com/issues/38606
+	//
+	// This design choice avoids the crash/restart situation in Rook
+	// If we don't set 0444 to the ceph.conf configuration file during its respawn (with exec) the ceph-mgr
+	// won't be able to read the ceph.conf and the container will die, the "restart" count will increase in k8s
+	// This will mislead users thinking something won't wrong but that a false positive
+	mode := int32(0444)
 	return v1.Volume{
 		Name: configVolumeName,
 		VolumeSource: v1.VolumeSource{

pkg/operator/ceph/file/mds/spec.go

@@ -117,6 +117,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
 			opspec.DaemonEnvVars(c.cephVersion.Image),
 		),
 		Resources: c.fs.Spec.MetadataServer.Resources,
+		Lifecycle: opspec.PodLifeCycle(""),
 	}
 
 	return container

pkg/operator/ceph/object/spec.go

@@ -102,7 +102,6 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec
 }
 
 func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
-
 	// start the rgw daemon in the foreground
 	container := v1.Container{
 		Name:  "rgw",
@@ -134,6 +133,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
 			},
 			InitialDelaySeconds: 10,
 		},
+		Lifecycle: opspec.PodLifeCycle(""),
 	}
 
 	if c.store.Spec.Gateway.SSLCertificateRef != "" {

pkg/operator/ceph/spec/spec.go

@@ -109,13 +109,23 @@ func DaemonFlags(cluster *cephconfig.ClusterInfo, daemonID string) []string {
 	return append(
 		config.DefaultFlags(cluster.FSID, keyring.VolumeMount().KeyringFilePath(), cluster.CephVersion),
 		config.NewFlag("id", daemonID),
+		// Ceph daemons in Rook will run as 'ceph' instead of 'root'
+		// If we run on a version of Ceph does not these flags it will simply ignore them
+		//run ceph daemon process under the 'ceph' user
+		config.NewFlag("setuser", "ceph"),
+		// run ceph daemon process under the 'ceph' group
+		config.NewFlag("setgroup", "ceph"),
 	)
 
 }
 
 // AdminFlags returns the command line flags used for Ceph commands requiring admin authentication.
 func AdminFlags(cluster *cephconfig.ClusterInfo) []string {
-	return config.DefaultFlags(cluster.FSID, keyring.VolumeMount().AdminKeyringFilePath(), cluster.CephVersion)
+	return append(
+		config.DefaultFlags(cluster.FSID, keyring.VolumeMount().AdminKeyringFilePath(), cluster.CephVersion),
+		config.NewFlag("setuser", "ceph"),
+		config.NewFlag("setgroup", "ceph"),
+	)
 }
 
 // ContainerEnvVarReference returns a reference to a Kubernetes container env var of the given name
@@ -202,3 +212,26 @@ func StoredLogVolumeMount() v1.VolumeMount {
 		MountPath: config.VarLogCephDir,
 	}
 }
+
+func generateLifeCycleCmd(dataDirHostPath string) []string {
+	cmd := config.ContainerPostStartCmd
+
+	if dataDirHostPath != "" {
+		cmd = append(cmd, dataDirHostPath)
+	}
+
+	return cmd
+}
+
+// PodLifeCycle returns a pod lifecycle resource to execute actions before a pod starts
+func PodLifeCycle(dataDirHostPath string) *v1.Lifecycle {
+	cmd := generateLifeCycleCmd(dataDirHostPath)
+
+	return &v1.Lifecycle{
+		PostStart: &v1.Handler{
+			Exec: &v1.ExecAction{
+				Command: cmd,
+			},
+		},
+	}
+}

pkg/operator/ceph/spec/spec_test.go

@@ -19,8 +19,10 @@ package spec
 import (
 	"testing"
 
+	"github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/operator/test"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestPodVolumes(t *testing.T) {
@@ -42,3 +44,11 @@ func TestMountsMatchVolumes(t *testing.T) {
 	}
 	volsMountsTestDef.TestMountsMatchVolumes(t)
 }
+
+func TestGenerateLifeCycleCmd(t *testing.T) {
+	cmd := generateLifeCycleCmd("")
+	assert.Equal(t, config.ContainerPostStartCmd, cmd)
+
+	cmd = generateLifeCycleCmd("foo")
+	assert.Equal(t, append(config.ContainerPostStartCmd, "foo"), cmd)
+}