From 9120791e24c2f911dc5935f2a78fdcbe7f8ea014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 18 Jul 2019 12:59:12 +0200 Subject: [PATCH] ceph: chown ceph directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the beginning of every container startup we now change the ownership of every known ceph directories to ceph:ceph. This will allow daemons to start properly as well as allowing daemon to writes their logs on files. This is needed because ceph daemon now run as 'ceph' instead of 'root' Signed-off-by: Sébastien Han --- pkg/operator/ceph/cluster/mgr/spec.go | 1 + pkg/operator/ceph/cluster/mon/spec.go | 1 + pkg/operator/ceph/cluster/osd/spec.go | 2 ++ pkg/operator/ceph/cluster/rbd/spec.go | 1 + pkg/operator/ceph/config/config.go | 13 +++++++++++++ pkg/operator/ceph/file/mds/spec.go | 1 + pkg/operator/ceph/object/spec.go | 2 +- pkg/operator/ceph/spec/spec.go | 23 +++++++++++++++++++++++ pkg/operator/ceph/spec/spec_test.go | 10 ++++++++++ 9 files changed, 53 insertions(+), 1 deletion(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 1eac50a51..e72a7d253 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -274,6 +274,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { }, InitialDelaySeconds: 60, }, + Lifecycle: opspec.PodLifeCycle(""), } return container } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 816ad2c40..a31bee0ce 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -233,6 +233,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: cephv1.GetMonResources(c.spec.Resources), + Lifecycle: opspec.PodLifeCycle(""), } // If host networking is enabled, we don't need a bind addr that is different from the public addr diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index c1697d963..95dc16c09 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -265,6 +265,7 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, if c.HostNetwork { DNSPolicy = v1.DNSClusterFirstWithHostNet } + deployment := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(osdAppNameFmt, osd.ID), @@ -324,6 +325,7 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, Env: envVars, Resources: resources, SecurityContext: securityContext, + Lifecycle: opspec.PodLifeCycle(osd.DataPath), }, }, Volumes: volumes, diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index c5ec8100c..2abad7a88 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -83,6 +83,7 @@ func (m *Mirroring) makeMirroringDaemonContainer(daemonConfig *daemonConfig) v1. VolumeMounts: opspec.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName), Env: opspec.DaemonEnvVars(m.cephVersion.Image), Resources: m.resources, + Lifecycle: opspec.PodLifeCycle(""), } return container } diff --git a/pkg/operator/ceph/config/config.go b/pkg/operator/ceph/config/config.go index c2c0bafe7..4fc0d5207 100644 --- a/pkg/operator/ceph/config/config.go +++ b/pkg/operator/ceph/config/config.go @@ -52,6 +52,12 @@ const ( // RbdMirrorType defines the rbd-mirror DaemonType RbdMirrorType DaemonType = "rbd-mirror" + + // CephUser is the Linux Ceph username + CephUser = "ceph" + + // CephGroup is the Linux Ceph groupname + CephGroup = "ceph" ) var ( @@ -66,6 +72,13 @@ var ( // VarLogCephDir defines Ceph logging directory. It is made overwriteable only for unit tests where it // may be needed to send data intended for /var/log/ceph to a temporary test dir. VarLogCephDir = "/var/log/ceph" + + // chownUserGroup represents ceph:ceph + chownUserGroup = CephUser + ":" + CephGroup + + // ContainerPostStartCmd is the command we run before starting any Ceph daemon + // It makes sure Ceph directories are owned by 'ceph' + ContainerPostStartCmd = []string{"chown", "--recursive", "--verbose", chownUserGroup, VarLogCephDir} ) // Config is a Ceph config struct containing zero or more Ceph config sections. diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 110ab1384..8cecb1e2e 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -117,6 +117,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { opspec.DaemonEnvVars(c.cephVersion.Image), ), Resources: c.fs.Spec.MetadataServer.Resources, + Lifecycle: opspec.PodLifeCycle(""), } return container diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 99c91d836..5fc899a17 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -102,7 +102,6 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec } func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { - // start the rgw daemon in the foreground container := v1.Container{ Name: "rgw", @@ -134,6 +133,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { }, InitialDelaySeconds: 10, }, + Lifecycle: opspec.PodLifeCycle(""), } if c.store.Spec.Gateway.SSLCertificateRef != "" { diff --git a/pkg/operator/ceph/spec/spec.go b/pkg/operator/ceph/spec/spec.go index a6805ad7a..2f029629c 100644 --- a/pkg/operator/ceph/spec/spec.go +++ b/pkg/operator/ceph/spec/spec.go @@ -212,3 +212,26 @@ func StoredLogVolumeMount() v1.VolumeMount { MountPath: config.VarLogCephDir, } } + +func generateLifeCycleCmd(dataDirHostPath string) []string { + cmd := config.ContainerPostStartCmd + + if dataDirHostPath != "" { + cmd = append(cmd, dataDirHostPath) + } + + return cmd +} + +// PodLifeCycle returns a pod lifecycle resource to execute actions before a pod starts +func PodLifeCycle(dataDirHostPath string) *v1.Lifecycle { + cmd := generateLifeCycleCmd(dataDirHostPath) + + return &v1.Lifecycle{ + PostStart: &v1.Handler{ + Exec: &v1.ExecAction{ + Command: cmd, + }, + }, + } +} diff --git a/pkg/operator/ceph/spec/spec_test.go b/pkg/operator/ceph/spec/spec_test.go index bf575541c..5a519593f 100644 --- a/pkg/operator/ceph/spec/spec_test.go +++ b/pkg/operator/ceph/spec/spec_test.go @@ -19,8 +19,10 @@ package spec import ( "testing" + "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" ) func TestPodVolumes(t *testing.T) { @@ -42,3 +44,11 @@ func TestMountsMatchVolumes(t *testing.T) { } volsMountsTestDef.TestMountsMatchVolumes(t) } + +func TestGenerateLifeCycleCmd(t *testing.T) { + cmd := generateLifeCycleCmd("") + assert.Equal(t, config.ContainerPostStartCmd, cmd) + + cmd = generateLifeCycleCmd("foo") + assert.Equal(t, append(config.ContainerPostStartCmd, "foo"), cmd) +}