From e3d24106d9db4e160d5f26a1fcbb3cc3bf78f744 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Wed, 12 Sep 2018 13:08:55 -0600 Subject: [PATCH] Ceph op: Add spec package, shared vol/mnt defns Add a package called 'spec' to the Ceph operator to store resource definition specs that are shared by all Ceph daemons. Signed-off-by: Blaine Gardner --- pkg/operator/ceph/cluster/mon/spec.go | 42 +++--------------- pkg/operator/ceph/spec/spec.go | 61 +++++++++++++++++++++++++++ pkg/operator/ceph/spec/spec_test.go | 60 ++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 35 deletions(-) create mode 100644 pkg/operator/ceph/spec/spec.go create mode 100644 pkg/operator/ceph/spec/spec_test.go diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index f43e8eda3..510107b96 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -24,6 +24,7 @@ import ( cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" mondaemon "github.com/rook/rook/pkg/daemon/ceph/mon" + opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" @@ -74,32 +75,6 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, hostname string) *extensi return d } -/* - * Pod / container Volumes - */ - -// Volumes mounted to the pod -func (c *Cluster) podVolumes() []v1.Volume { - dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} - if c.dataDirHostPath != "" { - dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}} - } - return []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}, - k8sutil.ConfigOverrideVolume(), - cephconfig.DefaultConfigVolume(), - } -} - -// Mounts for containers running Ceph binaries -func cephMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, - // Rook doesn't run in ceph containers, so it doesn't need the config override mounted - cephconfig.DefaultConfigMount(), - } -} - /* * Pod spec */ @@ -108,7 +83,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { podSpec := v1.PodSpec{ InitContainers: []v1.Container{ // Config file init performed by Rook - c.makeConfigInitContainer(monConfig, "config-init"), + c.makeConfigInitContainer(monConfig, opspec.ConfigInitContainerName), // Ceph monmap init performed by 'monmaptool' c.makeMonmapInitContainer(monConfig, "monmap-init"), // mon filesystem init performed by mon daemon @@ -119,7 +94,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { }, RestartPolicy: v1.RestartPolicyAlways, NodeSelector: map[string]string{apis.LabelHostname: hostname}, - Volumes: c.podVolumes(), + Volumes: opspec.PodVolumes(c.dataDirHostPath), HostNetwork: c.HostNetwork, } if c.HostNetwork { @@ -177,10 +152,7 @@ func (c *Cluster) makeConfigInitContainer(monConfig *monConfig, containerName st AdminSecretEnvVar(), k8sutil.ConfigOverrideEnvVar(), }, - VolumeMounts: append( - cephMounts(), - k8sutil.ConfigOverrideMount(), - ), + VolumeMounts: opspec.RookVolumeMounts(), SecurityContext: podSecurityContext(), Resources: c.resources, } @@ -215,7 +187,7 @@ func (c *Cluster) makeMonmapInitContainer(monConfig *monConfig, containerName st monmapAddMonArgs..., ), Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph: image - VolumeMounts: cephMounts(), + VolumeMounts: opspec.CephVolumeMounts(), SecurityContext: podSecurityContext(), // monmap creation does not require ports to be exposed Resources: c.resources, @@ -251,7 +223,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig, containerName str c.cephMonCommonArgs(monConfig)..., ), Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph: image - VolumeMounts: cephMounts(), + VolumeMounts: opspec.CephVolumeMounts(), SecurityContext: podSecurityContext(), // filesystem creation does not require ports to be exposed Resources: c.resources, @@ -281,7 +253,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig, containerName str c.cephMonCommonArgs(monConfig)..., ), Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph: image - VolumeMounts: cephMounts(), + VolumeMounts: opspec.CephVolumeMounts(), SecurityContext: podSecurityContext(), Ports: []v1.ContainerPort{ { diff --git a/pkg/operator/ceph/spec/spec.go b/pkg/operator/ceph/spec/spec.go new file mode 100644 index 000000000..f2f1c68e1 --- /dev/null +++ b/pkg/operator/ceph/spec/spec.go @@ -0,0 +1,61 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package spec provides Kubernetes controller/pod/container spec items used for many Ceph daemons +package spec + +import ( + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" +) + +const ( + // ConfigInitContainerName is the name which is given to the config initialization container + // in all Ceph pods. + ConfigInitContainerName = "config-init" +) + +// PodVolumes returns the common list of Kubernetes volumes for use in Ceph pods. +func PodVolumes(dataDirHostPath string) []v1.Volume { + dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} + if dataDirHostPath != "" { + dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: dataDirHostPath}} + } + return []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}, + cephconfig.DefaultConfigVolume(), + k8sutil.ConfigOverrideVolume(), + } +} + +// CephVolumeMounts returns the common list of Kubernetes volume mounts for Ceph containers. +func CephVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, + cephconfig.DefaultConfigMount(), + // Rook doesn't run in ceph containers, so it doesn't need the config override mounted + + } +} + +// RookVolumeMounts returns the common list of Kubernetes volume mounts for Rook containers. +func RookVolumeMounts() []v1.VolumeMount { + return append( + CephVolumeMounts(), + k8sutil.ConfigOverrideMount(), + ) +} diff --git a/pkg/operator/ceph/spec/spec_test.go b/pkg/operator/ceph/spec/spec_test.go new file mode 100644 index 000000000..e73370665 --- /dev/null +++ b/pkg/operator/ceph/spec/spec_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spec + +import ( + "testing" + + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/testlib" +) + +func TestPodVolumes(t *testing.T) { + type args struct { + dataDirHostPath string + } + t.Run("Empty string dataDirHostPath is EmptyDir volume", func(t *testing.T) { + err := testlib.VolumeIsEmptyDir(k8sutil.DataDirVolume, PodVolumes("")) + if err != nil { + t.Errorf("PodVolumes(\"\") - dataDirHostPath is not EmptyDir: %s", err.Error()) + } + }) + t.Run("Specified dataDirHostPath is HostPath volume", func(t *testing.T) { + err := testlib.VolumeIsHostPath(k8sutil.DataDirVolume, "/dev/sdb", PodVolumes("/dev/sdb")) + if err != nil { + t.Errorf("PodVolumes(\"/dev/sdb\") - dataDirHostPath is not HostPath: %s", err.Error()) + } + }) +} + +func TestMountsMatchVolumes(t *testing.T) { + volsSpec := testlib.VolumesSpec{ + Moniker: "PodVolumes(\"/dev/sdc\")", + Volumes: PodVolumes("/dev/sdc"), + } + mountsSpecItems := []testlib.MountsSpec{ + { + Moniker: "CephVolumeMounts()", + Mounts: CephVolumeMounts(), + }, + { + Moniker: "RookVolumeMounts()", + Mounts: RookVolumeMounts(), + }, + } + testlib.TestMountsMatchVolumes(t, volsSpec, mountsSpecItems...) +}