diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 405b8bdbb..06da4cbcd 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -10,14 +10,14 @@ - Rook Ceph block storage provisioner can now correctly create erasure coded block images. See [Advanced Example: Erasure Coded Block Storage](Documentation/block.md#advanced-example-erasure-coded-block-storage) for an example usage. - [Network File System (NFS)](https://github.com/nfs-ganesha/nfs-ganesha/wiki) is now supported by Rook with a new operator to deploy and manage this widely used server. NFS servers can be automatically deployed by creating an instance of the new `nfsservers.nfs.rook.io` custom resource. See the [NFS server user guide](Documentation/nfs.md) to get started with NFS. - The minimum version of Kubernetes supported by Rook changed from `1.7` to `1.8`. -- `reclaimPolicy` parameter of `StorageClass` definition is now supported. +- `reclaimPolicy` parameter of `StorageClass` definition is now supported. ## Breaking Changes - Ceph mons are [named consistently](https://github.com/rook/rook/issues/1751) with other daemons with the letters a, b, c, etc. - Ceph mons are now created with Deployments instead of ReplicaSets to improve the upgrade implementation. -- Ceph mon container names in pods have changed with the +- Ceph mon and mgr container names in pods have changed with the [refactor](https://github.com/rook/rook/pull/2095) to initialize the mon daemon environment via - pod **InitContainers** and run the `ceph-mon` daemon directly from the container entrypoint. + pod **InitContainers** and run the Ceph daemons directly from the container entrypoint. - The Rook container images are no longer published to quay.io, they are published only to Docker Hub. All manifests have referenced Docker Hub for multiple releases now, so we do not expect any directly affected users from this change. - Rook no longer supports kubernetes `1.7`. Users running Kubernetes `1.7` on their clusters are recommended to upgrade to Kubernetes `1.8` or higher. If you are using `kubeadm`, you can follow this [guide](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade-1-8/) to from Kubernetes `1.7` to `1.8`. If you are using `kops` or `kubespray` for managing your Kubernetes cluster, just follow the respective projects' `upgrade` guide. diff --git a/cmd/rook/ceph/mgr.go b/cmd/rook/ceph/mgr.go index 6d0211bf8..78f2e6a38 100644 --- a/cmd/rook/ceph/mgr.go +++ b/cmd/rook/ceph/mgr.go @@ -30,23 +30,25 @@ var ( ) var mgrCmd = &cobra.Command{ - Use: "mgr", - Short: "Generates mgr config and runs the mgr daemon", + Use: mgrdaemon.InitCommand, + Short: "Generates mgr config", Hidden: true, } func init() { - mgrCmd.Flags().StringVar(&mgrName, "mgr-name", "", "the mgr name") + mgrCmd.Flags().StringVar(&mgrName, "mgr-name", "", "name of the mgr") mgrCmd.Flags().StringVar(&mgrKeyring, "mgr-keyring", "", "the mgr keyring") addCephFlags(mgrCmd) flags.SetFlagsFromEnv(mgrCmd.Flags(), rook.RookEnvVarPrefix) - mgrCmd.RunE = startMgr + mgrCmd.RunE = initMgr } -func startMgr(cmd *cobra.Command, args []string) error { - required := []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret"} +func initMgr(cmd *cobra.Command, args []string) error { + required := []string{ + "mgr-name", "mgr-keyring", + "mon-endpoints", "cluster-name", "mon-secret", "admin-secret"} if err := flags.VerifyRequiredFlags(mgrCmd, required); err != nil { return err } @@ -66,7 +68,7 @@ func startMgr(cmd *cobra.Command, args []string) error { ClusterInfo: &clusterInfo, } - err := mgrdaemon.Run(createContext(), config) + err := mgrdaemon.Initialize(createContext(), config) if err != nil { rook.TerminateFatal(err) } diff --git a/cmd/rook/ceph/mon.go b/cmd/rook/ceph/mon.go index 96824c5b8..50f4f2a24 100644 --- a/cmd/rook/ceph/mon.go +++ b/cmd/rook/ceph/mon.go @@ -31,7 +31,7 @@ import ( var monCmd = &cobra.Command{ Use: mondaemon.InitCommand, - Short: "Generates mon config and runs the mon daemon", + Short: "Generates mon config", Hidden: true, } diff --git a/pkg/daemon/ceph/config/info.go b/pkg/daemon/ceph/config/info.go index a65b54c44..c4bb88ce9 100644 --- a/pkg/daemon/ceph/config/info.go +++ b/pkg/daemon/ceph/config/info.go @@ -19,6 +19,9 @@ package config import ( "fmt" "net" + "strings" + + "github.com/coreos/pkg/capnslog" ) // ClusterInfo is a collection of information about a particular Ceph cluster. Rook uses information @@ -41,3 +44,23 @@ type MonInfo struct { func NewMonInfo(name, ip string, port int32) *MonInfo { return &MonInfo{Name: name, Endpoint: net.JoinHostPort(ip, fmt.Sprintf("%d", port))} } + +// Log writes the cluster info struct to the logger +func (c *ClusterInfo) Log(logger *capnslog.PackageLogger) { + mons := []string{} + for _, m := range c.Monitors { + mons = append(mons, fmt.Sprintf("{Name: %s, Endpoint: %s}", m.Name, m.Endpoint)) + } + monsec := "" + if c.MonitorSecret != "" { + monsec = "" + } + admsec := "" + if c.AdminSecret != "" { + admsec = "" + } + s := fmt.Sprintf( + "ClusterInfo: {FSID: %s, MonitorSecret: %s, AdminSecret: %s, Name: %s, Monitors: %s}", + c.FSID, monsec, admsec, c.Name, strings.Join(mons, " ")) + logger.Info(s) +} diff --git a/pkg/daemon/ceph/mgr/daemon.go b/pkg/daemon/ceph/mgr/init.go similarity index 71% rename from pkg/daemon/ceph/mgr/daemon.go rename to pkg/daemon/ceph/mgr/init.go index 647e7c3fa..30b6b81d8 100644 --- a/pkg/daemon/ceph/mgr/daemon.go +++ b/pkg/daemon/ceph/mgr/init.go @@ -38,30 +38,33 @@ var ( ) const ( + // InitCommand is the `rook ceph` subcommand which will perform mgr initialization + InitCommand = "mgr-init" + cephmgr = "ceph-mgr" ) +// Config contains the necessary parameters Rook needs to know to set up a mgr for a Ceph cluster. type Config struct { ClusterInfo *cephconfig.ClusterInfo Name string Keyring string } -func Run(context *clusterd.Context, config *Config) error { - logger.Infof("Starting MGR %s with keyring %s", config.Name, config.Keyring) +// Initialize generates configuration files for a Ceph mgr +func Initialize(context *clusterd.Context, config *Config) error { + logger.Infof("Creating config for MGR %s with keyring %s", config.Name, config.Keyring) + config.ClusterInfo.Log(logger) if err := generateConfigFiles(context, config); err != nil { return fmt.Errorf("failed to generate mgr config files. %+v", err) } - if err := startMgr(context, config); err != nil { - return fmt.Errorf("failed to run mgr. %+v", err) - } + util.WriteFileToLog(logger, cephconfig.DefaultConfigFilePath()) return nil } func generateConfigFiles(context *clusterd.Context, config *Config) error { - keyringPath := getMgrKeyringPath(context.ConfigDir, config.Name) confDir := getMgrConfDir(context.ConfigDir, config.Name) username := fmt.Sprintf("mgr.%s", config.Name) @@ -87,30 +90,6 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error { return nil } -func startMgr(context *clusterd.Context, config *Config) error { - - // start the mgr daemon in the foreground with the given config - logger.Infof("starting ceph-mgr") - - confFile := getMgrConfFilePath(context.ConfigDir, config.Name, config.ClusterInfo.Name) - util.WriteFileToLog(logger, confFile) - - keyringPath := getMgrKeyringPath(context.ConfigDir, config.Name) - util.WriteFileToLog(logger, keyringPath) - args := []string{ - "--foreground", - fmt.Sprintf("--cluster=%s", config.ClusterInfo.Name), - fmt.Sprintf("--conf=%s", confFile), - fmt.Sprintf("--keyring=%s", keyringPath), - "-i", config.Name, - } - - if err := context.Executor.ExecuteCommand(false, cephmgr, cephmgr, args...); err != nil { - return fmt.Errorf("failed to start mgr: %+v", err) - } - return nil -} - func getMgrConfDir(dir, name string) string { return path.Join(dir, fmt.Sprintf("mgr-%s", name)) } diff --git a/pkg/daemon/ceph/mon/init.go b/pkg/daemon/ceph/mon/init.go index a63fc8ed9..a00efbfa9 100644 --- a/pkg/daemon/ceph/mon/init.go +++ b/pkg/daemon/ceph/mon/init.go @@ -23,6 +23,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/util" ) const ( @@ -44,11 +45,15 @@ type Config struct { // Initialize generates configuration files for a Ceph mon func Initialize(context *clusterd.Context, config *Config) error { + logger.Infof("Creating config for MON %s with port %d", config.Name, config.Port) + config.Cluster.Log(logger) err := generateConfigFiles(context, config) if err != nil { return fmt.Errorf("failed to generate mon config files. %+v", err) } + util.WriteFileToLog(logger, cephconfig.DefaultConfigFilePath()) + return err } diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 7a027d9cd..f9c8de6ed 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -19,17 +19,14 @@ package mgr import ( "fmt" - "strconv" "github.com/coreos/pkg/capnslog" cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" - opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" - extensions "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -47,7 +44,7 @@ const ( var mgrNames = []string{"a", "b"} -// Cluster is the ceph mgr manager +// Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs. type Cluster struct { Namespace string Version string @@ -61,6 +58,12 @@ type Cluster struct { dashboard cephv1beta1.DashboardSpec } +// mgrConfig for a single mgr +type mgrConfig struct { + ResourceName string // the name rook gives to mgr resources in k8s metadata + DaemonName string // the name of the Ceph daemon ("a", "b", ...) +} + // New creates an instance of the mgr func New(context *clusterd.Context, namespace, version string, placement rookalpha.Placement, hostNetwork bool, dashboard cephv1beta1.DashboardSpec, resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { @@ -78,7 +81,7 @@ func New(context *clusterd.Context, namespace, version string, placement rookalp } } -// Start the mgr instance +// Start begins the process of running a cluster of Ceph mgrs. func (c *Cluster) Start() error { logger.Infof("start running mgr") @@ -87,21 +90,27 @@ func (c *Cluster) Start() error { logger.Errorf("cannot have more than %d mgrs", len(mgrNames)) break } + daemonName := mgrNames[i] - name := fmt.Sprintf("%s-%s", appName, daemonName) - if err := c.createKeyring(c.Namespace, name, daemonName); err != nil { - return fmt.Errorf("failed to create %s keyring. %+v", name, err) + resourceName := fmt.Sprintf("%s-%s", appName, daemonName) + if err := c.createKeyring(c.Namespace, resourceName, daemonName); err != nil { + return fmt.Errorf("failed to create %s keyring. %+v", resourceName, err) + } + + mgrConfig := &mgrConfig{ + DaemonName: daemonName, + ResourceName: resourceName, } // start the deployment - deployment := c.makeDeployment(name, daemonName) + deployment := c.makeDeployment(mgrConfig) if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment); err != nil { if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create %s deployment. %+v", name, err) + return fmt.Errorf("failed to create %s deployment. %+v", resourceName, err) } - logger.Infof("%s deployment already exists", name) + logger.Infof("%s deployment already exists", resourceName) } else { - logger.Infof("%s deployment started", name) + logger.Infof("%s deployment started", resourceName) } } @@ -200,101 +209,6 @@ func (c *Cluster) makeDashboardService(name string) *v1.Service { return svc } -func (c *Cluster) makeDeployment(name, daemonName string) *extensions.Deployment { - - podSpec := v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: c.getDaemonLabels(daemonName), - Annotations: map[string]string{"prometheus.io/scrape": "true", - "prometheus.io/port": strconv.Itoa(metricsPort)}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{c.mgrContainer(name, daemonName)}, - RestartPolicy: v1.RestartPolicyAlways, - Volumes: []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, - k8sutil.ConfigOverrideVolume(), - }, - HostNetwork: c.HostNetwork, - }, - } - if c.HostNetwork { - podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet - } - c.placement.ApplyToPodSpec(&podSpec.Spec) - - replicas := int32(1) - d := &extensions.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: c.Namespace, - }, - Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &replicas}, - } - k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &d.ObjectMeta, &c.ownerRef) - return d -} - -func (c *Cluster) mgrContainer(name, daemonName string) v1.Container { - - return v1.Container{ - Args: []string{ - "ceph", - "mgr", - fmt.Sprintf("--config-dir=%s", k8sutil.DataDir), - }, - Name: name, - Image: k8sutil.MakeRookImage(c.Version), - VolumeMounts: []v1.VolumeMount{ - {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, - k8sutil.ConfigOverrideMount(), - }, - Env: []v1.EnvVar{ - {Name: "ROOK_MGR_NAME", Value: daemonName}, - {Name: "ROOK_MGR_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: name}, Key: keyringName}}}, - k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), - k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), - opmon.ClusterNameEnvVar(c.Namespace), - opmon.EndpointEnvVar(), - opmon.SecretEnvVar(), - opmon.AdminSecretEnvVar(), - k8sutil.ConfigOverrideEnvVar(), - }, - Resources: c.resources, - Ports: []v1.ContainerPort{ - { - Name: "mgr", - ContainerPort: int32(6800), - Protocol: v1.ProtocolTCP, - }, - { - Name: "http-metrics", - ContainerPort: int32(metricsPort), - Protocol: v1.ProtocolTCP, - }, - { - Name: "dashboard", - ContainerPort: int32(dashboardPort), - Protocol: v1.ProtocolTCP, - }, - }, - } -} - -func (c *Cluster) getLabels() map[string]string { - return map[string]string{ - k8sutil.AppAttr: appName, - k8sutil.ClusterAttr: c.Namespace, - } -} - -func (c *Cluster) getDaemonLabels(daemonName string) map[string]string { - labels := c.getLabels() - labels["instance"] = daemonName - return labels -} - func (c *Cluster) createKeyring(clusterName, name, daemonName string) error { _, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(name, metav1.GetOptions{}) if err == nil { diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index 1d723d197..8b5227ca4 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -19,7 +19,6 @@ import ( "fmt" "io/ioutil" "os" - "strconv" "testing" cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" @@ -30,7 +29,6 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -85,60 +83,3 @@ func validateStart(t *testing.T, c *Cluster) { assert.True(t, errors.IsNotFound(err)) } } - -func TestPodSpec(t *testing.T) { - c := New(&clusterd.Context{Clientset: testop.New(1)}, "ns", "rook/rook:myversion", rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{}, v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI), - }, - Requests: v1.ResourceList{ - v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), - }, - }, metav1.OwnerReference{}) - - d := c.makeDeployment("mgr-a", "a") - assert.NotNil(t, d) - assert.Equal(t, "mgr-a", d.Name) - assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy) - assert.Equal(t, 2, len(d.Spec.Template.Spec.Volumes)) - assert.Equal(t, 3, len(d.Spec.Template.Spec.Containers[0].Ports)) - assert.Equal(t, "rook-data", d.Spec.Template.Spec.Volumes[0].Name) - assert.Equal(t, "mgr-a", d.ObjectMeta.Name) - assert.Equal(t, appName, d.Spec.Template.ObjectMeta.Labels["app"]) - assert.Equal(t, c.Namespace, d.Spec.Template.ObjectMeta.Labels["rook_cluster"]) - assert.Equal(t, 0, len(d.ObjectMeta.Annotations)) - - assert.Equal(t, 2, len(d.Spec.Template.ObjectMeta.Annotations)) - assert.Equal(t, "true", d.Spec.Template.ObjectMeta.Annotations["prometheus.io/scrape"]) - assert.Equal(t, strconv.Itoa(metricsPort), d.Spec.Template.ObjectMeta.Annotations["prometheus.io/port"]) - - cont := d.Spec.Template.Spec.Containers[0] - assert.Equal(t, "rook/rook:myversion", cont.Image) - assert.Equal(t, 2, len(cont.VolumeMounts)) - - assert.Equal(t, "ceph", cont.Args[0]) - assert.Equal(t, "mgr", cont.Args[1]) - assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[2]) - - assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) - assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) -} - -func TestServiceSpec(t *testing.T) { - c := New(&clusterd.Context{}, "ns", "myversion", rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) - - s := c.makeMetricsService("rook-mgr") - assert.NotNil(t, s) - assert.Equal(t, "rook-mgr", s.Name) - assert.Equal(t, 1, len(s.Spec.Ports)) -} - -func TestHostNetwork(t *testing.T) { - c := New(&clusterd.Context{Clientset: testop.New(1)}, "ns", "myversion", rookalpha.Placement{}, true, cephv1beta1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) - - d := c.makeDeployment("mgr-a", "a") - assert.NotNil(t, d) - - assert.Equal(t, true, d.Spec.Template.Spec.HostNetwork) - assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy) -} diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go new file mode 100644 index 000000000..8a99d15ad --- /dev/null +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -0,0 +1,149 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mgr + +import ( + "fmt" + "strconv" + + mgrdaemon "github.com/rook/rook/pkg/daemon/ceph/mgr" + opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + opspec "github.com/rook/rook/pkg/operator/ceph/spec" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + mgrDaemonCommand = "ceph-mgr" +) + +func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *extensions.Deployment { + podSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: mgrConfig.ResourceName, + Labels: c.getDaemonLabels(mgrConfig.DaemonName), + Annotations: map[string]string{"prometheus.io/scrape": "true", + "prometheus.io/port": strconv.Itoa(metricsPort)}, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + // Config file init performed by Rook + c.makeConfigInitContainer(mgrConfig), + }, + Containers: []v1.Container{ + c.makeMgrDaemonContainer(mgrConfig), + }, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: opspec.PodVolumes(""), + HostNetwork: c.HostNetwork, + }, + } + if c.HostNetwork { + podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } + c.placement.ApplyToPodSpec(&podSpec.Spec) + + replicas := int32(1) + d := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: mgrConfig.ResourceName, + Namespace: c.Namespace, + }, + Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &replicas}, + } + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &d.ObjectMeta, &c.ownerRef) + return d +} + +func (c *Cluster) makeConfigInitContainer(mgrConfig *mgrConfig) v1.Container { + return v1.Container{ + Name: opspec.ConfigInitContainerName, + Args: []string{ + "ceph", + mgrdaemon.InitCommand, + fmt.Sprintf("--config-dir=%s", k8sutil.DataDir), + fmt.Sprintf("--mgr-name=%s", mgrConfig.DaemonName), + }, + Image: k8sutil.MakeRookImage(c.Version), + Env: []v1.EnvVar{ + {Name: "ROOK_MGR_KEYRING", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{Name: mgrConfig.ResourceName}, + Key: keyringName, + }}}, + k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), + k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), + opmon.ClusterNameEnvVar(c.Namespace), + opmon.EndpointEnvVar(), + opmon.SecretEnvVar(), + opmon.AdminSecretEnvVar(), + k8sutil.ConfigOverrideEnvVar(), + }, + VolumeMounts: opspec.RookVolumeMounts(), + // config file creation does not require ports to be open + Resources: c.resources, + } +} + +func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { + return v1.Container{ + Name: "mgr", + Command: []string{ + mgrDaemonCommand, + }, + Args: []string{ + "--foreground", + "--id", mgrConfig.DaemonName, + }, + Image: k8sutil.MakeRookImage(c.Version), + VolumeMounts: opspec.CephVolumeMounts(), + Ports: []v1.ContainerPort{ + { + Name: "mgr", + ContainerPort: int32(6800), + Protocol: v1.ProtocolTCP, + }, + { + Name: "http-metrics", + ContainerPort: int32(metricsPort), + Protocol: v1.ProtocolTCP, + }, + { + Name: "dashboard", + ContainerPort: int32(dashboardPort), + Protocol: v1.ProtocolTCP, + }, + }, + Resources: c.resources, + } +} + +func (c *Cluster) getLabels() map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: c.Namespace, + } +} + +func (c *Cluster) getDaemonLabels(daemonName string) map[string]string { + labels := c.getLabels() + labels["instance"] = daemonName + return labels +} diff --git a/pkg/operator/ceph/cluster/mgr/spec_test.go b/pkg/operator/ceph/cluster/mgr/spec_test.go new file mode 100644 index 000000000..7ba517f09 --- /dev/null +++ b/pkg/operator/ceph/cluster/mgr/spec_test.go @@ -0,0 +1,178 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mgr + +import ( + "fmt" + "strconv" + "testing" + + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + mgrdaemon "github.com/rook/rook/pkg/daemon/ceph/mgr" + cephtest "github.com/rook/rook/pkg/operator/ceph/test" + "github.com/rook/rook/pkg/operator/k8sutil" + optest "github.com/rook/rook/pkg/operator/test" + testop "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestPodSpec(t *testing.T) { + c := New( + &clusterd.Context{Clientset: testop.New(1)}, + "ns", + "rook/rook:myversion", + rookalpha.Placement{}, + false, + cephv1beta1.DashboardSpec{}, + v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), + }, + }, + metav1.OwnerReference{}, + ) + + mgrTestConfig := mgrConfig{ + DaemonName: "a", + ResourceName: "mgr-a", + } + + d := c.makeDeployment(&mgrTestConfig) + + assert.NotNil(t, d) + assert.Equal(t, "mgr-a", d.Name) + assert.Equal(t, "mgr-a", d.ObjectMeta.Name) + assert.Equal(t, 0, len(d.ObjectMeta.Annotations)) + + pod := d.Spec.Template + assert.Equal(t, appName, pod.ObjectMeta.Labels["app"]) + assert.Equal(t, c.Namespace, pod.ObjectMeta.Labels["rook_cluster"]) + assert.Equal(t, 2, len(pod.ObjectMeta.Annotations)) + assert.Equal(t, "true", pod.ObjectMeta.Annotations["prometheus.io/scrape"]) + assert.Equal(t, strconv.Itoa(metricsPort), pod.ObjectMeta.Annotations["prometheus.io/port"]) + assert.Equal(t, v1.RestartPolicyAlways, pod.Spec.RestartPolicy) + assert.Nil(t, optest.VolumeExists("rook-data", pod.Spec.Volumes)) + assert.Nil(t, optest.VolumeExists(cephconfig.DefaultConfigMountName, pod.Spec.Volumes)) + assert.Nil(t, optest.VolumeExists(k8sutil.ConfigOverrideName, pod.Spec.Volumes)) + + assert.Equal(t, 1, len(pod.Spec.InitContainers)) + assert.Equal(t, 1, len(pod.Spec.Containers)) + + configImage := "rook/rook:myversion" + configEnvs := 8 + configContainerDefinition := cephtest.ContainerTestDefinition{ + Image: &configImage, + Command: []string{}, // no command + Args: [][]string{ + {"ceph"}, + {mgrdaemon.InitCommand}, + {"--config-dir=/var/lib/rook"}, + {fmt.Sprintf("--mgr-name=%s", mgrTestConfig.DaemonName)}}, + InOrderArgs: map[int]string{ + 0: "ceph", // ceph must be first arg + 1: mgrdaemon.InitCommand}, // mgr init command must be second arg + VolumeMountNames: []string{ + "rook-data", + cephconfig.DefaultConfigMountName, + k8sutil.ConfigOverrideName}, + EnvCount: &configEnvs, + Ports: []v1.ContainerPort{}, + IsPrivileged: nil, // not set in spec + } + cont := &pod.Spec.InitContainers[0] + configContainerDefinition.TestContainer(t, "config init", cont, logger) + assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) + assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) + + daemonImage := "rook/rook:myversion" + daemonEnvs := 0 + daemonContainerDefinition := cephtest.ContainerTestDefinition{ + Image: &daemonImage, + Command: []string{ + "ceph-mgr"}, + Args: [][]string{ + {"--foreground"}, + {"--id", mgrTestConfig.DaemonName}}, + VolumeMountNames: []string{ + "rook-data", + cephconfig.DefaultConfigMountName}, + EnvCount: &daemonEnvs, + Ports: []v1.ContainerPort{ + {ContainerPort: int32(6800), + Protocol: v1.ProtocolTCP}, + {ContainerPort: int32(metricsPort), + Protocol: v1.ProtocolTCP}, + {ContainerPort: int32(dashboardPort), + Protocol: v1.ProtocolTCP}}, + IsPrivileged: nil, // not set in spec + } + cont = &pod.Spec.Containers[0] + daemonContainerDefinition.TestContainer(t, "main mon daemon", cont, logger) + assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) + assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) + + // Verify that all the mounts have volumes and that there are no extraneous volumes + volsMountsTestDef := optest.VolumesAndMountsTestDefinition{ + VolumesSpec: &optest.VolumesSpec{Moniker: "mon pod volumes", Volumes: pod.Spec.Volumes}, + MountsSpecItems: []*optest.MountsSpec{ + {Moniker: "mgr config init mounts", Mounts: pod.Spec.InitContainers[0].VolumeMounts}, + {Moniker: "mgr daemon mounts", Mounts: pod.Spec.Containers[0].VolumeMounts}}, + } + volsMountsTestDef.TestMountsMatchVolumes(t) +} + +func TestServiceSpec(t *testing.T) { + c := New(&clusterd.Context{}, "ns", "myversion", rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + s := c.makeMetricsService("rook-mgr") + assert.NotNil(t, s) + assert.Equal(t, "rook-mgr", s.Name) + assert.Equal(t, 1, len(s.Spec.Ports)) +} + +func TestHostNetwork(t *testing.T) { + c := New( + &clusterd.Context{Clientset: testop.New(1)}, + "ns", + "myversion", + rookalpha.Placement{}, + true, + cephv1beta1.DashboardSpec{}, + v1.ResourceRequirements{}, + metav1.OwnerReference{}, + ) + + mgrTestConfig := mgrConfig{ + DaemonName: "a", + ResourceName: "mgr-a", + } + + d := c.makeDeployment(&mgrTestConfig) + assert.NotNil(t, d) + + assert.Equal(t, true, d.Spec.Template.Spec.HostNetwork) + assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy) +} diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index fd52d2bb8..2e556063c 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -69,7 +69,7 @@ const ( MaxMonCount = 9 ) -// Cluster represents the Rook configuration settings for Ceph mons. +// Cluster represents the Rook and environment configuration settings needed to set up Ceph mons. type Cluster struct { context *clusterd.Context Namespace string diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index f46a683d1..b7f853eaf 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -35,7 +35,7 @@ const ( // Full path of command used to invoke the monmap tool monmaptoolCommand = "/usr/bin/monmaptool" // Full path of the command used to invoke the Ceph mon daemon - cephMonCommand = "/usr/bin/ceph-mon" + cephMonCommand = "ceph-mon" monmapFile = "monmap" ) @@ -82,14 +82,14 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { podSpec := v1.PodSpec{ InitContainers: []v1.Container{ // Config file init performed by Rook - c.makeConfigInitContainer(monConfig, opspec.ConfigInitContainerName), + c.makeConfigInitContainer(monConfig), // Ceph monmap init performed by 'monmaptool' - c.makeMonmapInitContainer(monConfig, "monmap-init"), + c.makeMonmapInitContainer(monConfig), // mon filesystem init performed by mon daemon - c.makeMonFSInitContainer(monConfig, "mon-fs-init"), + c.makeMonFSInitContainer(monConfig), }, Containers: []v1.Container{ - c.makeMonDaemonContainer(monConfig, "mon"), + c.makeMonDaemonContainer(monConfig), }, RestartPolicy: v1.RestartPolicyAlways, NodeSelector: map[string]string{apis.LabelHostname: hostname}, @@ -130,9 +130,9 @@ func podSecurityContext() *v1.SecurityContext { return &v1.SecurityContext{Privileged: &privileged} } -func (c *Cluster) makeConfigInitContainer(monConfig *monConfig, containerName string) v1.Container { +func (c *Cluster) makeConfigInitContainer(monConfig *monConfig) v1.Container { return v1.Container{ - Name: containerName, + Name: opspec.ConfigInitContainerName, Args: []string{ "ceph", mondaemon.InitCommand, @@ -164,7 +164,7 @@ func (c *Cluster) monmapFilePath(monConfig *monConfig) string { ) } -func (c *Cluster) makeMonmapInitContainer(monConfig *monConfig, containerName string) v1.Container { +func (c *Cluster) makeMonmapInitContainer(monConfig *monConfig) v1.Container { // Add mons w/ monmaptool w/ args: [--add ]... monmapAddMonArgs := []string{} for _, mon := range c.clusterInfo.Monitors { @@ -172,7 +172,7 @@ func (c *Cluster) makeMonmapInitContainer(monConfig *monConfig, containerName st } return v1.Container{ - Name: containerName, + Name: "monmap-init", Command: []string{ monmaptoolCommand, }, @@ -201,9 +201,9 @@ func (c *Cluster) cephMonCommonArgs(monConfig *monConfig) []string { } } -func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig, containerName string) v1.Container { +func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container { return v1.Container{ - Name: containerName, + Name: "mon-fs-init", Command: []string{ cephMonCommand, }, @@ -222,7 +222,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig, containerName str } } -func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig, containerName string) v1.Container { +func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { return v1.Container{ // The operator has set up the mon's service already, so the IP that the mon should // broadcast as its own (--public-addr) is known. But the pod's IP, which the mon should @@ -232,7 +232,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig, containerName str // - Chosen solution, but is not as transparent to inspection as using commandline arg // 2. Use bash to do variable substitution with the pod IP env var; but bash is a poor PID1 // 3. Use tini to do var substitution as above; but tini doesn't exist in the ceph images. - Name: containerName, + Name: "mon", Command: []string{ cephMonCommand, }, diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index 8fe6112dd..15884a607 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -153,7 +153,7 @@ func testPodSpec(t *testing.T, dataDir string) { monFsContDev := test_opceph.ContainerTestDefinition{ Image: &cephImage, Command: []string{ - "/usr/bin/ceph-mon"}, + "ceph-mon"}, Args: append( monCommonExpectedArgs(name, c), []string{"--mkfs"}, @@ -172,7 +172,7 @@ func testPodSpec(t *testing.T, dataDir string) { monDaemonContDev := test_opceph.ContainerTestDefinition{ Image: &cephImage, Command: []string{ - "/usr/bin/ceph-mon"}, + "ceph-mon"}, Args: append( monCommonExpectedArgs(name, c), []string{"--foreground"}, diff --git a/pkg/operator/ceph/spec/spec_test.go b/pkg/operator/ceph/spec/spec_test.go index 14766eeb5..0cf3f0569 100644 --- a/pkg/operator/ceph/spec/spec_test.go +++ b/pkg/operator/ceph/spec/spec_test.go @@ -24,21 +24,12 @@ import ( ) func TestPodVolumes(t *testing.T) { - type args struct { - dataDirHostPath string + if err := test.VolumeIsEmptyDir(k8sutil.DataDirVolume, PodVolumes("")); err != nil { + t.Errorf("PodVolumes(\"\") - data dir source is not EmptyDir: %s", err.Error()) + } + if err := test.VolumeIsHostPath(k8sutil.DataDirVolume, "/dev/sdb", PodVolumes("/dev/sdb")); err != nil { + t.Errorf("PodVolumes(\"/dev/sdb\") - data dir source is not HostPath: %s", err.Error()) } - t.Run("Empty string dataDirHostPath is EmptyDir volume", func(t *testing.T) { - err := test.VolumeIsEmptyDir(k8sutil.DataDirVolume, PodVolumes("")) - if err != nil { - t.Errorf("PodVolumes(\"\") - dataDirHostPath is not EmptyDir: %s", err.Error()) - } - }) - t.Run("Specified dataDirHostPath is HostPath volume", func(t *testing.T) { - err := test.VolumeIsHostPath(k8sutil.DataDirVolume, "/dev/sdb", PodVolumes("/dev/sdb")) - if err != nil { - t.Errorf("PodVolumes(\"/dev/sdb\") - dataDirHostPath is not HostPath: %s", err.Error()) - } - }) } func TestMountsMatchVolumes(t *testing.T) { diff --git a/pkg/operator/ceph/test/container.go b/pkg/operator/ceph/test/container.go index ad4a66c8b..58871e715 100644 --- a/pkg/operator/ceph/test/container.go +++ b/pkg/operator/ceph/test/container.go @@ -1,3 +1,21 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package test provides common resources useful for testing many Ceph daemons. This includes +// functions for testing that resources match what is expected. package test import ( diff --git a/pkg/operator/test/client.go b/pkg/operator/test/client.go index ab1060e31..195da08e6 100644 --- a/pkg/operator/test/client.go +++ b/pkg/operator/test/client.go @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package test for the operator tests. package test import ( diff --git a/pkg/operator/test/info.go b/pkg/operator/test/info.go index dfde5406f..ebdbb5ba4 100644 --- a/pkg/operator/test/info.go +++ b/pkg/operator/test/info.go @@ -14,7 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package test for the operator tests +// Package test provides common resources useful for testing many operators. This includes functions +// for creating fake/mock resources and functions for testing that resources match what is expected. package test import ( diff --git a/pkg/operator/test/spec.go b/pkg/operator/test/spec.go index 6a4c8cf1f..660784182 100644 --- a/pkg/operator/test/spec.go +++ b/pkg/operator/test/spec.go @@ -1,7 +1,19 @@ -// Package testlib provides common methods for testing code which applies to many Ceph daemons. -// -// Methods beginning with "TestSpec" can be used to test that Kubernetes resource specs (pods, etc.) -// are configured correctly. +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package test import ( diff --git a/pkg/operator/test/spec_test.go b/pkg/operator/test/spec_test.go index 2c8163c4e..e647bd671 100644 --- a/pkg/operator/test/spec_test.go +++ b/pkg/operator/test/spec_test.go @@ -1,7 +1,19 @@ -// Package testlib provides common methods for testing code which applies to many Ceph daemons. -// -// Methods beginning with "TestSpec" can be used to test that Kubernetes resource specs (pods, etc.) -// are configured correctly. +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package test import "testing" diff --git a/pkg/operator/test/volumes.go b/pkg/operator/test/volumes.go index 929e1f45e..ebc9748d6 100644 --- a/pkg/operator/test/volumes.go +++ b/pkg/operator/test/volumes.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package test import ( diff --git a/pkg/operator/test/volumes_test.go b/pkg/operator/test/volumes_test.go index eb1d7c5a8..1b6eeb6ce 100644 --- a/pkg/operator/test/volumes_test.go +++ b/pkg/operator/test/volumes_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package test import (