diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index f651a4f87..bb4fdc703 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -29,8 +29,8 @@ - Rook will no longer create a directory-based osd in the `dataDirHostPath` if no directories or devices are specified or if there are no disks on the host. -- Containers in `mon` and `mgr` pods have been removed and/or changed names. -- Config paths in `mon` and `mgr` containers are now always the Ceph default paths +- Containers in `mon`, `mgr`, and `mds` pods have been removed and/or changed names. +- Config paths in `mon`, `mgr`, and `mds` containers are now always the Ceph default paths (`/etc/ceph`, `/var/lib/ceph/...`) regardless of the `dataDirHostPath` setting. ## Known Issues diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index 1d9fa223f..70e0f7df3 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -59,7 +59,6 @@ func init() { agentCmd, osdCmd, rgwCmd, - mdsCmd, configCmd, nfsCmd) } diff --git a/cmd/rook/ceph/mds.go b/cmd/rook/ceph/mds.go deleted file mode 100644 index dd57d2a6c..000000000 --- a/cmd/rook/ceph/mds.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ceph - -import ( - "github.com/rook/rook/cmd/rook/rook" - mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds" - "github.com/rook/rook/pkg/operator/ceph/cluster/mon" - "github.com/rook/rook/pkg/util/flags" - "github.com/spf13/cobra" -) - -var ( - mdsName string - filesystemID string - activeStandby bool - keyring string -) - -var mdsCmd = &cobra.Command{ - Use: mdsdaemon.InitCommand, - Short: "Generates mds config and runs the mds daemon", -} - -func init() { - mdsCmd.Flags().StringVar(&mdsName, "mds-name", "", "name of the mds") - mdsCmd.Flags().StringVar(&keyring, "mds-keyring", "", "the mds keyring") - mdsCmd.Flags().StringVar(&filesystemID, "filesystem-id", "", "ID of the filesystem the mds will serve") - mdsCmd.Flags().BoolVar(&activeStandby, "active-standby", true, "whether to start in active standby mode") - addCephFlags(mdsCmd) - - flags.SetFlagsFromEnv(mdsCmd.Flags(), rook.RookEnvVarPrefix) - - mdsCmd.RunE = initMds -} - -func initMds(cmd *cobra.Command, args []string) error { - required := []string{"mon-endpoints", "cluster-name", "admin-secret", - "mds-name", "mds-keyring", "filesystem-id"} - if err := flags.VerifyRequiredFlags(mdsCmd, required); err != nil { - return err - } - - if err := verifyRenamedFlags(mdsCmd); err != nil { - return err - } - - rook.SetLogLevel() - - rook.LogStartupInfo(mdsCmd.Flags()) - - clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) - config := &mdsdaemon.Config{ - FilesystemID: filesystemID, - Name: mdsName, - ActiveStandby: activeStandby, - Keyring: keyring, - ClusterInfo: &clusterInfo, - } - - err := mdsdaemon.Initialize(createContext(), config) - if err != nil { - rook.TerminateFatal(err) - } - - return nil -} diff --git a/pkg/daemon/ceph/config/info.go b/pkg/daemon/ceph/config/info.go index e5e6ddfb1..ae646d0aa 100644 --- a/pkg/daemon/ceph/config/info.go +++ b/pkg/daemon/ceph/config/info.go @@ -41,6 +41,18 @@ type MonInfo struct { Endpoint string `json:"endpoint"` } +// IsInitialized returns true if the critical information in the ClusterInfo struct has been filled +// in. This method exists less out of necessity than the desire to be explicit about the lifecycle +// of the ClusterInfo struct during startup, specifically that it is expected to exist after the +// Rook operator has started up or connected to the first components of the Ceph cluster. +func (c *ClusterInfo) IsInitialized() bool { + if c == nil || c.FSID == "" || c.MonitorSecret == "" || c.AdminSecret == "" { + logger.Errorf("clusterInfo: %+v", c) + return false + } + return true +} + // NewMonInfo returns a new Ceph mon info struct from the given inputs. func NewMonInfo(name, ip string, port int32) *MonInfo { return &MonInfo{Name: name, Endpoint: net.JoinHostPort(ip, fmt.Sprintf("%d", port))} diff --git a/pkg/daemon/ceph/mds/filesystem.go b/pkg/daemon/ceph/mds/filesystem.go deleted file mode 100644 index 79c2bd59a..000000000 --- a/pkg/daemon/ceph/mds/filesystem.go +++ /dev/null @@ -1,185 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mds - -import ( - "fmt" - "time" - - "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/daemon/ceph/model" -) - -const ( - dataPoolSuffix = "data" - metadataPoolSuffix = "metadata" - appName = "cephfs" -) - -// Filesystem represents an instance of a Ceph file system (CephFS) -type Filesystem struct { - Name string - metadataPool *model.Pool - dataPools []*model.Pool - activeMDSCount int32 -} - -// NewFS creates a new instance of the file (MDS) service -func NewFS(name string, metadataPool *model.Pool, dataPools []*model.Pool, activeMDSCount int32) *Filesystem { - - metadataPool.Name = fmt.Sprintf("%s-%s", name, metadataPoolSuffix) - for i, pool := range dataPools { - pool.Name = fmt.Sprintf("%s-%s%d", name, dataPoolSuffix, i) - } - - return &Filesystem{ - Name: name, - metadataPool: metadataPool, - dataPools: dataPools, - activeMDSCount: activeMDSCount, - } -} - -// CreateFilesystem starts the Ceph file daemons and creates the filesystem in Ceph. -func (f *Filesystem) CreateFilesystem(context *clusterd.Context, clusterName string) error { - _, err := client.GetFilesystem(context, clusterName, f.Name) - if err == nil { - logger.Infof("file system %s already exists", f.Name) - // Even if the fs already exists, the num active mdses may have changed - if err := client.SetNumMDSRanks(context, clusterName, f.Name, f.activeMDSCount); err != nil { - logger.Errorf( - fmt.Sprintf("failed to set num mds ranks (max_mds) to %d for filesystem %s, still continuing. ", f.activeMDSCount, f.Name) + - "this error is not critical, but mdses may not be as failure tolerant as desired. " + - fmt.Sprintf("USER should verify that the number of active mdses is %d with 'ceph fs get %s'", f.activeMDSCount, f.Name) + - fmt.Sprintf(": %+v", err), - ) - } - return nil - } - if len(f.dataPools) == 0 { - return fmt.Errorf("at least one data pool must be specified") - } - - fslist, err := client.ListFilesystems(context, clusterName) - if err != nil { - return fmt.Errorf("Unable to list existing file systems: %+v", err) - } - if len(fslist) > 0 && !client.IsMultiFSEnabled() { - return fmt.Errorf("Cannot create multiple filesystems. Enable %s env variable to create more than one", client.MultiFsEnv) - } - - logger.Infof("Creating file system %s", f.Name) - err = client.CreatePoolWithProfile(context, clusterName, *f.metadataPool, appName) - if err != nil { - return fmt.Errorf("failed to create metadata pool '%s': %+v", f.metadataPool.Name, err) - } - - var dataPoolNames []string - for _, pool := range f.dataPools { - dataPoolNames = append(dataPoolNames, pool.Name) - err = client.CreatePoolWithProfile(context, clusterName, *pool, appName) - if err != nil { - return fmt.Errorf("failed to create data pool %s: %+v", pool.Name, err) - } - if pool.Type == model.ErasureCoded { - // An erasure coded data pool used for a file system must allow overwrites - if err := client.SetPoolProperty(context, clusterName, pool.Name, "allow_ec_overwrites", "true"); err != nil { - logger.Warningf("failed to set ec pool property: %+v", err) - } - } - } - - // create the filesystem - if err := client.CreateFilesystem(context, clusterName, f.Name, f.metadataPool.Name, dataPoolNames); err != nil { - return err - } - - logger.Infof("created file system %s on %d data pool(s) and metadata pool %s", f.Name, len(f.dataPools), f.metadataPool.Name) - return nil -} - -// DownFilesystem marks the filesystem as down and the MDS' as failed -func DownFilesystem(context *clusterd.Context, clusterName, filesystemName string) error { - logger.Infof("Removing file system %s", filesystemName) - - // mark the cephFS instance as cluster_down before removing - if err := client.MarkFilesystemAsDown(context, clusterName, filesystemName); err != nil { - return err - } - - // mark each MDS associated with the file system to "failed" - fsDetails, err := client.GetFilesystem(context, clusterName, filesystemName) - if err != nil { - return err - } - for _, mdsInfo := range fsDetails.MDSMap.Info { - if err := client.FailMDS(context, clusterName, mdsInfo.GID); err != nil { - return err - } - } - - logger.Infof("Removed file system %s", filesystemName) - return nil -} - -// PrepareForDaemonUpgrade performs all actions necessary to ensure the filesystem is prepared -// to have its daemon(s) updated. This helps ensure there is no aberrant behavior during upgrades. -// If the mds is not prepared within the timeout window, an error will be reported. -// Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/ -func PrepareForDaemonUpgrade( - context *clusterd.Context, - clusterName, fsName string, - timeout time.Duration, -) error { - logger.Infof("preparing filesystem %s for daemon upgrade", fsName) - // * Beginning of noted section 1 - // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. - // See more: https://ceph.com/releases/v13-2-0-mimic-released/ - // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ - // As of Oct. 2018, this is only necessary for Luminous and Mimic. - if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil { - return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err) - } - if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil { - return err - } - // * End of Noted section 1 - - logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName) - return nil -} - -// FinishedWithDaemonUpgrade performs all actions necessary to bring the filesystem back to its -// ideal state following an upgrade of its daemon(s). -func FinishedWithDaemonUpgrade( - context *clusterd.Context, - clusterName, fsName string, - activeMDSCount int32, -) error { - logger.Debugf("restoring filesystem %s from daemon upgrade", fsName) - logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount) - // * Beginning of noted section 1 - // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. - // See more: https://ceph.com/releases/v13-2-0-mimic-released/ - // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ - // TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic. - if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil { - return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err) - } // * End of noted section 1 - return nil -} diff --git a/pkg/daemon/ceph/mds/init.go b/pkg/daemon/ceph/mds/init.go deleted file mode 100644 index 0fe5b39ce..000000000 --- a/pkg/daemon/ceph/mds/init.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package mds provides methods for setting up Ceph configuration for mds daemons. -// It also provides methods for creating/deleting/managing Ceph filesystems served by the mds. -package mds - -import ( - "fmt" - "path" - "strconv" - - "github.com/coreos/pkg/capnslog" - "github.com/rook/rook/pkg/clusterd" - cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" - "github.com/rook/rook/pkg/util" -) - -const ( - // InitCommand is the `rook ceph` subcommand which will perform mds initialization - InitCommand = "mds-init" - - keyringTemplate = ` -[mds.%s] -key = %s -caps mon = "allow profile mds" -caps osd = "allow *" -caps mds = "allow" -` -) - -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephmds") - -// Config contains the necessary parameters Rook needs to know to set up a mds for a Ceph cluster. -type Config struct { - FilesystemID string - Name string - Keyring string - ActiveStandby bool - ClusterInfo *cephconfig.ClusterInfo -} - -// Initialize generates configuration files for a Ceph mds -func Initialize(context *clusterd.Context, config *Config) error { - logger.Infof("Creating config for mds %s [filesystem ID: %s] [active standby?: %+v]", - config.Name, config.FilesystemID, config.ActiveStandby) - err := generateConfigFiles(context, config) - if err != nil { - return fmt.Errorf("failed to generate mds config files: %+v", err) - } - - util.WriteFileToLog(logger, cephconfig.DefaultConfigFilePath()) - - return err -} - -func generateConfigFiles(context *clusterd.Context, config *Config) error { - // write the latest config to the config dir - if err := cephconfig.GenerateAdminConnectionConfig(context, config.ClusterInfo); err != nil { - return fmt.Errorf("failed to write connection config: %+v", err) - } - - settings := map[string]string{ - "mds_standby_for_fscid": config.FilesystemID, - "mds_standby_replay": strconv.FormatBool(config.ActiveStandby), - } - - keyringPath := getMdsKeyringPath(context.ConfigDir, config.Name) - _, err := cephconfig.GenerateConfigFile(context, config.ClusterInfo, - getMdsConfDir(context.ConfigDir, config.Name), fmt.Sprintf("mds.%s", config.Name), - keyringPath, nil, settings) - if err != nil { - return fmt.Errorf("failed to create mds config file: %+v", err) - } - - keyringEval := func(key string) string { - r := fmt.Sprintf(keyringTemplate, config.Name, key) - return r - } - - err = cephconfig.WriteKeyring(keyringPath, config.Keyring, keyringEval) - if err != nil { - return fmt.Errorf("failed to create mds keyring: %+v", err) - } - - return nil -} - -func getMdsConfDir(dir, name string) string { - return path.Join(dir, fmt.Sprintf("mds-%s", name)) -} - -func getMdsKeyringPath(dir, name string) string { - return path.Join(getMdsConfDir(dir, name), "keyring") -} diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 2bf2a9189..b86ccf92f 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -29,6 +29,7 @@ import ( rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" @@ -52,6 +53,7 @@ var ( ) type cluster struct { + Info *cephconfig.ClusterInfo context *clusterd.Context Namespace string Spec *cephv1.ClusterSpec @@ -61,9 +63,17 @@ type cluster struct { } func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster { - return &cluster{Namespace: c.Namespace, Spec: &c.Spec, context: context, - stopCh: make(chan struct{}), - ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID))} + return &cluster{ + // at this phase of the cluster creation process, the identity components of the cluster are + // not yet established. we reserve this struct which is filled in as soon as the cluster's + // identity can be established. + Info: nil, + Namespace: c.Namespace, + Spec: &c.Spec, + context: context, + stopCh: make(chan struct{}), + ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)), + } } func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (string, error) { @@ -143,20 +153,29 @@ func (c *cluster) createInstance(rookImage string) error { } // Start the mon pods - c.mons = mon.New(c.context, c.Namespace, c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon, cephv1.GetMonPlacement(c.Spec.Placement), - c.Spec.Network.HostNetwork, cephv1.GetMonResources(c.Spec.Resources), c.ownerRef) + c.mons = mon.New(c.Info, c.context, c.Namespace, + c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon, + cephv1.GetMonPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, + cephv1.GetMonResources(c.Spec.Resources), c.ownerRef) clusterInfo, err := c.mons.Start() if err != nil { return fmt.Errorf("failed to start the mons. %+v", err) } + c.Info = clusterInfo // mons return the cluster's info + + // The cluster Identity must be established at this point + if !c.Info.IsInitialized() { + return fmt.Errorf("the cluster identity was not established: %+v", c.Info) + } err = c.createInitialCrushMap() if err != nil { return fmt.Errorf("failed to create initial crushmap: %+v", err) } - mgrs := mgr.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, cephv1.GetMgrPlacement(c.Spec.Placement), - c.Spec.Network.HostNetwork, c.Spec.Dashboard, cephv1.GetMgrResources(c.Spec.Resources), c.ownerRef, clusterInfo) + mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage, + c.Spec.CephVersion, cephv1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, + c.Spec.Dashboard, cephv1.GetMgrResources(c.Spec.Resources), c.ownerRef) err = mgrs.Start() if err != nil { return fmt.Errorf("failed to start the ceph mgr. %+v", err) diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 5fc75f419..e30203691 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -238,7 +238,7 @@ func (c *ClusterController) onAdd(obj interface{}) { objectStoreUserController.StartWatch(cluster.Namespace, cluster.stopCh) // Start file system CRD watcher - fileController := file.NewFilesystemController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) + fileController := file.NewFilesystemController(cluster.Info, c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) fileController.StartWatch(cluster.Namespace, cluster.stopCh) // Start nfs ganesha CRD watcher diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 759993656..85df1cafd 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -44,6 +44,7 @@ const ( // Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs. type Cluster struct { + clusterInfo *cephconfig.ClusterInfo Namespace string Replicas int placement rookalpha.Placement @@ -56,13 +57,22 @@ type Cluster struct { cephVersion cephv1.CephVersionSpec rookVersion string exitCode func(err error) (int, bool) - clusterInfo *cephconfig.ClusterInfo } // New creates an instance of the mgr -func New(context *clusterd.Context, namespace, rookVersion string, cephVersion cephv1.CephVersionSpec, placement rookalpha.Placement, hostNetwork bool, dashboard cephv1.DashboardSpec, - resources v1.ResourceRequirements, ownerRef metav1.OwnerReference, cInfo *cephconfig.ClusterInfo) *Cluster { +func New( + clusterInfo *cephconfig.ClusterInfo, + context *clusterd.Context, + namespace, rookVersion string, + cephVersion cephv1.CephVersionSpec, + placement rookalpha.Placement, + hostNetwork bool, + dashboard cephv1.DashboardSpec, + resources v1.ResourceRequirements, + ownerRef metav1.OwnerReference, +) *Cluster { return &Cluster{ + clusterInfo: clusterInfo, context: context, Namespace: namespace, placement: placement, @@ -75,7 +85,6 @@ func New(context *clusterd.Context, namespace, rookVersion string, cephVersion c resources: resources, ownerRef: ownerRef, exitCode: getExitCode, - clusterInfo: cInfo, } } @@ -114,7 +123,7 @@ func (c *Cluster) Start() error { return fmt.Errorf("failed to create mgr deployment %s. %+v", resourceName, err) } logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName) - if err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil { + if _, err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil { return fmt.Errorf("failed to update mgr deployment %s. %+v", resourceName, err) } } diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index 080080a75..241dc1203 100755 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -25,8 +25,8 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test" - optest "github.com/rook/rook/pkg/operator/test" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -52,7 +52,9 @@ func TestStartMGR(t *testing.T) { Executor: executor, ConfigDir: configDir, Clientset: testop.New(3)} + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} c := New( + clusterInfo, context, "ns", "myversion", @@ -62,7 +64,6 @@ func TestStartMGR(t *testing.T) { cephv1.DashboardSpec{Enabled: true}, v1.ResourceRequirements{}, metav1.OwnerReference{}, - optest.CreateConfigDir(1), ) defer os.RemoveAll(c.dataDir) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index b3d7b1881..46656b870 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -118,8 +118,7 @@ func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule // L: config-key set mgr//server_addr $(ROOK_CEPH__SERVER_ADDR) // M: config set mgr.a mgr//server_addr $(ROOK_CEPH__SERVER_ADDR) // N: config set mgr.a mgr//server_addr $(ROOK_CEPH__SERVER_ADDR) --force - envVarName := fmt.Sprintf("ROOK_CEPH_%s_SERVER_ADDR", strings.ToUpper(mgrModule)) - envVarReference := fmt.Sprintf("$(%s)", envVarName) + podIPEnvVar := "ROOK_POD_IP" cfgSetArgs := []string{"config", "set"} if c.cephVersion.Name == cephv1.Luminous || c.cephVersion.Name == "" { cfgSetArgs[0] = "config-key" @@ -127,7 +126,7 @@ func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule cfgSetArgs = append(cfgSetArgs, fmt.Sprintf("mgr.%s", mgrConfig.DaemonID)) } cfgPath := fmt.Sprintf("mgr/%s/server_addr", mgrModule) - cfgSetArgs = append(cfgSetArgs, cfgPath, envVarReference) + cfgSetArgs = append(cfgSetArgs, cfgPath, opspec.ContainerEnvVarReference(podIPEnvVar)) if cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { cfgSetArgs = append(cfgSetArgs, "--force") } @@ -149,7 +148,7 @@ func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule ), Env: append( opspec.DaemonEnvVars(), - k8sutil.PodIPEnvVar(envVarName), + k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: c.resources, } diff --git a/pkg/operator/ceph/cluster/mgr/spec_test.go b/pkg/operator/ceph/cluster/mgr/spec_test.go index c4b4de54b..0c072a538 100644 --- a/pkg/operator/ceph/cluster/mgr/spec_test.go +++ b/pkg/operator/ceph/cluster/mgr/spec_test.go @@ -19,6 +19,7 @@ package mgr import ( "testing" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/stretchr/testify/assert" @@ -33,7 +34,9 @@ import ( ) func TestPodSpec(t *testing.T) { + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} c := New( + clusterInfo, &clusterd.Context{Clientset: optest.New(1)}, "ns", "rook/rook:myversion", @@ -50,7 +53,6 @@ func TestPodSpec(t *testing.T) { }, }, metav1.OwnerReference{}, - optest.CreateConfigDir(1), ) mgrTestConfig := mgrConfig{ @@ -73,7 +75,9 @@ func TestPodSpec(t *testing.T) { } func TestServiceSpec(t *testing.T) { + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} c := New( + clusterInfo, &clusterd.Context{Clientset: optest.New(1)}, "ns", "myversion", @@ -83,7 +87,6 @@ func TestServiceSpec(t *testing.T) { cephv1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}, - optest.CreateConfigDir(1), ) s := c.makeMetricsService("rook-mgr") @@ -93,7 +96,9 @@ func TestServiceSpec(t *testing.T) { } func TestHostNetwork(t *testing.T) { + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} c := New( + clusterInfo, &clusterd.Context{Clientset: optest.New(1)}, "ns", "myversion", @@ -103,7 +108,6 @@ func TestHostNetwork(t *testing.T) { cephv1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}, - optest.CreateConfigDir(1), ) mgrTestConfig := mgrConfig{ diff --git a/pkg/operator/ceph/cluster/mon/config.go b/pkg/operator/ceph/cluster/mon/config.go index d6e6ba2e7..3a3a5419a 100644 --- a/pkg/operator/ceph/cluster/mon/config.go +++ b/pkg/operator/ceph/cluster/mon/config.go @@ -184,19 +184,19 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, var err error // store the secrets for internal usage of the rook pods - secrets := map[string]string{ - clusterSecretName: clusterInfo.Name, - fsidSecretName: clusterInfo.FSID, - monSecretName: clusterInfo.MonitorSecret, - adminSecretName: clusterInfo.AdminSecret, + secrets := map[string][]byte{ + clusterSecretName: []byte(clusterInfo.Name), + fsidSecretName: []byte(clusterInfo.FSID), + monSecretName: []byte(clusterInfo.MonitorSecret), + adminSecretName: []byte(clusterInfo.AdminSecret), } secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: appName, Namespace: namespace, }, - StringData: secrets, - Type: k8sutil.RookType, + Data: secrets, + Type: k8sutil.RookType, } k8sutil.SetOwnerRef(clientset, namespace, &secret.ObjectMeta, ownerRef) diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 1ac37cc16..022c7f9d9 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -55,9 +55,9 @@ func TestCheckHealth(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, - rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(1) + c := New(test.CreateConfigDir(1), context, "ns", "", "myversion", cephv1.CephVersionSpec{}, + cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, + v1.ResourceRequirements{}, metav1.OwnerReference{}) logger.Infof("initial mons: %v", c.clusterInfo.Monitors) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) @@ -108,9 +108,9 @@ func TestCheckHealthNotFound(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, - rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(2) + c := New(test.CreateConfigDir(2), context, "ns", "", "myversion", cephv1.CephVersionSpec{}, + cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, + v1.ResourceRequirements{}, metav1.OwnerReference{}) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) @@ -180,9 +180,9 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, - rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(2) + c := New(test.CreateConfigDir(2), context, "ns", "", "myversion", cephv1.CephVersionSpec{}, + cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, + v1.ResourceRequirements{}, metav1.OwnerReference{}) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) @@ -290,9 +290,9 @@ func TestCheckMonsValid(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, - rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(1) + c := New(test.CreateConfigDir(1), context, "ns", "", "myversion", cephv1.CephVersionSpec{}, + cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, + v1.ResourceRequirements{}, metav1.OwnerReference{}) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) @@ -378,10 +378,10 @@ func TestAddRemoveMons(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 5, AllowMultiplePerNode: true}, - rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) + c := New(test.CreateConfigDir(0), context, "ns", "", "myversion", cephv1.CephVersionSpec{}, + cephv1.MonSpec{Count: 5, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, + v1.ResourceRequirements{}, metav1.OwnerReference{}) c.maxMonID = 0 - c.clusterInfo = test.CreateConfigDir(0) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index c09765aea..4a5565538 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -77,6 +77,7 @@ const ( // Cluster represents the Rook and environment configuration settings needed to set up Ceph mons. type Cluster struct { + clusterInfo *cephconfig.ClusterInfo context *clusterd.Context Namespace string Keyring string @@ -86,7 +87,6 @@ type Cluster struct { AllowMultiplePerNode bool MonCountMutex sync.Mutex Port int32 - clusterInfo *cephconfig.ClusterInfo placement rookalpha.Placement maxMonID int waitForStart bool @@ -130,6 +130,7 @@ type NodeInfo struct { // New creates an instance of a mon cluster func New( + clusterInfo *cephconfig.ClusterInfo, context *clusterd.Context, namespace, dataDirHostPath, rookVersion string, cephVersion cephv1.CephVersionSpec, @@ -140,6 +141,7 @@ func New( ownerRef metav1.OwnerReference, ) *Cluster { return &Cluster{ + clusterInfo: clusterInfo, context: context, placement: placement, dataDirHostPath: dataDirHostPath, @@ -172,6 +174,7 @@ func (c *Cluster) Start() (*cephconfig.ClusterInfo, error) { logger.Infof("start running mons") + logger.Debugf("establishing ceph cluster info") if err := c.initClusterInfo(); err != nil { return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err) } @@ -489,7 +492,7 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error { // image or Ceph image has changed. if p.Spec.Template.Spec.Containers[0].Image != d.Spec.Template.Spec.Containers[0].Image || p.Spec.Template.Spec.InitContainers[0].Image != d.Spec.Template.Spec.InitContainers[0].Image { - if err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil { + if _, err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil { return fmt.Errorf("failed to update mon deployment %s. %+v", m.ResourceName, err) } } diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index e68a3f355..086f5c851 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -102,6 +102,7 @@ func newTestStartClusterWithQuorumResponse(namespace string, monResponse func() func newCluster(context *clusterd.Context, namespace string, hostNetwork bool, allowMultiplePerNode bool, resources v1.ResourceRequirements) *Cluster { return &Cluster{ + clusterInfo: nil, HostNetwork: hostNetwork, context: context, Namespace: namespace, @@ -134,16 +135,14 @@ func TestStartMonPods(t *testing.T) { c := newCluster(context, namespace, false, true, v1.ResourceRequirements{}) // start a basic cluster - i, err := c.Start() + _, err := c.Start() assert.Nil(t, err) - assert.Equal(t, i, c.clusterInfo) validateStart(t, c) // starting again should be a no-op, but still results in an error - i, err = c.Start() + _, err = c.Start() assert.Nil(t, err) - assert.Equal(t, i, c.clusterInfo) validateStart(t, c) } @@ -155,18 +154,18 @@ func TestOperatorRestart(t *testing.T) { c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - i, err := c.Start() + info, err := c.Start() assert.Nil(t, err) - assert.Equal(t, i, c.clusterInfo) + assert.True(t, info.IsInitialized()) validateStart(t, c) c = newCluster(context, namespace, false, true, v1.ResourceRequirements{}) // starting again should be a no-op, but will not result in an error - i, err = c.Start() + info, err = c.Start() assert.Nil(t, err) - assert.Equal(t, i, c.clusterInfo) + assert.True(t, info.IsInitialized()) validateStart(t, c) } @@ -181,9 +180,9 @@ func TestOperatorRestartHostNetwork(t *testing.T) { c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - i, err := c.Start() + info, err := c.Start() assert.Nil(t, err) - assert.Equal(t, i, c.clusterInfo) + assert.True(t, info.IsInitialized()) validateStart(t, c) @@ -191,9 +190,9 @@ func TestOperatorRestartHostNetwork(t *testing.T) { c = newCluster(context, namespace, true, false, v1.ResourceRequirements{}) // starting again should be a no-op, but still results in an error - i, err = c.Start() + info, err = c.Start() assert.Nil(t, err) - assert.Equal(t, i, c.clusterInfo) + assert.True(t, info.IsInitialized(), info) validateStart(t, c) } @@ -201,7 +200,7 @@ func TestOperatorRestartHostNetwork(t *testing.T) { func validateStart(t *testing.T, c *Cluster) { s, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{}) assert.Nil(t, err) // there shouldn't be an error due the secret existing - assert.Equal(t, 4, len(s.StringData)) + assert.Equal(t, 4, len(s.Data)) // there is only one pod created. the other two won't be created since the first one doesn't start _, err = c.context.Clientset.Apps().Deployments(c.Namespace).Get("rook-ceph-mon-a", metav1.GetOptions{}) @@ -212,10 +211,10 @@ func TestSaveMonEndpoints(t *testing.T) { clientset := test.New(1) configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", "", "myversion", cephv1.CephVersionSpec{}, + c := New(test.CreateConfigDir(1), &clusterd.Context{Clientset: clientset, ConfigDir: configDir}, + "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(1) // create the initial config map err := c.saveMonConfig() diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index 948235c48..5e2100a25 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -30,10 +30,10 @@ import ( func TestAvailableMonNodes(t *testing.T) { clientset := test.New(1) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{}, - cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, - false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(0) + c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) + nodes, err := c.getMonNodes() assert.Nil(t, err) assert.Equal(t, 1, len(nodes)) @@ -57,10 +57,9 @@ func TestAvailableMonNodes(t *testing.T) { func TestAvailableNodesInUse(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{}, - cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, - false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(0) + c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // all three nodes are available by default nodes, err := c.getMonNodes() @@ -99,10 +98,9 @@ func TestAvailableNodesInUse(t *testing.T) { func TestTaintedNodes(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{}, - cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, - false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(0) + c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) nodes, err := c.getMonNodes() assert.Nil(t, err) @@ -134,10 +132,9 @@ func TestTaintedNodes(t *testing.T) { func TestNodeAffinity(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{}, - cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, - false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(0) + c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) nodes, err := c.getMonNodes() assert.Nil(t, err) @@ -190,10 +187,9 @@ func TestHostNetworkSameNode(t *testing.T) { func TestHostNetwork(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{}, - cephv1.MonSpec{Count: 3, AllowMultiplePerNode: false}, rookalpha.Placement{}, - false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(0) + c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: false}, + rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.HostNetwork = true @@ -222,11 +218,6 @@ func TestGetNodeInfoFromNode(t *testing.T) { }, } - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{}, - cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, - true, v1.ResourceRequirements{}, metav1.OwnerReference{}) - c.clusterInfo = test.CreateConfigDir(0) - var info *NodeInfo info, err = getNodeInfoFromNode(*node) assert.Nil(t, err) diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 87accd11f..3264aa4e0 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -152,6 +152,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container { } func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { + podIPEnvVar := "ROOK_POD_IP" container := v1.Container{ Name: "mon", Command: []string{ @@ -161,7 +162,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { opspec.DaemonFlags(c.clusterInfo, config.MonType, monConfig.DaemonName), "--foreground", config.NewFlag("public-addr", monConfig.PublicIP), - config.NewFlag("public-bind-addr", "$(ROOK_PRIVATE_IP)"), + config.NewFlag("public-bind-addr", opspec.ContainerEnvVarReference(podIPEnvVar)), ), Image: c.cephVersion.Image, VolumeMounts: opspec.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName), @@ -175,7 +176,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { }, Env: append( opspec.DaemonEnvVars(), - k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), + k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: c.resources, } diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index 5f23e8d44..7a51919a4 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -22,13 +22,11 @@ import ( "strings" "testing" - "github.com/rook/rook/pkg/operator/ceph/config" - - "github.com/rook/rook/pkg/operator/ceph/spec" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/spec" test_opceph "github.com/rook/rook/pkg/operator/ceph/test" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" @@ -45,6 +43,7 @@ func TestPodSpecs(t *testing.T) { func testPodSpec(t *testing.T, monID string) { clientset := testop.New(1) c := New( + testop.CreateConfigDir(0), &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"}, "ns", "/var/lib/rook", @@ -63,7 +62,6 @@ func testPodSpec(t *testing.T, monID string) { }, metav1.OwnerReference{}, ) - c.clusterInfo = testop.CreateConfigDir(0) monConfig := testGenMonConfig(monID) pod := c.makeMonPod(monConfig, "foo") @@ -129,7 +127,7 @@ func testPodSpec(t *testing.T, monID string) { Args: append(commonFlags, []string{"--foreground"}, []string{"--public-addr=2.4.6.1"}, - []string{"--public-bind-addr=$(ROOK_PRIVATE_IP)"}), + []string{"--public-bind-addr=$(ROOK_POD_IP)"}), VolumeMountNames: cephVolumeMountNames, EnvCount: &monDaemonEnvs, Ports: []v1.ContainerPort{ diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index e461c913a..e67f7bb25 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -320,7 +320,7 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig continue } logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID) - if err = k8sutil.UpdateDeploymentAndWait(c.context, dp, c.Namespace); err != nil { + if _, err = k8sutil.UpdateDeploymentAndWait(c.context, dp, c.Namespace); err != nil { config.addError(fmt.Sprintf("failed to update osd deployment %d. %+v", osd.ID, err)) } } diff --git a/pkg/operator/ceph/config/defaults.go b/pkg/operator/ceph/config/defaults.go index 7f4ebb6b8..09a644e00 100644 --- a/pkg/operator/ceph/config/defaults.go +++ b/pkg/operator/ceph/config/defaults.go @@ -17,10 +17,35 @@ limitations under the License. // Package config provides default configurations which Rook will set in Ceph clusters. package config +import ( + "bytes" + "fmt" +) + // DefaultFlags returns the default configuration flags Rook will set on the command line for all // calls to Ceph daemons and tools. Values specified here will not be able to be overridden using // the mon's central KV store, and that is (and should be) by intent. func DefaultFlags(fsid, mountedKeyringPath string) []string { + return defaultFlagConfigs(fsid, mountedKeyringPath).GlobalFlags() +} + +// DefaultFlagsAsConfigFile is the same as DefaultFlags except that it returns the ceph.conf file +// text which corresponds to the flags. In some versions of Ceph and for some Daemons, the default +// flags are not accepted in flag form. +func DefaultFlagsAsConfigFile(fsid, mountedKeyringPath string) (string, error) { + c := defaultFlagConfigs(fsid, mountedKeyringPath) + f, err := c.IniFile() + if err != nil { + return "", fmt.Errorf("failed to generate text for default flags compatible with ceph.conf. %v", err) + } + b := new(bytes.Buffer) + if _, err := f.WriteTo(b); err != nil { + return "", fmt.Errorf("failed to generate text for default flags compatible with ceph.conf. %v", err) + } + return b.String(), nil +} + +func defaultFlagConfigs(fsid, mountedKeyringPath string) *Config { c := NewConfig() c.Section("global"). // fsid unnecessary but is a safety to make sure daemons can only connect to their cluster @@ -32,10 +57,9 @@ func DefaultFlags(fsid, mountedKeyringPath string) []string { Set("mon-cluster-log-to-stderr", "true"). Set("log-stderr-prefix", "debug ") // ^ differentiate debug text from audit text, and the space after 'debug' is critical - return append( - c.GlobalFlags(), - StoredMonHostEnvVarFlags()..., - ) + m := StoredMonHostEnvVarReferences() + c.Merge(m) + return c } // DefaultCentralizedConfigs returns the default configuration options Rook will set in Ceph's @@ -68,7 +92,10 @@ func DefaultLegacyConfigs() *Config { Set("osd pool default pgp num", "100"). // Set("rbd_default_features", "3"). // TODO: still needed? - // - Set("fatal signal handlers", "false") // TODO: still needed? + // Setting fatal signal handlers to true (the default) will print a lot of extra information + // from daemons when they encounter a failures, but it is VERY verbose. When the mon kv + // store is available, it will probably be best to set this to false by default if it is + // unset, but leave it set to true if the user has specified it as true during runtime. + Set("fatal signal handlers", "false") return c } diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index 39cde4add..d78a63462 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -256,10 +256,13 @@ func StoredMonHostEnvVars() []v1.EnvVar { } } -// StoredMonHostEnvVarFlags returns a flag compatible with Ceph daemons and tools to read "mon_host" -// and "mon_initial_members" information from the StoredMonHostEnvVar. -func StoredMonHostEnvVarFlags() []string { - return []string{ - NewFlag(monHostKey, "$(ROOK_CEPH_MON_HOST)"), - NewFlag(monInitialMembersKey, "$(ROOK_CEPH_MON_INITIAL_MEMBERS)")} +// StoredMonHostEnvVarReferences returns a small Ceph Config which references "mon_host" and +// "mon_initial_members" information from the StoredMonHostEnvVars. This config can be used to +// generate flags referencing the env vars or to generate the string representation of a Config. +func StoredMonHostEnvVarReferences() *Config { + c := NewConfig() + c.Section("global"). + Set(monHostKey, "$(ROOK_CEPH_MON_HOST)"). + Set(monInitialMembersKey, "$(ROOK_CEPH_MON_INITIAL_MEMBERS)") + return c } diff --git a/pkg/operator/ceph/config/store_test.go b/pkg/operator/ceph/config/store_test.go index 72f8ec287..aaba878db 100644 --- a/pkg/operator/ceph/config/store_test.go +++ b/pkg/operator/ceph/config/store_test.go @@ -204,7 +204,7 @@ func TestEnvVarsAndFlags(t *testing.T) { s.CreateOrUpdate(testop.CreateConfigDir(3)) v := StoredMonHostEnvVars() - f := StoredMonHostEnvVarFlags() + f := StoredMonHostEnvVarReferences().GlobalFlags() // make sure the env var names and flags are matching pairs mh := v[0].Name diff --git a/pkg/operator/ceph/file/config.go b/pkg/operator/ceph/file/config.go new file mode 100644 index 000000000..aa7263de7 --- /dev/null +++ b/pkg/operator/ceph/file/config.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package file + +import ( + "fmt" + + "github.com/rook/rook/pkg/operator/ceph/config/keyring" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + keyringTemplate = ` +[mds.%s] +key = %s +caps mon = "allow profile mds" +caps osd = "allow *" +caps mds = "allow" +` +) + +func (c *cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error { + user := fmt.Sprintf("mds.%s", m.DaemonID) + access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"} + ownerRef := &metav1.OwnerReference{ + UID: deploymentUID, + APIVersion: "v1", + Kind: "deployment", + Name: m.ResourceName, + } + s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef) + + key, err := s.GenerateKey(m.ResourceName, user, access) + if err != nil { + return err + } + + // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x + err = c.context.Clientset.CoreV1().Secrets(c.fs.Namespace).Delete(m.ResourceName, &metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + logger.Debugf("legacy mgr key %s is already removed", m.ResourceName) + } else { + logger.Warningf("legacy mgr key %s could not be removed: %+v", m.ResourceName, err) + } + } + + keyring := fmt.Sprintf(keyringTemplate, m.DaemonID, key) + return s.CreateOrUpdate(m.ResourceName, keyring) +} diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index d70cd01c3..6838d4f27 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -21,13 +21,13 @@ import ( "fmt" "reflect" - "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/ceph/pool" - "github.com/coreos/pkg/capnslog" opkit "github.com/rook/operator-kit" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephbeta "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" + "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/pool" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,7 +36,7 @@ import ( var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-file") -// FilesystemResource represents the file system custom resource +// FilesystemResource represents the filesystem custom resource var FilesystemResource = opkit.CustomResource{ Name: "cephfilesystem", Plural: "cephfilesystems", @@ -55,8 +55,9 @@ var filesystemResourceRookLegacy = opkit.CustomResource{ Kind: reflect.TypeOf(cephbeta.Filesystem{}).Name(), } -// FilesystemController represents a controller for file system custom resources +// FilesystemController represents a controller for filesystem custom resources type FilesystemController struct { + clusterInfo *cephconfig.ClusterInfo context *clusterd.Context rookVersion string cephVersion cephv1.CephVersionSpec @@ -64,8 +65,9 @@ type FilesystemController struct { ownerRef metav1.OwnerReference } -// NewFilesystemController create controller for watching file system custom resources created +// NewFilesystemController create controller for watching filesystem custom resources created func NewFilesystemController( + clusterInfo *cephconfig.ClusterInfo, context *clusterd.Context, rookVersion string, cephVersion cephv1.CephVersionSpec, @@ -73,6 +75,7 @@ func NewFilesystemController( ownerRef metav1.OwnerReference, ) *FilesystemController { return &FilesystemController{ + clusterInfo: clusterInfo, context: context, rookVersion: rookVersion, cephVersion: cephVersion, @@ -114,9 +117,9 @@ func (c *FilesystemController) onAdd(obj interface{}) { return } - err = createFilesystem(c.context, *filesystem, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(filesystem)) + err = createFilesystem(c.clusterInfo, c.context, *filesystem, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(filesystem)) if err != nil { - logger.Errorf("failed to create file system %s: %+v", filesystem.Name, err) + logger.Errorf("failed to create filesystem %s: %+v", filesystem.Name, err) } } @@ -144,11 +147,11 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) { return } - // if the file system is modified, allow the file system to be created if it wasn't already + // if the filesystem is modified, allow the filesystem to be created if it wasn't already logger.Infof("updating filesystem %s", newFS.Name) - err = createFilesystem(c.context, *newFS, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(newFS)) + err = createFilesystem(c.clusterInfo, c.context, *newFS, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(newFS)) if err != nil { - logger.Errorf("failed to create (modify) file system %s: %+v", newFS.Name, err) + logger.Errorf("failed to create (modify) filesystem %s: %+v", newFS.Name, err) } } @@ -166,7 +169,7 @@ func (c *FilesystemController) onDelete(obj interface{}) { err = deleteFilesystem(c.context, *filesystem) if err != nil { - logger.Errorf("failed to delete file system %s: %+v", filesystem.Name, err) + logger.Errorf("failed to delete filesystem %s: %+v", filesystem.Name, err) } } diff --git a/pkg/operator/ceph/file/controller_test.go b/pkg/operator/ceph/file/controller_test.go index 70dca6379..96de56478 100644 --- a/pkg/operator/ceph/file/controller_test.go +++ b/pkg/operator/ceph/file/controller_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package mds to manage a rook file system. +// Package mds to manage a rook filesystem. package file import ( @@ -25,6 +25,8 @@ import ( rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" rookfake "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" @@ -83,7 +85,9 @@ func TestMigrateFilesystemObject(t *testing.T) { Clientset: clientset, RookClientset: rookfake.NewSimpleClientset(legacyFilesystem), } - controller := NewFilesystemController(context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{}) + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} + + controller := NewFilesystemController(clusterInfo, context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{}) // convert the legacy filesystem object in memory and assert that a migration is needed convertedFilesystem, migrationNeeded, err := getFilesystemObject(legacyFilesystem) diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index fee00936f..d9aa76bf7 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -18,18 +18,34 @@ package file import ( "fmt" + "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" - mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/daemon/ceph/model" "github.com/rook/rook/pkg/operator/ceph/pool" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + dataPoolSuffix = "data" + metadataPoolSuffix = "metadata" + appName = "cephfs" +) + +// Filesystem represents an instance of a Ceph filesystem (CephFS) +type Filesystem struct { + Name string + metadataPool *model.Pool + dataPools []*model.Pool + activeMDSCount int32 +} + // createFilesystem creates a Ceph filesystem with metadata servers func createFilesystem( + clusterInfo *cephconfig.ClusterInfo, context *clusterd.Context, fs cephv1.CephFilesystem, rookVersion string, @@ -46,15 +62,15 @@ func createFilesystem( for _, p := range fs.Spec.DataPools { dataPools = append(dataPools, p.ToModel("")) } - f := mdsdaemon.NewFS(fs.Name, fs.Spec.MetadataPool.ToModel(""), dataPools, fs.Spec.MetadataServer.ActiveCount) - if err := f.CreateFilesystem(context, fs.Namespace); err != nil { - return fmt.Errorf("failed to create file system %s: %+v", fs.Name, err) + f := newFS(fs.Name, fs.Spec.MetadataPool.ToModel(""), dataPools, fs.Spec.MetadataServer.ActiveCount) + if err := f.doFilesystemCreate(context, fs.Namespace); err != nil { + return fmt.Errorf("failed to create filesystem %s: %+v", fs.Name, err) } } filesystem, err := client.GetFilesystem(context, fs.Namespace, fs.Name) if err != nil { - return fmt.Errorf("failed to get file system %s: %+v", fs.Name, err) + return fmt.Errorf("failed to get filesystem %s: %+v", fs.Name, err) } // set the number of active mds instances @@ -64,8 +80,8 @@ func createFilesystem( } } - logger.Infof("start running mdses for file system %s", fs.Name) - c := newCluster(context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs) + logger.Infof("start running mdses for filesystem %s", fs.Name) + c := newCluster(clusterInfo, context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs) if err := c.start(); err != nil { return err } @@ -73,15 +89,15 @@ func createFilesystem( return nil } -// deleteFileSystem deletes the file system and the metadata servers +// deleteFileSystem deletes the filesystem and the metadata servers func deleteFilesystem(context *clusterd.Context, fs cephv1.CephFilesystem) error { // The most important part of deletion is that the filesystem gets removed from Ceph - if err := mdsdaemon.DownFilesystem(context, fs.Namespace, fs.Name); err != nil { + if err := downFilesystem(context, fs.Namespace, fs.Name); err != nil { // If the fs isn't deleted from Ceph, leave the daemons so it can still be used. return fmt.Errorf("failed to down filesystem %s: %+v", fs.Name, err) } - // Permanently remove the file system if it was created by rook + // Permanently remove the filesystem if it was created by rook if len(fs.Spec.DataPools) != 0 { if err := client.RemoveFilesystem(context, fs.Namespace, fs.Name); err != nil { return fmt.Errorf("failed to remove filesystem %s: %+v", fs.Name, err) @@ -116,3 +132,148 @@ func validateFilesystem(context *clusterd.Context, f cephv1.CephFilesystem) erro return nil } + +// newFS creates a new instance of the file (MDS) service +func newFS(name string, metadataPool *model.Pool, dataPools []*model.Pool, activeMDSCount int32) *Filesystem { + + metadataPool.Name = fmt.Sprintf("%s-%s", name, metadataPoolSuffix) + for i, pool := range dataPools { + pool.Name = fmt.Sprintf("%s-%s%d", name, dataPoolSuffix, i) + } + + return &Filesystem{ + Name: name, + metadataPool: metadataPool, + dataPools: dataPools, + activeMDSCount: activeMDSCount, + } +} + +// doFilesystemCreate starts the Ceph file daemons and creates the filesystem in Ceph. +func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, clusterName string) error { + _, err := client.GetFilesystem(context, clusterName, f.Name) + if err == nil { + logger.Infof("filesystem %s already exists", f.Name) + // Even if the fs already exists, the num active mdses may have changed + if err := client.SetNumMDSRanks(context, clusterName, f.Name, f.activeMDSCount); err != nil { + logger.Errorf( + fmt.Sprintf("failed to set num mds ranks (max_mds) to %d for filesystem %s, still continuing. ", f.activeMDSCount, f.Name) + + "this error is not critical, but mdses may not be as failure tolerant as desired. " + + fmt.Sprintf("USER should verify that the number of active mdses is %d with 'ceph fs get %s'", f.activeMDSCount, f.Name) + + fmt.Sprintf(": %+v", err), + ) + } + return nil + } + if len(f.dataPools) == 0 { + return fmt.Errorf("at least one data pool must be specified") + } + + fslist, err := client.ListFilesystems(context, clusterName) + if err != nil { + return fmt.Errorf("Unable to list existing filesystem: %+v", err) + } + if len(fslist) > 0 && !client.IsMultiFSEnabled() { + return fmt.Errorf("Cannot create multiple filesystems. Enable %s env variable to create more than one", client.MultiFsEnv) + } + + logger.Infof("Creating filesystem %s", f.Name) + err = client.CreatePoolWithProfile(context, clusterName, *f.metadataPool, appName) + if err != nil { + return fmt.Errorf("failed to create metadata pool '%s': %+v", f.metadataPool.Name, err) + } + + var dataPoolNames []string + for _, pool := range f.dataPools { + dataPoolNames = append(dataPoolNames, pool.Name) + err = client.CreatePoolWithProfile(context, clusterName, *pool, appName) + if err != nil { + return fmt.Errorf("failed to create data pool %s: %+v", pool.Name, err) + } + if pool.Type == model.ErasureCoded { + // An erasure coded data pool used for a filesystem must allow overwrites + if err := client.SetPoolProperty(context, clusterName, pool.Name, "allow_ec_overwrites", "true"); err != nil { + logger.Warningf("failed to set ec pool property: %+v", err) + } + } + } + + // create the filesystem + if err := client.CreateFilesystem(context, clusterName, f.Name, f.metadataPool.Name, dataPoolNames); err != nil { + return err + } + + logger.Infof("created filesystem%s on %d data pool(s) and metadata pool %s", f.Name, len(f.dataPools), f.metadataPool.Name) + return nil +} + +// downFilesystem marks the filesystem as down and the MDS' as failed +func downFilesystem(context *clusterd.Context, clusterName, filesystemName string) error { + logger.Infof("Removing filesystem %s", filesystemName) + + // mark the cephFS instance as cluster_down before removing + if err := client.MarkFilesystemAsDown(context, clusterName, filesystemName); err != nil { + return err + } + + // mark each MDS associated with the filesystem to "failed" + fsDetails, err := client.GetFilesystem(context, clusterName, filesystemName) + if err != nil { + return err + } + for _, mdsInfo := range fsDetails.MDSMap.Info { + if err := client.FailMDS(context, clusterName, mdsInfo.GID); err != nil { + return err + } + } + + logger.Infof("Removed filesystem %s", filesystemName) + return nil +} + +// prepareForDaemonUpgrade performs all actions necessary to ensure the filesystem is prepared +// to have its daemon(s) updated. This helps ensure there is no aberrant behavior during upgrades. +// If the mds is not prepared within the timeout window, an error will be reported. +// Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/ +func prepareForDaemonUpgrade( + context *clusterd.Context, + clusterName, fsName string, + timeout time.Duration, +) error { + logger.Infof("preparing filesystem %s for daemon upgrade", fsName) + // * Beginning of noted section 1 + // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. + // See more: https://ceph.com/releases/v13-2-0-mimic-released/ + // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ + // As of Oct. 2018, this is only necessary for Luminous and Mimic. + if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil { + return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err) + } + if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil { + return err + } + // * End of Noted section 1 + + logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName) + return nil +} + +// finishedWithDaemonUpgrade performs all actions necessary to bring the filesystem back to its +// ideal state following an upgrade of its daemon(s). +func finishedWithDaemonUpgrade( + context *clusterd.Context, + clusterName, fsName string, + activeMDSCount int32, +) error { + logger.Debugf("restoring filesystem %s from daemon upgrade", fsName) + logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount) + // * Beginning of noted section 1 + // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. + // See more: https://ceph.com/releases/v13-2-0-mimic-released/ + // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ + // TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic. + if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil { + return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err) + } // * End of noted section 1 + return nil +} diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index 2d52d1476..89ca1cb0b 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -26,6 +26,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" cephtest "github.com/rook/rook/pkg/daemon/ceph/test" testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test" testop "github.com/rook/rook/pkg/operator/test" @@ -71,7 +72,7 @@ func TestCreateFilesystem(t *testing.T) { updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub() configDir, _ := ioutil.TempDir("", "") - // Output to check multiple file system creation + // Output to check multiple filesystem creation fses := `[{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":1,"data_pool_ids":[2],"data_pools":["myfs-data0"]}]` executor := &exectest.MockExecutor{ @@ -109,16 +110,17 @@ func TestCreateFilesystem(t *testing.T) { }, }, } + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} // start a basic cluster - err := createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) + err := createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, context, fs) assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) // starting again should be a no-op - err = createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) + err = createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, context, fs) assert.ElementsMatch(t, []string{"rook-ceph-mds-myfs-a", "rook-ceph-mds-myfs-b"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) @@ -139,13 +141,13 @@ func TestCreateFilesystem(t *testing.T) { Clientset: testop.New(3)} //Create another filesystem which should fail - err = createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) - assert.Equal(t, "failed to create file system myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error()) + err = createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) + assert.Equal(t, "failed to create filesystem myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error()) } func TestCreateNopoolFilesystem(t *testing.T) { configDir, _ := ioutil.TempDir("", "") - // Output to check multiple file system creation + // Output to check multiple filesystem creation fses := `[{"name":"myfs"}]` executor := &exectest.MockExecutor{ @@ -173,14 +175,15 @@ func TestCreateNopoolFilesystem(t *testing.T) { }, }, } + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} // start a basic cluster - err := createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) + err := createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, context, fs) // starting again should be a no-op - err = createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) + err = createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, context, fs) diff --git a/pkg/operator/ceph/file/mds.go b/pkg/operator/ceph/file/mds.go index 15c59d889..647d64f94 100644 --- a/pkg/operator/ceph/file/mds.go +++ b/pkg/operator/ceph/file/mds.go @@ -24,10 +24,10 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" - mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -43,6 +43,7 @@ const ( ) type cluster struct { + clusterInfo *cephconfig.ClusterInfo context *clusterd.Context rookVersion string cephVersion cephv1.CephVersionSpec @@ -52,7 +53,14 @@ type cluster struct { ownerRefs []metav1.OwnerReference } +type mdsConfig struct { + ResourceName string + DaemonID string + DataPathMap *config.DataPathMap // location to store data in container +} + func newCluster( + clusterInfo *cephconfig.ClusterInfo, context *clusterd.Context, rookVersion string, cephVersion cephv1.CephVersionSpec, @@ -62,6 +70,7 @@ func newCluster( ownerRefs []metav1.OwnerReference, ) *cluster { return &cluster{ + clusterInfo: clusterInfo, context: context, rookVersion: rookVersion, cephVersion: cephVersion, @@ -72,31 +81,6 @@ func newCluster( } } -type mdsConfig struct { - ResourceName string - DaemonName string -} - -// return true if an attempt was made to prepare the filesystem associated for upgrade -func (c *cluster) deleteLegacyMdsDeployment() bool { - legacyName := fmt.Sprintf("%s-%s", AppName, c.fs.Name) // rook-ceph-mds- - logger.Debugf("getting legacy mds deployment %s for filesystem %s if it exists", legacyName, c.fs.Name) - _, err := c.context.Clientset.Apps().Deployments(c.fs.Namespace).Get(legacyName, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - logger.Infof("legacy mds deployment %s not found, no update needed", legacyName) - return false - // if any other error, failed to get legacy dep., but it exists, so still try deleting it. - } - logger.Infof("deleting legacy mds deployment %s", legacyName) - if err := mdsdaemon.PrepareForDaemonUpgrade(c.context, c.fs.Namespace, c.fs.Name, fsWaitForActiveTimeout); err != nil { - logger.Errorf("failed to prepare filesystem %s for mds upgrade while deleting legacy deployment %s, continuing anyway: %+v", c.fs.Name, legacyName, err) - } - if err := k8sutil.DeleteDeployment(c.context.Clientset, c.fs.Namespace, legacyName); err != nil { - logger.Errorf("failed to delete legacy deployment %s, USER should delete the legacy mds deployment manually if it still exists: %+v", legacyName, err) - } - return true -} - var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait func (c *cluster) start() error { @@ -105,15 +89,12 @@ func (c *cluster) start() error { var fsPreparedForUpgrade = false defer func() { if fsPreparedForUpgrade { - if err := mdsdaemon.FinishedWithDaemonUpgrade(c.context, c.fs.Namespace, c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount); err != nil { + if err := finishedWithDaemonUpgrade(c.context, c.fs.Namespace, c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount); err != nil { logger.Errorf("for filesystem %s, USER should make sure the Ceph fs max_mds property is set to %d: %+v", c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount, err) } } }() - // Delete legacy daemonset, and remove it if it exists. - // This is relevant to the upgrade from Rook 0.8 to 0.9. - fsPreparedForUpgrade = c.deleteLegacyMdsDeployment() // Always create double the number of metadata servers to have standby mdses available replicas := c.fs.Spec.MetadataServer.ActiveCount * 2 @@ -129,18 +110,14 @@ func (c *cluster) start() error { mdsConfig := &mdsConfig{ ResourceName: resourceName, - DaemonName: daemonName, - } - - // create unique key for each mds - if err := c.getOrCreateKeyring(mdsConfig); err != nil { - return fmt.Errorf("failed to create mds keyring for filesystem %s: %+v", c.fs.Name, err) + DaemonID: daemonName, + DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, daemonName), } // start the deployment d := c.makeDeployment(mdsConfig) logger.Debugf("starting mds: %+v", d) - _, err := c.context.Clientset.Apps().Deployments(c.fs.Namespace).Create(d) + createdDeployment, err := c.context.Clientset.Apps().Deployments(c.fs.Namespace).Create(d) if err != nil { if !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create mds deployment %s: %+v", mdsConfig.ResourceName, err) @@ -149,11 +126,16 @@ func (c *cluster) start() error { // TODO: need to prepare for upgrade here each time. Also, before a given deployment is // terminated, I think we should somehow make sure that it isn't running the single // active daemon. If it is, then we should have another daemon take over as active. @Jan? - if err := updateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil { + if createdDeployment, err = updateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil { return fmt.Errorf("failed to update mds deployment %s. %+v", mdsConfig.ResourceName, err) } } desiredDeployments[d.GetName()] = true // add deployment name to improvised set + + // create unique key for each mds saved to k8s secret + if err := c.generateKeyring(mdsConfig, createdDeployment.UID); err != nil { + return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err) + } } // Remove extraneous mds deployments if they exist @@ -204,62 +186,6 @@ func (c *cluster) start() error { return nil } -func (c *cluster) getOrCreateKeyring(mdsConfig *mdsConfig) error { - _, err := c.context.Clientset.CoreV1().Secrets(c.fs.Namespace).Get( - mdsConfig.ResourceName, metav1.GetOptions{}) - if err == nil { - logger.Infof("the mds keyring was already generated") - return nil - } - if !errors.IsNotFound(err) { - return fmt.Errorf("failed to get mds secrets: %+v", err) - } - - // get or create key for the mds user - key, err := getOrCreateKey(c.context, c.fs.Namespace, mdsConfig.DaemonName) - if err != nil { - return err // there isn't any additional useful information to add here - } - - // Store the keyring in a secret - secrets := map[string]string{ - keyringSecretKeyName: key, - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: mdsConfig.ResourceName, - Namespace: c.fs.Namespace, - }, - StringData: secrets, - Type: k8sutil.RookType, - } - k8sutil.SetOwnerRefs(c.context.Clientset, c.fs.Namespace, &secret.ObjectMeta, c.ownerRefs) - - _, err = c.context.Clientset.CoreV1().Secrets(c.fs.Namespace).Create(secret) - if err != nil { - return fmt.Errorf("failed to save mds secret: %+v", err) - } - - return nil -} - -// create a keyring for the mds cluster with a limited set of privileges -func getOrCreateKey(context *clusterd.Context, clusterName, daemonName string) (string, error) { - access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"} - username := fmt.Sprintf("mds.%s", daemonName) - // "mds."" without a letter ID creates a key that can work for all mdses. This also means that - // it *could* work for mdses serving a different filesystem, though we want to have a unique key - // for each filesystem so that access can be quickly revoked by deleting the secret. - - // get-or-create-key for the user account - key, err := client.AuthGetOrCreateKey(context, clusterName, username, access) - if err != nil { - return "", fmt.Errorf("failed to get or create auth key for user %s: %+v", username, err) - } - - return key, nil -} - func deleteMdsCluster(context *clusterd.Context, namespace, fsName string) error { // Try to delete all mds deployments and secret keys serving the filesystem, and aggregate // failures together to report all at once at the end. diff --git a/pkg/operator/ceph/file/spec.go b/pkg/operator/ceph/file/spec.go index 611d78576..1f6e120c2 100644 --- a/pkg/operator/ceph/file/spec.go +++ b/pkg/operator/ceph/file/spec.go @@ -19,8 +19,7 @@ package file import ( "strconv" - mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds" - opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" @@ -40,14 +39,12 @@ func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { Annotations: map[string]string{}, }, Spec: v1.PodSpec{ - InitContainers: []v1.Container{ - c.makeConfigInitContainer(mdsConfig), - }, + InitContainers: []v1.Container{}, Containers: []v1.Container{ c.makeMdsDaemonContainer(mdsConfig), }, RestartPolicy: v1.RestartPolicyAlways, - Volumes: opspec.PodVolumes(""), + Volumes: opspec.DaemonVolumes(mdsConfig.DataPathMap, mdsConfig.ResourceName), HostNetwork: c.HostNetwork, }, } @@ -61,6 +58,7 @@ func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { ObjectMeta: metav1.ObjectMeta{ Name: mdsConfig.ResourceName, Namespace: c.fs.Namespace, + Labels: c.podLabels(mdsConfig), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -77,60 +75,29 @@ func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { return d } -func (c *cluster) makeConfigInitContainer(mdsConfig *mdsConfig) v1.Container { - return v1.Container{ - Name: opspec.ConfigInitContainerName, - Args: []string{ - "ceph", - mdsdaemon.InitCommand, - "--config-dir", k8sutil.DataDir, - "--mds-name", mdsConfig.DaemonName, - "--filesystem-id", c.fsID, - "--active-standby", strconv.FormatBool(c.fs.Spec.MetadataServer.ActiveStandby), - }, - Image: k8sutil.MakeRookImage(c.rookVersion), - Env: []v1.EnvVar{ - // Set '--mds-keyring' flag with an env var sourced from the secret - {Name: "ROOK_MDS_KEYRING", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{Name: mdsConfig.ResourceName}, - Key: keyringSecretKeyName, - }}}, - k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), - k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), - opmon.ClusterNameEnvVar(c.fs.Namespace), - opmon.EndpointEnvVar(), - opmon.SecretEnvVar(), - opmon.AdminSecretEnvVar(), - k8sutil.ConfigOverrideEnvVar(), - }, - VolumeMounts: opspec.RookVolumeMounts(), - Resources: c.fs.Spec.MetadataServer.Resources, - } -} - func (c *cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { return v1.Container{ - Name: "mgr", + Name: "mds", Command: []string{ - mdsDaemonCommand, + "ceph-mds", }, - Args: []string{ + Args: append( + opspec.DaemonFlags(c.clusterInfo, config.MdsType, mdsConfig.DaemonID), "--foreground", - "--id", mdsConfig.DaemonName, - // do not add the '--cluster/--conf/--keyring' flags; rook wants their default values - }, + config.NewFlag("mds-standby-for-fscid", c.fsID), + config.NewFlag("mds-standby-replay", strconv.FormatBool(c.fs.Spec.MetadataServer.ActiveStandby)), + ), Image: c.cephVersion.Image, - Env: k8sutil.ClusterDaemonEnvVars(), - VolumeMounts: opspec.CephVolumeMounts(), - // TODO: mds doesn't need ports? + VolumeMounts: opspec.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName), + Env: append( + opspec.DaemonEnvVars(), + ), Resources: c.fs.Spec.MetadataServer.Resources, } } func (c *cluster) podLabels(mdsConfig *mdsConfig) map[string]string { - labels := opspec.PodLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonName) + labels := opspec.PodLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID) labels["rook_file_system"] = c.fs.Name return labels } diff --git a/pkg/operator/ceph/file/spec_test.go b/pkg/operator/ceph/file/spec_test.go index 2682d04f4..069657664 100644 --- a/pkg/operator/ceph/file/spec_test.go +++ b/pkg/operator/ceph/file/spec_test.go @@ -19,13 +19,14 @@ package file import ( "testing" + "github.com/rook/rook/pkg/operator/ceph/config" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" - mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds" + cephtest "github.com/rook/rook/pkg/operator/ceph/test" - "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" apps "k8s.io/api/apps/v1" @@ -49,7 +50,10 @@ func testDeploymentObject(hostNetwork bool) *apps.Deployment { v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), }, }}}} + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} + c := newCluster( + clusterInfo, &clusterd.Context{Clientset: testop.New(1)}, "rook/rook:myversion", cephv1.CephVersionSpec{Image: "ceph/ceph:testversion", Name: "mimic"}, @@ -59,8 +63,9 @@ func testDeploymentObject(hostNetwork bool) *apps.Deployment { []metav1.OwnerReference{{}}, ) mdsTestConfig := &mdsConfig{ - DaemonName: "myfs-a", + DaemonID: "myfs-a", ResourceName: "rook-ceph-mds-myfs-a", + DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, "myfs-a"), } return c.makeDeployment(mdsTestConfig) } @@ -69,84 +74,15 @@ func TestPodSpecs(t *testing.T) { d := testDeploymentObject(false) // no host network assert.NotNil(t, d) - assert.Equal(t, "rook-ceph-mds-myfs-a", d.Name) - assert.Equal(t, "rook-ceph-mds-myfs-a", d.ObjectMeta.Name) assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy) - assert.Equal(t, 0, len(d.ObjectMeta.Annotations)) - pod := d.Spec.Template + // Deployment should have Ceph labels + cephtest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, + config.MdsType, "myfs-a", "rook-ceph-mds", "ns") - assert.Nil(t, cephtest.VerifyPodLabels("rook-ceph-mds", "ns", "mds", "myfs-a", pod.ObjectMeta.Labels)) - assert.Equal(t, "myfs", pod.ObjectMeta.Labels["rook_file_system"]) - assert.Equal(t, 0, len(d.ObjectMeta.Annotations)) - assert.Nil(t, testop.VolumeExists("rook-data", pod.Spec.Volumes)) - assert.Nil(t, testop.VolumeExists(cephconfig.DefaultConfigMountName, pod.Spec.Volumes)) - assert.Nil(t, testop.VolumeExists(k8sutil.ConfigOverrideName, pod.Spec.Volumes)) - - assert.Equal(t, 1, len(pod.Spec.InitContainers)) - assert.Equal(t, 1, len(pod.Spec.Containers)) - - configImage := "rook/rook:myversion" - configEnvs := 8 - configContainerDefinition := cephtest.ContainerTestDefinition{ - Image: &configImage, - Command: []string{}, // no command - Args: [][]string{ - {"ceph"}, - {mdsdaemon.InitCommand}, - {"--config-dir", "/var/lib/rook"}, - {"--mds-name", "myfs-a"}, - {"--filesystem-id", "15"}, - {"--active-standby", "false"}}, - InOrderArgs: map[int]string{ - 0: "ceph", // ceph must be first arg - 1: mdsdaemon.InitCommand}, // mds init command must be second arg - VolumeMountNames: []string{ - "rook-data", - cephconfig.DefaultConfigMountName, - k8sutil.ConfigOverrideName}, - EnvCount: &configEnvs, - Ports: []v1.ContainerPort{}, - IsPrivileged: nil, // not set in spec - } - cont := &pod.Spec.InitContainers[0] - configContainerDefinition.TestContainer(t, "config init", cont, logger) - assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) - assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) - - daemonImage := "ceph/ceph:testversion" - daemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) - daemonContainerDefinition := cephtest.ContainerTestDefinition{ - Image: &daemonImage, - Command: []string{ - "ceph-mds"}, - Args: [][]string{ - {"--foreground"}, - {"--id", "myfs-a"}}, - VolumeMountNames: []string{ - "rook-data", - cephconfig.DefaultConfigMountName}, - EnvCount: &daemonEnvs, - Ports: []v1.ContainerPort{}, - IsPrivileged: nil, // not set in spec - } - keyringMount, err := testop.GetEnv("ROOK_MDS_KEYRING", cont.Env) - assert.Nil(t, err) // keyring should get the secret named by the filesystem without daemon id - assert.Equal(t, "rook-ceph-mds-myfs-a", keyringMount.ValueFrom.SecretKeyRef.LocalObjectReference.Name) - - cont = &pod.Spec.Containers[0] - daemonContainerDefinition.TestContainer(t, "main mon daemon", cont, logger) - assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) - assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) - - // Verify that all the mounts have volumes and that there are no extraneous volumes - volsMountsTestDef := testop.VolumesAndMountsTestDefinition{ - VolumesSpec: &testop.VolumesSpec{Moniker: "mon pod volumes", Volumes: pod.Spec.Volumes}, - MountsSpecItems: []*testop.MountsSpec{ - {Moniker: "mds config init mounts", Mounts: pod.Spec.InitContainers[0].VolumeMounts}, - {Moniker: "mds daemon mounts", Mounts: pod.Spec.Containers[0].VolumeMounts}}, - } - volsMountsTestDef.TestMountsMatchVolumes(t) + podTemplate := cephtest.NewPodTemplateSpecTester(t, &d.Spec.Template) + podTemplate.RunFullSuite(config.MdsType, "myfs-a", "rook-ceph-mds", "ns", "ceph/ceph:testversion", + "100", "1337" /* resources */) } func TestHostNetwork(t *testing.T) { diff --git a/pkg/operator/ceph/spec/spec.go b/pkg/operator/ceph/spec/spec.go index 77d5bd24f..4ae1a01c6 100644 --- a/pkg/operator/ceph/spec/spec.go +++ b/pkg/operator/ceph/spec/spec.go @@ -92,20 +92,23 @@ func DaemonVolumeMounts(dataPaths *config.DataPathMap, keyringResourceName strin } // DaemonFlags returns the command line flags used by all Ceph daemons. -func DaemonFlags( - clusterInfo *cephconfig.ClusterInfo, - daemonType config.DaemonType, daemonID string, -) []string { +func DaemonFlags(cluster *cephconfig.ClusterInfo, daemonType config.DaemonType, daemonID string) []string { return append( - config.DefaultFlags(clusterInfo.FSID, keyring.VolumeMount().KeyringFilePath()), + config.DefaultFlags(cluster.FSID, keyring.VolumeMount().KeyringFilePath()), // all daemons are named in the format . config.NewFlag("name", fmt.Sprintf("%s.%s", string(daemonType), daemonID)), ) } // AdminFlags returns the command line flags used for Ceph commands requiring admin authentication. -func AdminFlags(clusterInfo *cephconfig.ClusterInfo) []string { - return config.DefaultFlags(clusterInfo.FSID, keyring.VolumeMount().AdminKeyringFilePath()) +func AdminFlags(cluster *cephconfig.ClusterInfo) []string { + return config.DefaultFlags(cluster.FSID, keyring.VolumeMount().AdminKeyringFilePath()) +} + +// ContainerEnvVarReference returns a reference to a Kubernetes container env var of the given name +// which can be used in command or argument fields. +func ContainerEnvVarReference(envVarName string) string { + return fmt.Sprintf("$(%s)", envVarName) } // DaemonEnvVars returns the container environment variables used by all Ceph daemons. diff --git a/pkg/operator/ceph/test/containers.go b/pkg/operator/ceph/test/containers.go index 1b874794b..72837c5c8 100644 --- a/pkg/operator/ceph/test/containers.go +++ b/pkg/operator/ceph/test/containers.go @@ -17,6 +17,7 @@ limitations under the License. package test import ( + "fmt" "regexp" "strings" "testing" @@ -121,7 +122,10 @@ func (ct *ContainersTester) AssertArgReferencesMatchEnvVars() { } // also make sure there are no extraneous env vars // the only allowed extraneous vars are the required vars - assert.ElementsMatch(ct.t, ct.allNonrequiredVarNames(), ct.allNonrequiredArgEnvReferences()) + assert.ElementsMatch(ct.t, ct.allNonrequiredVarNames(), ct.allNonrequiredArgEnvReferences(), + "there are extra arguments or references which do not have a corresponding ref/arg", + fmt.Sprintf("%+v", ct.allNonrequiredVarNames()), + fmt.Sprintf("%+v", ct.allNonrequiredArgEnvReferences())) } // AssertCephImagesMatch asserts that for all Ceph containers under test, the Ceph image used is the diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index d8b1f7014..e25198174 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -20,6 +20,7 @@ import ( "time" "github.com/rook/rook/pkg/clusterd" + "k8s.io/api/apps/v1" apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -80,15 +81,17 @@ func WaitForDeploymentImage(clientset kubernetes.Interface, namespace, label, co return nil } -func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string) error { +// UpdateDeploymentAndWait updates a deployment and waits until it is running to return. It will +// error if the deployment does not exist to be updated or if it takes too long. +func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error) { original, err := context.Clientset.Apps().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err) + return nil, fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err) } logger.Infof("updating deployment %s", deployment.Name) if _, err := context.Clientset.Apps().Deployments(namespace).Update(deployment); err != nil { - return fmt.Errorf("failed to update deployment %s. %+v", deployment.Name, err) + return nil, fmt.Errorf("failed to update deployment %s. %+v", deployment.Name, err) } // wait for the deployment to be restarted @@ -102,18 +105,18 @@ func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deploym // check for the status of the deployment d, err := context.Clientset.Apps().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err) + return nil, fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err) } if d.Status.ObservedGeneration != original.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 { logger.Infof("finished waiting for updated deployment %s", d.Name) - return nil + return d, nil } logger.Debugf("deployment %s status=%v", d.Name, d.Status) time.Sleep(time.Duration(sleepTime) * time.Second) } - return fmt.Errorf("gave up waiting for deployment %s to update", deployment.Name) + return nil, fmt.Errorf("gave up waiting for deployment %s to update", deployment.Name) } // GetDeployments returns a list of deployment names labels matching a given selector diff --git a/pkg/operator/k8sutil/test/deployment.go b/pkg/operator/k8sutil/test/deployment.go index 0e9b0e287..9a3dcb1af 100644 --- a/pkg/operator/k8sutil/test/deployment.go +++ b/pkg/operator/k8sutil/test/deployment.go @@ -2,7 +2,9 @@ package test import ( "github.com/rook/rook/pkg/clusterd" + "k8s.io/api/apps/v1" apps "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // UpdateDeploymentAndWaitStub returns a stub replacement for the UpdateDeploymentAndWait function @@ -14,13 +16,13 @@ import ( // returns a pointer to this slice which the calling func may use to verify the expected contents of // deploymentsUpdated based on expected behavior. func UpdateDeploymentAndWaitStub() ( - stubFunc func(context *clusterd.Context, deployment *apps.Deployment, namespace string) error, + stubFunc func(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error), deploymentsUpdated *[]*apps.Deployment, ) { deploymentsUpdated = &[]*apps.Deployment{} - stubFunc = func(context *clusterd.Context, deployment *apps.Deployment, namespace string) error { + stubFunc = func(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error) { *deploymentsUpdated = append(*deploymentsUpdated, deployment) - return nil + return &v1.Deployment{ObjectMeta: metav1.ObjectMeta{UID: "stub-deployment-uid"}}, nil } return stubFunc, deploymentsUpdated }