From 379cb98689d76f1b514551e034acbc19e6992070 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 15 Mar 2019 14:22:09 -0700 Subject: [PATCH] ceph: replace release name with fine-grained version The basic idea here is to replace the release name in the ClusterInfo object with the fine grained version information queried at runtime. This patch also removes the Name field from the cluster spec, which was also runtime determined and was redundant with ClusterInfo relase name. Signed-off-by: Noah Watkins --- cmd/rook/ceph/ceph.go | 1 - pkg/apis/ceph.rook.io/v1/types.go | 3 -- pkg/apis/ceph.rook.io/v1/version.go | 18 -------- pkg/apis/ceph.rook.io/v1/version_test.go | 39 ---------------- pkg/daemon/ceph/client/filesystem.go | 6 +-- pkg/daemon/ceph/client/mgr.go | 12 ++--- pkg/daemon/ceph/config/config.go | 3 +- pkg/daemon/ceph/config/config_test.go | 8 ++-- pkg/daemon/ceph/config/info.go | 13 +++--- pkg/operator/ceph/cluster/cluster.go | 43 ++++-------------- pkg/operator/ceph/cluster/controller.go | 23 +++++----- pkg/operator/ceph/cluster/mgr/config.go | 3 +- pkg/operator/ceph/cluster/mgr/dashboard.go | 9 ++-- .../ceph/cluster/mgr/dashboard_test.go | 9 +++- pkg/operator/ceph/cluster/mgr/orchestrator.go | 3 +- .../ceph/cluster/mgr/orchestrator_test.go | 15 ++++--- pkg/operator/ceph/cluster/mgr/spec.go | 7 ++- pkg/operator/ceph/cluster/mon/mon.go | 9 ++-- pkg/operator/ceph/cluster/mon/mon_test.go | 13 +++--- pkg/operator/ceph/cluster/mon/node_test.go | 13 +++--- pkg/operator/ceph/cluster/mon/service.go | 5 +-- pkg/operator/ceph/cluster/mon/spec.go | 2 +- pkg/operator/ceph/cluster/osd/spec_test.go | 2 +- pkg/operator/ceph/config/store.go | 3 +- pkg/operator/ceph/file/filesystem.go | 11 ++--- pkg/operator/ceph/file/mds/mds.go | 13 +++--- pkg/operator/ceph/file/mds/spec.go | 3 +- pkg/operator/ceph/file/mds/spec_test.go | 8 +++- pkg/operator/ceph/nfs/controller.go | 11 +++-- pkg/operator/ceph/object/spec_test.go | 7 ++- pkg/operator/ceph/version/version.go | 10 ++--- pkg/operator/ceph/version/version_test.go | 44 +++++++++---------- tests/framework/installer/ceph_installer.go | 6 +-- tests/longhaul/base_block_test.go | 2 +- 34 files changed, 155 insertions(+), 222 deletions(-) delete mode 100644 pkg/apis/ceph.rook.io/v1/version_test.go diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index 1e71da6ad..f5609492e 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -83,7 +83,6 @@ func addCephFlags(command *cobra.Command) { command.Flags().StringVar(&cfg.monEndpoints, "mon-endpoints", "", "ceph mon endpoints") command.Flags().StringVar(&cfg.dataDir, "config-dir", "/var/lib/rook", "directory for storing configuration") command.Flags().StringVar(&cfg.cephConfigOverride, "ceph-config-override", "", "optional path to a ceph config file that will be appended to the config files that rook generates") - command.Flags().StringVar(&clusterInfo.CephVersionName, "ceph-version-name", "", "name of the ceph version") // deprecated ipv4 format address // TODO: remove these legacy flags in the future diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index ce0996f83..b40c0cd6b 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -81,9 +81,6 @@ type CephVersionSpec struct { // Image is the container image used to launch the ceph daemons, such as ceph/ceph:v12.2.7 or ceph/ceph:v13.2.1 Image string `json:"image,omitempty"` - // The name of the major release of Ceph: luminous, mimic, or nautilus - Name string `json:"name,omitempty"` - // Whether to allow unsupported versions (do not set to true in production) AllowUnsupported bool `json:"allowUnsupported,omitempty"` } diff --git a/pkg/apis/ceph.rook.io/v1/version.go b/pkg/apis/ceph.rook.io/v1/version.go index b44b7464f..99408b6c0 100644 --- a/pkg/apis/ceph.rook.io/v1/version.go +++ b/pkg/apis/ceph.rook.io/v1/version.go @@ -16,23 +16,5 @@ limitations under the License. package v1 const ( - Luminous = "luminous" - Mimic = "mimic" - Nautilus = "nautilus" DefaultLuminousImage = "ceph/ceph:v12.2.9-20181026" ) - -func VersionAtLeast(version, minimumVersion string) bool { - orderedVersions := []string{Luminous, Mimic, Nautilus} - found := false - for _, v := range orderedVersions { - if v == minimumVersion { - found = true - } - if v == version { - // if we found the matching version, the version is at least the minimum version if found==true - return found - } - } - return false -} diff --git a/pkg/apis/ceph.rook.io/v1/version_test.go b/pkg/apis/ceph.rook.io/v1/version_test.go deleted file mode 100644 index d49626ea9..000000000 --- a/pkg/apis/ceph.rook.io/v1/version_test.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2018 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package v1 - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestVersionAtLeast(t *testing.T) { - // Valid combinations - assert.True(t, VersionAtLeast(Luminous, Luminous)) - assert.False(t, VersionAtLeast(Luminous, Mimic)) - assert.False(t, VersionAtLeast(Luminous, Nautilus)) - assert.True(t, VersionAtLeast(Mimic, Luminous)) - assert.True(t, VersionAtLeast(Mimic, Mimic)) - assert.False(t, VersionAtLeast(Mimic, Nautilus)) - assert.True(t, VersionAtLeast(Nautilus, Luminous)) - assert.True(t, VersionAtLeast(Nautilus, Mimic)) - assert.True(t, VersionAtLeast(Nautilus, Nautilus)) - - // Invalid combinations - assert.False(t, VersionAtLeast(Mimic, "foo")) - assert.False(t, VersionAtLeast("foo", Luminous)) -} diff --git a/pkg/daemon/ceph/client/filesystem.go b/pkg/daemon/ceph/client/filesystem.go index c32a4c882..e61ab0730 100644 --- a/pkg/daemon/ceph/client/filesystem.go +++ b/pkg/daemon/ceph/client/filesystem.go @@ -23,8 +23,8 @@ import ( "strconv" "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "k8s.io/apimachinery/pkg/util/wait" ) @@ -170,7 +170,7 @@ func IsMultiFSEnabled() bool { } // SetNumMDSRanks sets the number of mds ranks (max_mds) for a Ceph filesystem. -func SetNumMDSRanks(context *clusterd.Context, cephVersionName, clusterName, fsName string, activeMDSCount int32) error { +func SetNumMDSRanks(context *clusterd.Context, cephVersion cephver.CephVersion, clusterName, fsName string, activeMDSCount int32) error { // Noted sections 1 and 2 are necessary for reducing max_mds in Luminous. // See more: [1] http://docs.ceph.com/docs/luminous/cephfs/upgrading/ // [2] https://tracker.ceph.com/issues/23172 @@ -187,7 +187,7 @@ func SetNumMDSRanks(context *clusterd.Context, cephVersionName, clusterName, fsN fsName, activeMDSCount, err) } - if cephv1.VersionAtLeast(cephVersionName, cephv1.Mimic) { + if cephVersion.IsAtLeastMimic() { return nil } diff --git a/pkg/daemon/ceph/client/mgr.go b/pkg/daemon/ceph/client/mgr.go index f74866817..8c4c4879a 100755 --- a/pkg/daemon/ceph/client/mgr.go +++ b/pkg/daemon/ceph/client/mgr.go @@ -20,8 +20,8 @@ import ( "fmt" "strings" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" ) // MgrEnableModule enables a mgr module @@ -35,15 +35,15 @@ func MgrDisableModule(context *clusterd.Context, clusterName, name string) error } // MgrSetAllConfig applies a setting for all mgr daemons -func MgrSetAllConfig(context *clusterd.Context, clusterName, cephVersionName, key, val string) (bool, error) { - return MgrSetConfig(context, clusterName, "", cephVersionName, key, val, false) +func MgrSetAllConfig(context *clusterd.Context, clusterName string, cephVersion cephver.CephVersion, key, val string) (bool, error) { + return MgrSetConfig(context, clusterName, "", cephVersion, key, val, false) } // MgrSetConfig applies a setting for a single mgr daemon -func MgrSetConfig(context *clusterd.Context, clusterName, mgrName, cephVersionName, key, val string, force bool) (bool, error) { +func MgrSetConfig(context *clusterd.Context, clusterName, mgrName string, cephVersion cephver.CephVersion, key, val string, force bool) (bool, error) { var getArgs, setArgs []string mgrID := fmt.Sprintf("mgr.%s", mgrName) - if cephVersionName == cephv1.Luminous || cephVersionName == "" { + if cephVersion.IsLuminous() { getArgs = append(getArgs, "config-key", "get", key) if val == "" { setArgs = append(setArgs, "config-key", "del", key) @@ -57,7 +57,7 @@ func MgrSetConfig(context *clusterd.Context, clusterName, mgrName, cephVersionNa } else { setArgs = append(setArgs, "config", "set", mgrID, key, val) } - if force && cephv1.VersionAtLeast(cephVersionName, cephv1.Nautilus) { + if force && cephVersion.IsAtLeastNautilus() { setArgs = append(setArgs, "--force") } } diff --git a/pkg/daemon/ceph/config/config.go b/pkg/daemon/ceph/config/config.go index 104f4ae06..6909a54e0 100644 --- a/pkg/daemon/ceph/config/config.go +++ b/pkg/daemon/ceph/config/config.go @@ -27,7 +27,6 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/go-ini/ini" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephutil "github.com/rook/rook/pkg/daemon/ceph/util" @@ -202,7 +201,7 @@ func CreateDefaultCephConfig(context *clusterd.Context, cluster *ClusterInfo, ru msgr2Endpoint := net.JoinHostPort(monIP, monPorts[0]) msgr1Endpoint := net.JoinHostPort(monIP, monPorts[1]) - if cephv1.VersionAtLeast(cluster.CephVersionName, cephv1.Nautilus) { + if cluster.CephVersion.IsAtLeastNautilus() { monHosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]" } else { monHosts[i] = msgr1Endpoint diff --git a/pkg/daemon/ceph/config/config_test.go b/pkg/daemon/ceph/config/config_test.go index 48ba097a4..ea12d526f 100644 --- a/pkg/daemon/ceph/config/config_test.go +++ b/pkg/daemon/ceph/config/config_test.go @@ -25,8 +25,8 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/go-ini/ini" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/stretchr/testify/assert" ) @@ -40,7 +40,7 @@ func TestCreateDefaultCephConfig(t *testing.T) { "node0": {Name: "mon0", Endpoint: "10.0.0.1:6789"}, "node1": {Name: "mon1", Endpoint: "10.0.0.2:6789"}, }, - CephVersionName: "mimic", + CephVersion: cephver.Mimic, } // start with INFO level logging @@ -100,7 +100,7 @@ debug bluestore = 1234` Monitors: map[string]*MonInfo{ "node0": {Name: "mon0", Endpoint: "10.0.0.1:6789"}, }, - CephVersionName: "mimic", + CephVersion: cephver.Mimic, } // generate the config file to disk now @@ -136,7 +136,7 @@ func verifyConfig(t *testing.T, cephConfig *CephConfig, cluster *ClusterInfo, lo // Testing mon_host expectedMons := "10.0.0.1:6789,10.0.0.2:6789" - if cephv1.VersionAtLeast(cluster.CephVersionName, cephv1.Nautilus) { + if cluster.CephVersion.IsAtLeastNautilus() { expectedMons = "[v2:10.0.0.1:3300,v1:10.0.0.1:6789],[v2:10.0.0.2:3300,v1:10.0.0.2:6789]" } diff --git a/pkg/daemon/ceph/config/info.go b/pkg/daemon/ceph/config/info.go index ae646d0aa..771242ad4 100644 --- a/pkg/daemon/ceph/config/info.go +++ b/pkg/daemon/ceph/config/info.go @@ -22,17 +22,18 @@ import ( "strings" "github.com/coreos/pkg/capnslog" + cephver "github.com/rook/rook/pkg/operator/ceph/version" ) // ClusterInfo is a collection of information about a particular Ceph cluster. Rook uses information // about the cluster to configure daemons to connect to the desired cluster. type ClusterInfo struct { - FSID string - MonitorSecret string - AdminSecret string - Name string - Monitors map[string]*MonInfo - CephVersionName string + FSID string + MonitorSecret string + AdminSecret string + Name string + Monitors map[string]*MonInfo + CephVersion cephver.CephVersion } // MonInfo is a collection of information about a Ceph mon. diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 97a846937..debe77ee9 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -21,7 +21,6 @@ import ( "fmt" "reflect" "sort" - "strings" "sync" "time" @@ -35,6 +34,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" @@ -46,13 +46,6 @@ const ( detectVersionName = "rook-ceph-detect-version" ) -var ( - // supportedVersions are production-ready versions that rook supports - supportedVersions = []string{cephv1.Luminous, cephv1.Mimic} - // allVersions includes all supportedVersions as well as unreleased versions that are being tested with rook - allVersions = append(supportedVersions, cephv1.Nautilus) -) - type cluster struct { Info *cephconfig.ClusterInfo context *clusterd.Context @@ -83,7 +76,7 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster { } } -func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (string, error) { +func (c *cluster) detectCephVersion(image string, timeout time.Duration) (*cephver.CephVersion, error) { // get the major ceph version by running "ceph --version" in the ceph image podSpec := v1.PodSpec{ Containers: []v1.Container{ @@ -120,21 +113,21 @@ func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (s // run the job to detect the version if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil { - return "", fmt.Errorf("failed to start version job. %+v", err) + return nil, fmt.Errorf("failed to start version job. %+v", err) } if err := k8sutil.WaitForJobCompletion(c.context.Clientset, job, timeout); err != nil { - return "", fmt.Errorf("failed to complete version job. %+v", err) + return nil, fmt.Errorf("failed to complete version job. %+v", err) } log, err := k8sutil.GetPodLog(c.context.Clientset, c.Namespace, "job="+detectVersionName) if err != nil { - return "", fmt.Errorf("failed to get version job log to detect version. %+v", err) + return nil, fmt.Errorf("failed to get version job log to detect version. %+v", err) } - version, err := extractCephVersion(log) + version, err := cephver.ExtractCephVersion(log) if err != nil { - return "", fmt.Errorf("failed to extract ceph version. %+v", err) + return nil, fmt.Errorf("failed to extract ceph version. %+v", err) } // delete the job since we're done with it @@ -144,7 +137,7 @@ func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (s return version, nil } -func (c *cluster) createInstance(rookImage string) error { +func (c *cluster) createInstance(rookImage string, cephVersion cephver.CephVersion) error { if c.checkSetOrchestrationRunning() == true { logger.Debugf("As createInstance is currently running added this request as pending.") c.setOrchestrationPending() @@ -176,7 +169,7 @@ func (c *cluster) createInstance(rookImage string) error { } // Start the mon pods - clusterInfo, err := c.mons.Start(c.Info, rookImage, *c.Spec) + clusterInfo, err := c.mons.Start(c.Info, rookImage, cephVersion, *c.Spec) if err != nil { return fmt.Errorf("failed to start the mons. %+v", err) } @@ -297,24 +290,6 @@ func clusterChanged(oldCluster, newCluster cephv1.ClusterSpec, clusterRef *clust return false, "" } -func extractCephVersion(version string) (string, error) { - for _, v := range allVersions { - if strings.Contains(version, v) { - return v, nil - } - } - return "", fmt.Errorf("failed to parse version from: %s", version) -} - -func versionSupported(version string) bool { - for _, v := range supportedVersions { - if v == version { - return true - } - } - return false -} - func (c *cluster) unsetOrchestrationRunning() { c.orchRunMux.Lock() c.orchestrationRunning = false diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 3b2071d35..2fbfdca22 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -177,7 +177,7 @@ func (c *ClusterController) onK8sNodeAdd(obj interface{}) { if valid, _ := k8sutil.ValidNode(*newNode, cluster.Spec.Placement.All()); valid == true { logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace) - err := cluster.createInstance(c.rookImage) + err := cluster.createInstance(c.rookImage, cluster.Info.CephVersion) if err != nil { logger.Errorf("Failed to update cluster in namespace %s. Was not able to add %s. %+v", cluster.Namespace, newNode.Labels[apis.LabelHostname], err) } @@ -238,15 +238,15 @@ func (c *ClusterController) onAdd(obj interface{}) { logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count) } - cluster.Spec.CephVersion.Name, err = cluster.detectCephMajorVersion(cluster.Spec.CephVersion.Image, 15*time.Minute) + cephVersion, err := cluster.detectCephVersion(cluster.Spec.CephVersion.Image, 15*time.Minute) if err != nil { logger.Errorf("unknown ceph major version. %+v", err) return } if !cluster.Spec.CephVersion.AllowUnsupported { - if !versionSupported(cluster.Spec.CephVersion.Name) { - logger.Errorf("unsupported ceph version detected: %s. allowUnsupported must be set to true to run with this version.", cluster.Spec.CephVersion.Name) + if !cephVersion.Supported() { + logger.Errorf("unsupported ceph version detected: %s. allowUnsupported must be set to true to run with this version.", cephVersion) return } } @@ -258,7 +258,7 @@ func (c *ClusterController) onAdd(obj interface{}) { return false, nil } - err := cluster.createInstance(c.rookImage) + err := cluster.createInstance(c.rookImage, *cephVersion) if err != nil { logger.Errorf("failed to create cluster in namespace %s. %+v", cluster.Namespace, err) return false, nil @@ -298,7 +298,7 @@ func (c *ClusterController) onAdd(obj interface{}) { fileController.StartWatch(cluster.Namespace, cluster.stopCh) // Start nfs ganesha CRD watcher - ganeshaController := nfs.NewCephNFSController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) + ganeshaController := nfs.NewCephNFSController(cluster.Info, c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) ganeshaController.StartWatch(cluster.Namespace, cluster.stopCh) // Start mon health checker @@ -352,7 +352,7 @@ func (c *ClusterController) onK8sNodeUpdate(oldObj, newObj interface{}) { for _, cluster := range c.clusterMap { if valid, _ := k8sutil.ValidNode(*newNode, cephv1.GetOSDPlacement(cluster.Spec.Placement)); valid == true { logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace) - err := cluster.createInstance(c.rookImage) + err := cluster.createInstance(c.rookImage, cluster.Info.CephVersion) if err != nil { logger.Errorf("Failed adding the updated node %s to cluster in namespace %s. %+v", newNode.Labels[apis.LabelHostname], cluster.Namespace, err) continue @@ -429,15 +429,14 @@ func (c *ClusterController) onUpdate(oldObj, newObj interface{}) { // if the image changed, we need to detect the new image version if oldClust.Spec.CephVersion.Image != newClust.Spec.CephVersion.Image { logger.Infof("the ceph version changed. detecting the new image version...") - version, err := cluster.detectCephMajorVersion(newClust.Spec.CephVersion.Image, 15*time.Minute) + version, err := cluster.detectCephVersion(newClust.Spec.CephVersion.Image, 15*time.Minute) if err != nil { logger.Errorf("unknown ceph major version. %+v", err) return } - newClust.Spec.CephVersion.Name = version + cluster.Info.CephVersion = *version } else { - logger.Infof("ceph version is still %s on image %s", cluster.Spec.CephVersion.Name, cluster.Spec.CephVersion.Image) - newClust.Spec.CephVersion.Name = cluster.Spec.CephVersion.Name + logger.Infof("ceph version is still %s on image %s", &cluster.Info.CephVersion, cluster.Spec.CephVersion.Image) } logger.Debugf("old cluster: %+v", oldClust.Spec) @@ -471,7 +470,7 @@ func (c *ClusterController) handleUpdate(crdName string, cluster *cluster) (bool return false, nil } - if err := cluster.createInstance(c.rookImage); err != nil { + if err := cluster.createInstance(c.rookImage, cluster.Info.CephVersion); err != nil { logger.Errorf("failed to update cluster in namespace %s. %+v", cluster.Namespace, err) return false, nil } diff --git a/pkg/operator/ceph/cluster/mgr/config.go b/pkg/operator/ceph/cluster/mgr/config.go index d6745ca81..0910fb49a 100644 --- a/pkg/operator/ceph/cluster/mgr/config.go +++ b/pkg/operator/ceph/cluster/mgr/config.go @@ -19,7 +19,6 @@ package mgr import ( "fmt" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "k8s.io/apimachinery/pkg/api/errors" @@ -47,7 +46,7 @@ type mgrConfig struct { func (c *Cluster) dashboardPort() int { if c.dashboard.Port == 0 { // select default ports - if c.cephVersion.Name == cephv1.Luminous { + if c.clusterInfo.CephVersion.IsLuminous() { return dashboardPortHTTP } return dashboardPortHTTPS diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index 3d85672c5..60d2481b7 100755 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -25,7 +25,6 @@ import ( "syscall" "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" @@ -116,14 +115,14 @@ func (c *Cluster) toggleDashboardModule(dashboardPort int) error { func (c *Cluster) configureDashboardModule(dashboardPort int) error { // url prefix - hasChanged, err := client.MgrSetAllConfig(c.context, c.Namespace, c.cephVersion.Name, "mgr/dashboard/url_prefix", c.dashboard.UrlPrefix) + hasChanged, err := client.MgrSetAllConfig(c.context, c.Namespace, c.clusterInfo.CephVersion, "mgr/dashboard/url_prefix", c.dashboard.UrlPrefix) if err != nil { return err } // server port port := strconv.Itoa(dashboardPort) - changed, err := client.MgrSetAllConfig(c.context, c.Namespace, c.cephVersion.Name, "mgr/dashboard/server_port", port) + changed, err := client.MgrSetAllConfig(c.context, c.Namespace, c.clusterInfo.CephVersion, "mgr/dashboard/server_port", port) if err != nil { return err } @@ -136,7 +135,7 @@ func (c *Cluster) configureDashboardModule(dashboardPort int) error { } else { ssl = strconv.FormatBool(*c.dashboard.SSL) } - changed, err = client.MgrSetAllConfig(c.context, c.Namespace, c.cephVersion.Name, "mgr/dashboard/ssl", ssl) + changed, err = client.MgrSetAllConfig(c.context, c.Namespace, c.clusterInfo.CephVersion, "mgr/dashboard/ssl", ssl) if err != nil { return err } @@ -150,7 +149,7 @@ func (c *Cluster) configureDashboardModule(dashboardPort int) error { } func (c *Cluster) initializeSecureDashboard() error { - if c.cephVersion.Name == cephv1.Luminous || c.cephVersion.Name == "" { + if c.clusterInfo.CephVersion.IsLuminous() { logger.Infof("skipping cert and user configuration on luminous") return nil } diff --git a/pkg/operator/ceph/cluster/mgr/dashboard_test.go b/pkg/operator/ceph/cluster/mgr/dashboard_test.go index f3ff1facb..376336d1e 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard_test.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard_test.go @@ -21,6 +21,8 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -91,8 +93,11 @@ func TestStartSecureDashboard(t *testing.T) { return "", nil }, } - c := &Cluster{context: &clusterd.Context{Clientset: test.New(3), Executor: executor}, Namespace: "myns", - dashboard: cephv1.DashboardSpec{Enabled: true}, cephVersion: cephv1.CephVersionSpec{Name: cephv1.Mimic, Image: "ceph/ceph:v13.2.2"}} + clusterInfo := &cephconfig.ClusterInfo{ + CephVersion: cephver.Mimic, + } + c := &Cluster{clusterInfo: clusterInfo, context: &clusterd.Context{Clientset: test.New(3), Executor: executor}, Namespace: "myns", + dashboard: cephv1.DashboardSpec{Enabled: true}, cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.2"}} c.exitCode = func(err error) (int, bool) { if exitCodeResponse != 0 { return exitCodeResponse, true diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator.go b/pkg/operator/ceph/cluster/mgr/orchestrator.go index a3ad0443b..8a93cf69a 100644 --- a/pkg/operator/ceph/cluster/mgr/orchestrator.go +++ b/pkg/operator/ceph/cluster/mgr/orchestrator.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" ) @@ -36,7 +35,7 @@ var ( // Ceph docs about the orchestrator modules: http://docs.ceph.com/docs/master/mgr/orchestrator_cli/ func (c *Cluster) configureOrchestratorModules() error { - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Infof("skipping enabling orchestrator modules on releases older than nautilus") return nil } diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go index f2be85d3f..2754e0f4e 100644 --- a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go +++ b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go @@ -19,8 +19,9 @@ import ( "fmt" "testing" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + cephver "github.com/rook/rook/pkg/operator/ceph/version" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" ) @@ -50,10 +51,14 @@ func TestOrchestratorModules(t *testing.T) { return "", fmt.Errorf("unexpected ceph command '%v'", args) } - c := &Cluster{context: context} + clusterInfo := &cephconfig.ClusterInfo{ + CephVersion: cephver.Mimic, + } + + c := &Cluster{clusterInfo: clusterInfo, context: context} // the modules are skipped on luminous - c.cephVersion.Name = cephv1.Luminous + c.clusterInfo.CephVersion = cephver.Luminous err := c.configureOrchestratorModules() assert.Nil(t, err) assert.False(t, orchestratorModuleEnabled) @@ -61,7 +66,7 @@ func TestOrchestratorModules(t *testing.T) { assert.False(t, rookBackendSet) // the modules are skipped on mimic - c.cephVersion.Name = cephv1.Mimic + c.clusterInfo.CephVersion = cephver.Mimic err = c.configureOrchestratorModules() assert.Nil(t, err) assert.False(t, orchestratorModuleEnabled) @@ -69,7 +74,7 @@ func TestOrchestratorModules(t *testing.T) { assert.False(t, rookBackendSet) // the modules are configured on nautilus - c.cephVersion.Name = cephv1.Nautilus + c.clusterInfo.CephVersion = cephver.Nautilus err = c.configureOrchestratorModules() assert.Nil(t, err) assert.True(t, orchestratorModuleEnabled) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index e4d539dd1..d6b8dcc10 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -21,7 +21,6 @@ import ( "strconv" "strings" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/config/keyring" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" @@ -59,7 +58,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *apps.Deployment { podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet } c.placement.ApplyToPodSpec(&podSpec.Spec) - if c.cephVersion.Name == cephv1.Luminous || c.cephVersion.Name == "" { + if c.clusterInfo.CephVersion.IsLuminous() { // prepend the keyring-copy workaround for luminous clusters podSpec.Spec.InitContainers = append( []v1.Container{c.makeCopyKeyringInitContainer(mgrConfig)}, @@ -119,14 +118,14 @@ func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule // N: config set mgr.a mgr//server_addr $(ROOK_CEPH__SERVER_ADDR) --force podIPEnvVar := "ROOK_POD_IP" cfgSetArgs := []string{"config", "set"} - if c.cephVersion.Name == cephv1.Luminous || c.cephVersion.Name == "" { + if c.clusterInfo.CephVersion.IsLuminous() { cfgSetArgs[0] = "config-key" } else { cfgSetArgs = append(cfgSetArgs, fmt.Sprintf("mgr.%s", mgrConfig.DaemonID)) } cfgPath := fmt.Sprintf("mgr/%s/server_addr", mgrModule) cfgSetArgs = append(cfgSetArgs, cfgPath, opspec.ContainerEnvVarReference(podIPEnvVar)) - if cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { cfgSetArgs = append(cfgSetArgs, "--force") } cfgSetArgs = append(cfgSetArgs, "--verbose") diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 0f1c1ffe2..6aecdca65 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -36,6 +36,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" opspec "github.com/rook/rook/pkg/operator/ceph/spec" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -147,7 +148,7 @@ func New(context *clusterd.Context, namespace, dataDirHostPath string, hostNetwo } // Start begins the process of running a cluster of Ceph mons. -func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string, spec cephv1.ClusterSpec) (*cephconfig.ClusterInfo, error) { +func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string, cephVersion cephver.CephVersion, spec cephv1.ClusterSpec) (*cephconfig.ClusterInfo, error) { // Only one goroutine can orchestrate the mons at a time c.acquireOrchestrationLock() @@ -171,7 +172,7 @@ func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string, logger.Infof("start running mons") logger.Debugf("establishing ceph cluster info") - if err := c.initClusterInfo(); err != nil { + if err := c.initClusterInfo(cephVersion); err != nil { return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err) } @@ -252,11 +253,11 @@ func (c *Cluster) ensureMonsRunning(mons []*monConfig, i int, requireAllInQuorum // initClusterInfo retrieves the ceph cluster info if it already exists. // If a new cluster, create new keys. -func (c *Cluster) initClusterInfo() error { +func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion) error { var err error // get the cluster info from secret c.clusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, c.Namespace, &c.ownerRef) - c.clusterInfo.CephVersionName = c.spec.CephVersion.Name + c.clusterInfo.CephVersion = cephVersion if err != nil { return fmt.Errorf("failed to get cluster info. %+v", err) diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 1083a378b..47135e00a 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -35,6 +35,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" cephtest "github.com/rook/rook/pkg/daemon/ceph/test" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -146,13 +147,13 @@ func TestStartMonPods(t *testing.T) { c := newCluster(context, namespace, false, true, v1.ResourceRequirements{}) // start a basic cluster - _, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) validateStart(t, c) // starting again should be a no-op, but still results in an error - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) validateStart(t, c) @@ -165,7 +166,7 @@ func TestOperatorRestart(t *testing.T) { c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - info, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + info, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) assert.True(t, info.IsInitialized()) @@ -174,7 +175,7 @@ func TestOperatorRestart(t *testing.T) { c = newCluster(context, namespace, false, true, v1.ResourceRequirements{}) // starting again should be a no-op, but will not result in an error - info, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + info, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) assert.True(t, info.IsInitialized()) @@ -191,7 +192,7 @@ func TestOperatorRestartHostNetwork(t *testing.T) { c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - info, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + info, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) assert.True(t, info.IsInitialized()) @@ -201,7 +202,7 @@ func TestOperatorRestartHostNetwork(t *testing.T) { c = newCluster(context, namespace, true, false, v1.ResourceRequirements{}) // starting again should be a no-op, but still results in an error - info, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + info, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) assert.True(t, info.IsInitialized(), info) diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index dc4286860..38c5a6376 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -22,6 +22,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -180,7 +181,7 @@ func TestHostNetworkSameNode(t *testing.T) { c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Error(t, err) } @@ -198,7 +199,7 @@ func TestPodMemory(t *testing.T) { c := newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Error(t, err) // Test REQUEST == LIMIT @@ -214,7 +215,7 @@ func TestPodMemory(t *testing.T) { c = newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Error(t, err) // Test LIMIT != REQUEST but obviously LIMIT > REQUEST @@ -230,7 +231,7 @@ func TestPodMemory(t *testing.T) { c = newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Error(t, err) // Test valid case where pod resource is set approprietly @@ -246,7 +247,7 @@ func TestPodMemory(t *testing.T) { c = newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) // Test no resources were specified on the pod @@ -254,7 +255,7 @@ func TestPodMemory(t *testing.T) { c = newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) } diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index 58d3a46ab..8218d92cd 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -20,7 +20,6 @@ import ( "fmt" "strconv" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -53,7 +52,7 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { } // If deploying Nautilus or newer we need a new port for the monitor service - if cephv1.VersionAtLeast(c.spec.CephVersion.Name, cephv1.Nautilus) { + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { addServicePort(s, "msgr2", Msgr2port) } @@ -76,7 +75,7 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { // mon endpoint are not actually like, they remain with the mgrs1 format // however it's interesting to show that monitors can be addressed via 2 different ports // in the end the service has msgr1 and msgr2 ports configured so it's not entirely wrong - if cephv1.VersionAtLeast(c.spec.CephVersion.Name, cephv1.Nautilus) { + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Infof("mon %s endpoint are [v2:%s:%s,v1:%s:%d]", mon.DaemonName, s.Spec.ClusterIP, strconv.Itoa(int(Msgr2port)), s.Spec.ClusterIP, mon.Port) } else { logger.Infof("mon %s endpoint is %s:%d", mon.DaemonName, s.Spec.ClusterIP, mon.Port) diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 97d85a7bd..5c3b58f3c 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -185,7 +185,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { } // If deploying Nautilus and newer we need a new port of the monitor container - if cephv1.VersionAtLeast(c.spec.CephVersion.Name, cephv1.Nautilus) { + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { addContainerPort(container, "msgr2", 3300) } diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 9b714125c..a6707cbc6 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -36,7 +36,7 @@ import ( ) func TestPodContainer(t *testing.T) { - cluster := &Cluster{Namespace: "myosd", rookVersion: "23", cephVersion: cephv1.CephVersionSpec{Name: "mimic"}} + cluster := &Cluster{Namespace: "myosd", rookVersion: "23", cephVersion: cephv1.CephVersionSpec{}} c, err := cluster.provisionPodTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "node", "", v1.RestartPolicyAlways) assert.NotNil(t, c) assert.Nil(t, err) diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index 47651f51a..68c038250 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -26,7 +26,6 @@ import ( "strings" "github.com/go-ini/ini" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" cephutil "github.com/rook/rook/pkg/daemon/ceph/util" @@ -169,7 +168,7 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephconfig.ClusterInfo msgr2Endpoint := net.JoinHostPort(monIP, monPorts[0]) msgr1Endpoint := net.JoinHostPort(monIP, monPorts[1]) - if cephv1.VersionAtLeast(clusterInfo.CephVersionName, cephv1.Nautilus) { + if clusterInfo.CephVersion.IsAtLeastNautilus() { hosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]" } else { hosts[i] = msgr1Endpoint diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index 163b75b63..82921b5f3 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/model" "github.com/rook/rook/pkg/operator/ceph/file/mds" "github.com/rook/rook/pkg/operator/ceph/pool" + cephver "github.com/rook/rook/pkg/operator/ceph/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -63,7 +64,7 @@ func createFilesystem( dataPools = append(dataPools, p.ToModel("")) } f := newFS(fs.Name, fs.Spec.MetadataPool.ToModel(""), dataPools, fs.Spec.MetadataServer.ActiveCount) - if err := f.doFilesystemCreate(context, cephVersion.Name, fs.Namespace); err != nil { + if err := f.doFilesystemCreate(context, clusterInfo.CephVersion, fs.Namespace); err != nil { return fmt.Errorf("failed to create filesystem %s: %+v", fs.Name, err) } } @@ -74,7 +75,7 @@ func createFilesystem( } // As of Nautilus, allow_standby_replay is a fs property so we need to apply it - if cephv1.VersionAtLeast(clusterInfo.CephVersionName, cephv1.Nautilus) { + if clusterInfo.CephVersion.IsAtLeastNautilus() { if fs.Spec.MetadataServer.ActiveStandby { if err = client.AllowStandbyReplay(context, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveStandby); err != nil { return fmt.Errorf("failed to set allow_standby_replay to filesystem %s: %v", fs.Name, err) @@ -84,7 +85,7 @@ func createFilesystem( // set the number of active mds instances if fs.Spec.MetadataServer.ActiveCount > 1 { - if err = client.SetNumMDSRanks(context, cephVersion.Name, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil { + if err = client.SetNumMDSRanks(context, clusterInfo.CephVersion, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil { logger.Warningf("failed setting active mds count to %d. %+v", fs.Spec.MetadataServer.ActiveCount, err) } } @@ -159,13 +160,13 @@ func newFS(name string, metadataPool *model.Pool, dataPools []*model.Pool, activ } // doFilesystemCreate starts the Ceph file daemons and creates the filesystem in Ceph. -func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, cephVersionName, clusterName string) error { +func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, cephVersion cephver.CephVersion, clusterName string) error { _, err := client.GetFilesystem(context, clusterName, f.Name) if err == nil { logger.Infof("filesystem %s already exists", f.Name) // Even if the fs already exists, the num active mdses may have changed - if err := client.SetNumMDSRanks(context, cephVersionName, clusterName, f.Name, f.activeMDSCount); err != nil { + if err := client.SetNumMDSRanks(context, cephVersion, clusterName, f.Name, f.activeMDSCount); err != nil { logger.Errorf( fmt.Sprintf("failed to set num mds ranks (max_mds) to %d for filesystem %s, still continuing. ", f.activeMDSCount, f.Name) + "this error is not critical, but mdses may not be as failure tolerant as desired. " + diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index f1ca4832e..ac3ba7d63 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -29,6 +29,7 @@ import ( cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -105,7 +106,7 @@ func (c *Cluster) Start() error { var fsPreparedForUpgrade = false defer func() { if fsPreparedForUpgrade { - if err := finishedWithDaemonUpgrade(c.context, c.cephVersion.Name, c.fs.Namespace, c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount); err != nil { + if err := finishedWithDaemonUpgrade(c.context, c.clusterInfo.CephVersion, c.fs.Namespace, c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount); err != nil { logger.Errorf("for filesystem %s, USER should make sure the Ceph fs max_mds property is set to %d: %+v", c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount, err) } @@ -238,7 +239,8 @@ func DeleteCluster(context *clusterd.Context, namespace, fsName string) error { // Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/ func prepareForDaemonUpgrade( context *clusterd.Context, - cephVersionName, clusterName, fsName string, + cephVersion cephver.CephVersion, + clusterName, fsName string, timeout time.Duration, ) error { logger.Infof("preparing filesystem %s for daemon upgrade", fsName) @@ -247,7 +249,7 @@ func prepareForDaemonUpgrade( // See more: https://ceph.com/releases/v13-2-0-mimic-released/ // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ // As of Oct. 2018, this is only necessary for Luminous and Mimic. - if err := client.SetNumMDSRanks(context, cephVersionName, clusterName, fsName, 1); err != nil { + if err := client.SetNumMDSRanks(context, cephVersion, clusterName, fsName, 1); err != nil { return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err) } if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil { @@ -263,7 +265,8 @@ func prepareForDaemonUpgrade( // ideal state following an upgrade of its daemon(s). func finishedWithDaemonUpgrade( context *clusterd.Context, - cephVersionName, clusterName, fsName string, + cephVersion cephver.CephVersion, + clusterName, fsName string, activeMDSCount int32, ) error { logger.Debugf("restoring filesystem %s from daemon upgrade", fsName) @@ -273,7 +276,7 @@ func finishedWithDaemonUpgrade( // See more: https://ceph.com/releases/v13-2-0-mimic-released/ // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ // TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic. - if err := client.SetNumMDSRanks(context, cephVersionName, clusterName, fsName, activeMDSCount); err != nil { + if err := client.SetNumMDSRanks(context, cephVersion, clusterName, fsName, activeMDSCount); err != nil { return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err) } // * End of noted section 1 return nil diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index b00c13f24..f86c7b65f 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -20,7 +20,6 @@ import ( "fmt" "strconv" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" @@ -89,7 +88,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { ) // These flags are obsoleted as of Nautilus - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { args = append( args, config.NewFlag("mds-standby-for-fscid", c.fsID), diff --git a/pkg/operator/ceph/file/mds/spec_test.go b/pkg/operator/ceph/file/mds/spec_test.go index 6241c9d58..97b7966e6 100644 --- a/pkg/operator/ceph/file/mds/spec_test.go +++ b/pkg/operator/ceph/file/mds/spec_test.go @@ -25,6 +25,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + cephver "github.com/rook/rook/pkg/operator/ceph/version" cephtest "github.com/rook/rook/pkg/operator/ceph/test" testop "github.com/rook/rook/pkg/operator/test" @@ -55,13 +56,16 @@ func testDeploymentObject(hostNetwork bool) *apps.Deployment { }, }, } - clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} + clusterInfo := &cephconfig.ClusterInfo{ + FSID: "myfsid", + CephVersion: cephver.Mimic, + } c := NewCluster( clusterInfo, &clusterd.Context{Clientset: testop.New(1)}, "rook/rook:myversion", - cephv1.CephVersionSpec{Image: "ceph/ceph:testversion", Name: "mimic"}, + cephv1.CephVersionSpec{Image: "ceph/ceph:testversion"}, hostNetwork, fs, &client.CephFilesystemDetails{ID: 15}, diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index 498b0dfa5..2b062e738 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -24,6 +24,7 @@ import ( opkit "github.com/rook/operator-kit" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" @@ -43,6 +44,7 @@ var CephNFSResource = opkit.CustomResource{ // NFSCephNFSController represents a controller for NFS custom resources type CephNFSController struct { + clusterInfo *cephconfig.ClusterInfo context *clusterd.Context rookImage string cephVersion cephv1.CephVersionSpec @@ -51,8 +53,9 @@ type CephNFSController struct { } // NewNFSCephNFSController create controller for watching NFS custom resources created -func NewCephNFSController(context *clusterd.Context, rookImage string, cephVersion cephv1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *CephNFSController { +func NewCephNFSController(clusterInfo *cephconfig.ClusterInfo, context *clusterd.Context, rookImage string, cephVersion cephv1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *CephNFSController { return &CephNFSController{ + clusterInfo: clusterInfo, context: context, rookImage: rookImage, cephVersion: cephVersion, @@ -79,7 +82,7 @@ func (c *CephNFSController) StartWatch(namespace string, stopCh chan struct{}) e func (c *CephNFSController) onAdd(obj interface{}) { nfs := obj.(*cephv1.CephNFS).DeepCopy() - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Errorf("Ceph NFS is only supported with Nautilus or newer. CRD %s will be ignored.", nfs.Name) return } @@ -93,7 +96,7 @@ func (c *CephNFSController) onAdd(obj interface{}) { func (c *CephNFSController) onUpdate(oldObj, newObj interface{}) { oldNFS := oldObj.(*cephv1.CephNFS).DeepCopy() newNFS := newObj.(*cephv1.CephNFS).DeepCopy() - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Errorf("Ceph NFS is only supported with Nautilus or newer. CRD %s will be ignored.", newNFS.Name) return } @@ -120,7 +123,7 @@ func (c *CephNFSController) onUpdate(oldObj, newObj interface{}) { func (c *CephNFSController) onDelete(obj interface{}) { nfs := obj.(*cephv1.CephNFS).DeepCopy() - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Errorf("Ceph NFS is only supported with Nautilus or newer. CRD %s cleanup will be ignored.", nfs.Name) return } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 968ea3859..412df4266 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -23,6 +23,7 @@ import ( "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" cephtest "github.com/rook/rook/pkg/operator/ceph/test" + cephver "github.com/rook/rook/pkg/operator/ceph/version" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -43,13 +44,14 @@ func TestPodSpecs(t *testing.T) { }, } info := testop.CreateConfigDir(1) + info.CephVersion = cephver.Mimic data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default") c := &clusterConfig{ clusterInfo: info, store: store, rookVersion: "rook/rook:myversion", - cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, + cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1"}, hostNetwork: true, DataPathMap: data, } @@ -74,6 +76,7 @@ func TestSSLPodSpec(t *testing.T) { }, } info := testop.CreateConfigDir(1) + info.CephVersion = cephver.Mimic data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default") store.Spec.Gateway.SSLCertificateRef = "mycert" store.Spec.Gateway.SecurePort = 443 @@ -82,7 +85,7 @@ func TestSSLPodSpec(t *testing.T) { clusterInfo: info, store: store, rookVersion: "rook/rook:myversion", - cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, + cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1"}, hostNetwork: true, DataPathMap: data, } diff --git a/pkg/operator/ceph/version/version.go b/pkg/operator/ceph/version/version.go index 85bcbacab..589a473e7 100644 --- a/pkg/operator/ceph/version/version.go +++ b/pkg/operator/ceph/version/version.go @@ -90,7 +90,7 @@ func (v *CephVersion) IsLuminous() bool { return v.isRelease(Luminous) } -func (v *CephVersion) AtLeast(other CephVersion) bool { +func (v *CephVersion) IsAtLeast(other CephVersion) bool { if v.Major > other.Major { return true } else if v.Major < other.Major { @@ -109,10 +109,10 @@ func (v *CephVersion) AtLeast(other CephVersion) bool { return true } -func (v *CephVersion) AtLeastNautilus() bool { - return v.AtLeast(Nautilus) +func (v *CephVersion) IsAtLeastNautilus() bool { + return v.IsAtLeast(Nautilus) } -func (v *CephVersion) AtLeastMimic() bool { - return v.AtLeast(Mimic) +func (v *CephVersion) IsAtLeastMimic() bool { + return v.IsAtLeast(Mimic) } diff --git a/pkg/operator/ceph/version/version_test.go b/pkg/operator/ceph/version/version_test.go index d784b6f60..535f74ac0 100644 --- a/pkg/operator/ceph/version/version_test.go +++ b/pkg/operator/ceph/version/version_test.go @@ -113,30 +113,30 @@ func TestIsReleaseX(t *testing.T) { } func TestVersionAtLeast(t *testing.T) { - assert.True(t, Luminous.AtLeast(Luminous)) - assert.False(t, Luminous.AtLeast(Mimic)) - assert.False(t, Luminous.AtLeast(Nautilus)) - assert.True(t, Mimic.AtLeast(Luminous)) - assert.True(t, Mimic.AtLeast(Mimic)) - assert.False(t, Mimic.AtLeast(Nautilus)) - assert.True(t, Nautilus.AtLeast(Luminous)) - assert.True(t, Nautilus.AtLeast(Mimic)) - assert.True(t, Nautilus.AtLeast(Nautilus)) + assert.True(t, Luminous.IsAtLeast(Luminous)) + assert.False(t, Luminous.IsAtLeast(Mimic)) + assert.False(t, Luminous.IsAtLeast(Nautilus)) + assert.True(t, Mimic.IsAtLeast(Luminous)) + assert.True(t, Mimic.IsAtLeast(Mimic)) + assert.False(t, Mimic.IsAtLeast(Nautilus)) + assert.True(t, Nautilus.IsAtLeast(Luminous)) + assert.True(t, Nautilus.IsAtLeast(Mimic)) + assert.True(t, Nautilus.IsAtLeast(Nautilus)) - assert.True(t, (&CephVersion{1, 0, 0}).AtLeast(CephVersion{0, 0, 0})) - assert.False(t, (&CephVersion{0, 0, 0}).AtLeast(CephVersion{1, 0, 0})) - assert.True(t, (&CephVersion{1, 1, 0}).AtLeast(CephVersion{1, 0, 0})) - assert.False(t, (&CephVersion{1, 0, 0}).AtLeast(CephVersion{1, 1, 0})) - assert.True(t, (&CephVersion{1, 1, 1}).AtLeast(CephVersion{1, 1, 0})) - assert.False(t, (&CephVersion{1, 1, 0}).AtLeast(CephVersion{1, 1, 1})) - assert.True(t, (&CephVersion{1, 1, 1}).AtLeast(CephVersion{1, 1, 1})) + assert.True(t, (&CephVersion{1, 0, 0}).IsAtLeast(CephVersion{0, 0, 0})) + assert.False(t, (&CephVersion{0, 0, 0}).IsAtLeast(CephVersion{1, 0, 0})) + assert.True(t, (&CephVersion{1, 1, 0}).IsAtLeast(CephVersion{1, 0, 0})) + assert.False(t, (&CephVersion{1, 0, 0}).IsAtLeast(CephVersion{1, 1, 0})) + assert.True(t, (&CephVersion{1, 1, 1}).IsAtLeast(CephVersion{1, 1, 0})) + assert.False(t, (&CephVersion{1, 1, 0}).IsAtLeast(CephVersion{1, 1, 1})) + assert.True(t, (&CephVersion{1, 1, 1}).IsAtLeast(CephVersion{1, 1, 1})) } func TestVersionAtLeastX(t *testing.T) { - assert.True(t, Nautilus.AtLeastNautilus()) - assert.True(t, Nautilus.AtLeastMimic()) - assert.True(t, Mimic.AtLeastMimic()) - assert.False(t, Luminous.AtLeastMimic()) - assert.False(t, Luminous.AtLeastNautilus()) - assert.False(t, Mimic.AtLeastNautilus()) + assert.True(t, Nautilus.IsAtLeastNautilus()) + assert.True(t, Nautilus.IsAtLeastMimic()) + assert.True(t, Mimic.IsAtLeastMimic()) + assert.False(t, Luminous.IsAtLeastMimic()) + assert.False(t, Luminous.IsAtLeastNautilus()) + assert.False(t, Mimic.IsAtLeastNautilus()) } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 5e6575040..924debdc7 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -48,8 +48,8 @@ const ( ) var ( - LuminousVersion = cephv1.CephVersionSpec{Image: luminousTestImage, Name: cephv1.Luminous} - MimicVersion = cephv1.CephVersionSpec{Image: mimicTestImage, Name: cephv1.Mimic} + LuminousVersion = cephv1.CephVersionSpec{Image: luminousTestImage} + MimicVersion = cephv1.CephVersionSpec{Image: mimicTestImage} ) // CephInstaller wraps installing and uninstalling rook on a platform @@ -474,7 +474,7 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rook panic("failed to get kubectl client :" + err.Error()) } logger.Infof("Rook Version: %s", rookVersion) - logger.Infof("Ceph Version: %s (%s)", cephVersion.Image, cephVersion.Name) + logger.Infof("Ceph Version: %s", cephVersion.Image) h := &CephInstaller{ Manifests: NewCephManifests(rookVersion), k8shelper: k8shelp, diff --git a/tests/longhaul/base_block_test.go b/tests/longhaul/base_block_test.go index 853ba018f..acb922579 100644 --- a/tests/longhaul/base_block_test.go +++ b/tests/longhaul/base_block_test.go @@ -145,7 +145,7 @@ func StartLoadTestCluster(t func() *testing.T, namespace string) (LoadTestCluste kh, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster, cephv1.CephVersionSpec{Image: "ceph/ceph:v12.2.7", Name: "luminous"}) + i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster, cephv1.CephVersionSpec{Image: "ceph/ceph:v12.2.7"}) op := LoadTestCluster{i, kh, nil, t, namespace} op.Setup()