diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index 1e71da6ad..f5609492e 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -83,7 +83,6 @@ func addCephFlags(command *cobra.Command) { command.Flags().StringVar(&cfg.monEndpoints, "mon-endpoints", "", "ceph mon endpoints") command.Flags().StringVar(&cfg.dataDir, "config-dir", "/var/lib/rook", "directory for storing configuration") command.Flags().StringVar(&cfg.cephConfigOverride, "ceph-config-override", "", "optional path to a ceph config file that will be appended to the config files that rook generates") - command.Flags().StringVar(&clusterInfo.CephVersionName, "ceph-version-name", "", "name of the ceph version") // deprecated ipv4 format address // TODO: remove these legacy flags in the future diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index ce0996f83..b40c0cd6b 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -81,9 +81,6 @@ type CephVersionSpec struct { // Image is the container image used to launch the ceph daemons, such as ceph/ceph:v12.2.7 or ceph/ceph:v13.2.1 Image string `json:"image,omitempty"` - // The name of the major release of Ceph: luminous, mimic, or nautilus - Name string `json:"name,omitempty"` - // Whether to allow unsupported versions (do not set to true in production) AllowUnsupported bool `json:"allowUnsupported,omitempty"` } diff --git a/pkg/apis/ceph.rook.io/v1/version.go b/pkg/apis/ceph.rook.io/v1/version.go index b44b7464f..99408b6c0 100644 --- a/pkg/apis/ceph.rook.io/v1/version.go +++ b/pkg/apis/ceph.rook.io/v1/version.go @@ -16,23 +16,5 @@ limitations under the License. package v1 const ( - Luminous = "luminous" - Mimic = "mimic" - Nautilus = "nautilus" DefaultLuminousImage = "ceph/ceph:v12.2.9-20181026" ) - -func VersionAtLeast(version, minimumVersion string) bool { - orderedVersions := []string{Luminous, Mimic, Nautilus} - found := false - for _, v := range orderedVersions { - if v == minimumVersion { - found = true - } - if v == version { - // if we found the matching version, the version is at least the minimum version if found==true - return found - } - } - return false -} diff --git a/pkg/apis/ceph.rook.io/v1/version_test.go b/pkg/apis/ceph.rook.io/v1/version_test.go deleted file mode 100644 index d49626ea9..000000000 --- a/pkg/apis/ceph.rook.io/v1/version_test.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2018 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package v1 - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestVersionAtLeast(t *testing.T) { - // Valid combinations - assert.True(t, VersionAtLeast(Luminous, Luminous)) - assert.False(t, VersionAtLeast(Luminous, Mimic)) - assert.False(t, VersionAtLeast(Luminous, Nautilus)) - assert.True(t, VersionAtLeast(Mimic, Luminous)) - assert.True(t, VersionAtLeast(Mimic, Mimic)) - assert.False(t, VersionAtLeast(Mimic, Nautilus)) - assert.True(t, VersionAtLeast(Nautilus, Luminous)) - assert.True(t, VersionAtLeast(Nautilus, Mimic)) - assert.True(t, VersionAtLeast(Nautilus, Nautilus)) - - // Invalid combinations - assert.False(t, VersionAtLeast(Mimic, "foo")) - assert.False(t, VersionAtLeast("foo", Luminous)) -} diff --git a/pkg/daemon/ceph/client/filesystem.go b/pkg/daemon/ceph/client/filesystem.go index c32a4c882..e61ab0730 100644 --- a/pkg/daemon/ceph/client/filesystem.go +++ b/pkg/daemon/ceph/client/filesystem.go @@ -23,8 +23,8 @@ import ( "strconv" "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "k8s.io/apimachinery/pkg/util/wait" ) @@ -170,7 +170,7 @@ func IsMultiFSEnabled() bool { } // SetNumMDSRanks sets the number of mds ranks (max_mds) for a Ceph filesystem. -func SetNumMDSRanks(context *clusterd.Context, cephVersionName, clusterName, fsName string, activeMDSCount int32) error { +func SetNumMDSRanks(context *clusterd.Context, cephVersion cephver.CephVersion, clusterName, fsName string, activeMDSCount int32) error { // Noted sections 1 and 2 are necessary for reducing max_mds in Luminous. // See more: [1] http://docs.ceph.com/docs/luminous/cephfs/upgrading/ // [2] https://tracker.ceph.com/issues/23172 @@ -187,7 +187,7 @@ func SetNumMDSRanks(context *clusterd.Context, cephVersionName, clusterName, fsN fsName, activeMDSCount, err) } - if cephv1.VersionAtLeast(cephVersionName, cephv1.Mimic) { + if cephVersion.IsAtLeastMimic() { return nil } diff --git a/pkg/daemon/ceph/client/mgr.go b/pkg/daemon/ceph/client/mgr.go index f74866817..8c4c4879a 100755 --- a/pkg/daemon/ceph/client/mgr.go +++ b/pkg/daemon/ceph/client/mgr.go @@ -20,8 +20,8 @@ import ( "fmt" "strings" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" ) // MgrEnableModule enables a mgr module @@ -35,15 +35,15 @@ func MgrDisableModule(context *clusterd.Context, clusterName, name string) error } // MgrSetAllConfig applies a setting for all mgr daemons -func MgrSetAllConfig(context *clusterd.Context, clusterName, cephVersionName, key, val string) (bool, error) { - return MgrSetConfig(context, clusterName, "", cephVersionName, key, val, false) +func MgrSetAllConfig(context *clusterd.Context, clusterName string, cephVersion cephver.CephVersion, key, val string) (bool, error) { + return MgrSetConfig(context, clusterName, "", cephVersion, key, val, false) } // MgrSetConfig applies a setting for a single mgr daemon -func MgrSetConfig(context *clusterd.Context, clusterName, mgrName, cephVersionName, key, val string, force bool) (bool, error) { +func MgrSetConfig(context *clusterd.Context, clusterName, mgrName string, cephVersion cephver.CephVersion, key, val string, force bool) (bool, error) { var getArgs, setArgs []string mgrID := fmt.Sprintf("mgr.%s", mgrName) - if cephVersionName == cephv1.Luminous || cephVersionName == "" { + if cephVersion.IsLuminous() { getArgs = append(getArgs, "config-key", "get", key) if val == "" { setArgs = append(setArgs, "config-key", "del", key) @@ -57,7 +57,7 @@ func MgrSetConfig(context *clusterd.Context, clusterName, mgrName, cephVersionNa } else { setArgs = append(setArgs, "config", "set", mgrID, key, val) } - if force && cephv1.VersionAtLeast(cephVersionName, cephv1.Nautilus) { + if force && cephVersion.IsAtLeastNautilus() { setArgs = append(setArgs, "--force") } } diff --git a/pkg/daemon/ceph/config/config.go b/pkg/daemon/ceph/config/config.go index 104f4ae06..6909a54e0 100644 --- a/pkg/daemon/ceph/config/config.go +++ b/pkg/daemon/ceph/config/config.go @@ -27,7 +27,6 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/go-ini/ini" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephutil "github.com/rook/rook/pkg/daemon/ceph/util" @@ -202,7 +201,7 @@ func CreateDefaultCephConfig(context *clusterd.Context, cluster *ClusterInfo, ru msgr2Endpoint := net.JoinHostPort(monIP, monPorts[0]) msgr1Endpoint := net.JoinHostPort(monIP, monPorts[1]) - if cephv1.VersionAtLeast(cluster.CephVersionName, cephv1.Nautilus) { + if cluster.CephVersion.IsAtLeastNautilus() { monHosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]" } else { monHosts[i] = msgr1Endpoint diff --git a/pkg/daemon/ceph/config/config_test.go b/pkg/daemon/ceph/config/config_test.go index 48ba097a4..ea12d526f 100644 --- a/pkg/daemon/ceph/config/config_test.go +++ b/pkg/daemon/ceph/config/config_test.go @@ -25,8 +25,8 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/go-ini/ini" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/stretchr/testify/assert" ) @@ -40,7 +40,7 @@ func TestCreateDefaultCephConfig(t *testing.T) { "node0": {Name: "mon0", Endpoint: "10.0.0.1:6789"}, "node1": {Name: "mon1", Endpoint: "10.0.0.2:6789"}, }, - CephVersionName: "mimic", + CephVersion: cephver.Mimic, } // start with INFO level logging @@ -100,7 +100,7 @@ debug bluestore = 1234` Monitors: map[string]*MonInfo{ "node0": {Name: "mon0", Endpoint: "10.0.0.1:6789"}, }, - CephVersionName: "mimic", + CephVersion: cephver.Mimic, } // generate the config file to disk now @@ -136,7 +136,7 @@ func verifyConfig(t *testing.T, cephConfig *CephConfig, cluster *ClusterInfo, lo // Testing mon_host expectedMons := "10.0.0.1:6789,10.0.0.2:6789" - if cephv1.VersionAtLeast(cluster.CephVersionName, cephv1.Nautilus) { + if cluster.CephVersion.IsAtLeastNautilus() { expectedMons = "[v2:10.0.0.1:3300,v1:10.0.0.1:6789],[v2:10.0.0.2:3300,v1:10.0.0.2:6789]" } diff --git a/pkg/daemon/ceph/config/info.go b/pkg/daemon/ceph/config/info.go index ae646d0aa..771242ad4 100644 --- a/pkg/daemon/ceph/config/info.go +++ b/pkg/daemon/ceph/config/info.go @@ -22,17 +22,18 @@ import ( "strings" "github.com/coreos/pkg/capnslog" + cephver "github.com/rook/rook/pkg/operator/ceph/version" ) // ClusterInfo is a collection of information about a particular Ceph cluster. Rook uses information // about the cluster to configure daemons to connect to the desired cluster. type ClusterInfo struct { - FSID string - MonitorSecret string - AdminSecret string - Name string - Monitors map[string]*MonInfo - CephVersionName string + FSID string + MonitorSecret string + AdminSecret string + Name string + Monitors map[string]*MonInfo + CephVersion cephver.CephVersion } // MonInfo is a collection of information about a Ceph mon. diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 97a846937..debe77ee9 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -21,7 +21,6 @@ import ( "fmt" "reflect" "sort" - "strings" "sync" "time" @@ -35,6 +34,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" @@ -46,13 +46,6 @@ const ( detectVersionName = "rook-ceph-detect-version" ) -var ( - // supportedVersions are production-ready versions that rook supports - supportedVersions = []string{cephv1.Luminous, cephv1.Mimic} - // allVersions includes all supportedVersions as well as unreleased versions that are being tested with rook - allVersions = append(supportedVersions, cephv1.Nautilus) -) - type cluster struct { Info *cephconfig.ClusterInfo context *clusterd.Context @@ -83,7 +76,7 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster { } } -func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (string, error) { +func (c *cluster) detectCephVersion(image string, timeout time.Duration) (*cephver.CephVersion, error) { // get the major ceph version by running "ceph --version" in the ceph image podSpec := v1.PodSpec{ Containers: []v1.Container{ @@ -120,21 +113,21 @@ func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (s // run the job to detect the version if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil { - return "", fmt.Errorf("failed to start version job. %+v", err) + return nil, fmt.Errorf("failed to start version job. %+v", err) } if err := k8sutil.WaitForJobCompletion(c.context.Clientset, job, timeout); err != nil { - return "", fmt.Errorf("failed to complete version job. %+v", err) + return nil, fmt.Errorf("failed to complete version job. %+v", err) } log, err := k8sutil.GetPodLog(c.context.Clientset, c.Namespace, "job="+detectVersionName) if err != nil { - return "", fmt.Errorf("failed to get version job log to detect version. %+v", err) + return nil, fmt.Errorf("failed to get version job log to detect version. %+v", err) } - version, err := extractCephVersion(log) + version, err := cephver.ExtractCephVersion(log) if err != nil { - return "", fmt.Errorf("failed to extract ceph version. %+v", err) + return nil, fmt.Errorf("failed to extract ceph version. %+v", err) } // delete the job since we're done with it @@ -144,7 +137,7 @@ func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (s return version, nil } -func (c *cluster) createInstance(rookImage string) error { +func (c *cluster) createInstance(rookImage string, cephVersion cephver.CephVersion) error { if c.checkSetOrchestrationRunning() == true { logger.Debugf("As createInstance is currently running added this request as pending.") c.setOrchestrationPending() @@ -176,7 +169,7 @@ func (c *cluster) createInstance(rookImage string) error { } // Start the mon pods - clusterInfo, err := c.mons.Start(c.Info, rookImage, *c.Spec) + clusterInfo, err := c.mons.Start(c.Info, rookImage, cephVersion, *c.Spec) if err != nil { return fmt.Errorf("failed to start the mons. %+v", err) } @@ -297,24 +290,6 @@ func clusterChanged(oldCluster, newCluster cephv1.ClusterSpec, clusterRef *clust return false, "" } -func extractCephVersion(version string) (string, error) { - for _, v := range allVersions { - if strings.Contains(version, v) { - return v, nil - } - } - return "", fmt.Errorf("failed to parse version from: %s", version) -} - -func versionSupported(version string) bool { - for _, v := range supportedVersions { - if v == version { - return true - } - } - return false -} - func (c *cluster) unsetOrchestrationRunning() { c.orchRunMux.Lock() c.orchestrationRunning = false diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 3b2071d35..2fbfdca22 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -177,7 +177,7 @@ func (c *ClusterController) onK8sNodeAdd(obj interface{}) { if valid, _ := k8sutil.ValidNode(*newNode, cluster.Spec.Placement.All()); valid == true { logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace) - err := cluster.createInstance(c.rookImage) + err := cluster.createInstance(c.rookImage, cluster.Info.CephVersion) if err != nil { logger.Errorf("Failed to update cluster in namespace %s. Was not able to add %s. %+v", cluster.Namespace, newNode.Labels[apis.LabelHostname], err) } @@ -238,15 +238,15 @@ func (c *ClusterController) onAdd(obj interface{}) { logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count) } - cluster.Spec.CephVersion.Name, err = cluster.detectCephMajorVersion(cluster.Spec.CephVersion.Image, 15*time.Minute) + cephVersion, err := cluster.detectCephVersion(cluster.Spec.CephVersion.Image, 15*time.Minute) if err != nil { logger.Errorf("unknown ceph major version. %+v", err) return } if !cluster.Spec.CephVersion.AllowUnsupported { - if !versionSupported(cluster.Spec.CephVersion.Name) { - logger.Errorf("unsupported ceph version detected: %s. allowUnsupported must be set to true to run with this version.", cluster.Spec.CephVersion.Name) + if !cephVersion.Supported() { + logger.Errorf("unsupported ceph version detected: %s. allowUnsupported must be set to true to run with this version.", cephVersion) return } } @@ -258,7 +258,7 @@ func (c *ClusterController) onAdd(obj interface{}) { return false, nil } - err := cluster.createInstance(c.rookImage) + err := cluster.createInstance(c.rookImage, *cephVersion) if err != nil { logger.Errorf("failed to create cluster in namespace %s. %+v", cluster.Namespace, err) return false, nil @@ -298,7 +298,7 @@ func (c *ClusterController) onAdd(obj interface{}) { fileController.StartWatch(cluster.Namespace, cluster.stopCh) // Start nfs ganesha CRD watcher - ganeshaController := nfs.NewCephNFSController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) + ganeshaController := nfs.NewCephNFSController(cluster.Info, c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) ganeshaController.StartWatch(cluster.Namespace, cluster.stopCh) // Start mon health checker @@ -352,7 +352,7 @@ func (c *ClusterController) onK8sNodeUpdate(oldObj, newObj interface{}) { for _, cluster := range c.clusterMap { if valid, _ := k8sutil.ValidNode(*newNode, cephv1.GetOSDPlacement(cluster.Spec.Placement)); valid == true { logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace) - err := cluster.createInstance(c.rookImage) + err := cluster.createInstance(c.rookImage, cluster.Info.CephVersion) if err != nil { logger.Errorf("Failed adding the updated node %s to cluster in namespace %s. %+v", newNode.Labels[apis.LabelHostname], cluster.Namespace, err) continue @@ -429,15 +429,14 @@ func (c *ClusterController) onUpdate(oldObj, newObj interface{}) { // if the image changed, we need to detect the new image version if oldClust.Spec.CephVersion.Image != newClust.Spec.CephVersion.Image { logger.Infof("the ceph version changed. detecting the new image version...") - version, err := cluster.detectCephMajorVersion(newClust.Spec.CephVersion.Image, 15*time.Minute) + version, err := cluster.detectCephVersion(newClust.Spec.CephVersion.Image, 15*time.Minute) if err != nil { logger.Errorf("unknown ceph major version. %+v", err) return } - newClust.Spec.CephVersion.Name = version + cluster.Info.CephVersion = *version } else { - logger.Infof("ceph version is still %s on image %s", cluster.Spec.CephVersion.Name, cluster.Spec.CephVersion.Image) - newClust.Spec.CephVersion.Name = cluster.Spec.CephVersion.Name + logger.Infof("ceph version is still %s on image %s", &cluster.Info.CephVersion, cluster.Spec.CephVersion.Image) } logger.Debugf("old cluster: %+v", oldClust.Spec) @@ -471,7 +470,7 @@ func (c *ClusterController) handleUpdate(crdName string, cluster *cluster) (bool return false, nil } - if err := cluster.createInstance(c.rookImage); err != nil { + if err := cluster.createInstance(c.rookImage, cluster.Info.CephVersion); err != nil { logger.Errorf("failed to update cluster in namespace %s. %+v", cluster.Namespace, err) return false, nil } diff --git a/pkg/operator/ceph/cluster/mgr/config.go b/pkg/operator/ceph/cluster/mgr/config.go index d6745ca81..0910fb49a 100644 --- a/pkg/operator/ceph/cluster/mgr/config.go +++ b/pkg/operator/ceph/cluster/mgr/config.go @@ -19,7 +19,6 @@ package mgr import ( "fmt" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "k8s.io/apimachinery/pkg/api/errors" @@ -47,7 +46,7 @@ type mgrConfig struct { func (c *Cluster) dashboardPort() int { if c.dashboard.Port == 0 { // select default ports - if c.cephVersion.Name == cephv1.Luminous { + if c.clusterInfo.CephVersion.IsLuminous() { return dashboardPortHTTP } return dashboardPortHTTPS diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index 3d85672c5..60d2481b7 100755 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -25,7 +25,6 @@ import ( "syscall" "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" @@ -116,14 +115,14 @@ func (c *Cluster) toggleDashboardModule(dashboardPort int) error { func (c *Cluster) configureDashboardModule(dashboardPort int) error { // url prefix - hasChanged, err := client.MgrSetAllConfig(c.context, c.Namespace, c.cephVersion.Name, "mgr/dashboard/url_prefix", c.dashboard.UrlPrefix) + hasChanged, err := client.MgrSetAllConfig(c.context, c.Namespace, c.clusterInfo.CephVersion, "mgr/dashboard/url_prefix", c.dashboard.UrlPrefix) if err != nil { return err } // server port port := strconv.Itoa(dashboardPort) - changed, err := client.MgrSetAllConfig(c.context, c.Namespace, c.cephVersion.Name, "mgr/dashboard/server_port", port) + changed, err := client.MgrSetAllConfig(c.context, c.Namespace, c.clusterInfo.CephVersion, "mgr/dashboard/server_port", port) if err != nil { return err } @@ -136,7 +135,7 @@ func (c *Cluster) configureDashboardModule(dashboardPort int) error { } else { ssl = strconv.FormatBool(*c.dashboard.SSL) } - changed, err = client.MgrSetAllConfig(c.context, c.Namespace, c.cephVersion.Name, "mgr/dashboard/ssl", ssl) + changed, err = client.MgrSetAllConfig(c.context, c.Namespace, c.clusterInfo.CephVersion, "mgr/dashboard/ssl", ssl) if err != nil { return err } @@ -150,7 +149,7 @@ func (c *Cluster) configureDashboardModule(dashboardPort int) error { } func (c *Cluster) initializeSecureDashboard() error { - if c.cephVersion.Name == cephv1.Luminous || c.cephVersion.Name == "" { + if c.clusterInfo.CephVersion.IsLuminous() { logger.Infof("skipping cert and user configuration on luminous") return nil } diff --git a/pkg/operator/ceph/cluster/mgr/dashboard_test.go b/pkg/operator/ceph/cluster/mgr/dashboard_test.go index f3ff1facb..376336d1e 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard_test.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard_test.go @@ -21,6 +21,8 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -91,8 +93,11 @@ func TestStartSecureDashboard(t *testing.T) { return "", nil }, } - c := &Cluster{context: &clusterd.Context{Clientset: test.New(3), Executor: executor}, Namespace: "myns", - dashboard: cephv1.DashboardSpec{Enabled: true}, cephVersion: cephv1.CephVersionSpec{Name: cephv1.Mimic, Image: "ceph/ceph:v13.2.2"}} + clusterInfo := &cephconfig.ClusterInfo{ + CephVersion: cephver.Mimic, + } + c := &Cluster{clusterInfo: clusterInfo, context: &clusterd.Context{Clientset: test.New(3), Executor: executor}, Namespace: "myns", + dashboard: cephv1.DashboardSpec{Enabled: true}, cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.2"}} c.exitCode = func(err error) (int, bool) { if exitCodeResponse != 0 { return exitCodeResponse, true diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator.go b/pkg/operator/ceph/cluster/mgr/orchestrator.go index a3ad0443b..8a93cf69a 100644 --- a/pkg/operator/ceph/cluster/mgr/orchestrator.go +++ b/pkg/operator/ceph/cluster/mgr/orchestrator.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" ) @@ -36,7 +35,7 @@ var ( // Ceph docs about the orchestrator modules: http://docs.ceph.com/docs/master/mgr/orchestrator_cli/ func (c *Cluster) configureOrchestratorModules() error { - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Infof("skipping enabling orchestrator modules on releases older than nautilus") return nil } diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go index f2be85d3f..2754e0f4e 100644 --- a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go +++ b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go @@ -19,8 +19,9 @@ import ( "fmt" "testing" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + cephver "github.com/rook/rook/pkg/operator/ceph/version" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" ) @@ -50,10 +51,14 @@ func TestOrchestratorModules(t *testing.T) { return "", fmt.Errorf("unexpected ceph command '%v'", args) } - c := &Cluster{context: context} + clusterInfo := &cephconfig.ClusterInfo{ + CephVersion: cephver.Mimic, + } + + c := &Cluster{clusterInfo: clusterInfo, context: context} // the modules are skipped on luminous - c.cephVersion.Name = cephv1.Luminous + c.clusterInfo.CephVersion = cephver.Luminous err := c.configureOrchestratorModules() assert.Nil(t, err) assert.False(t, orchestratorModuleEnabled) @@ -61,7 +66,7 @@ func TestOrchestratorModules(t *testing.T) { assert.False(t, rookBackendSet) // the modules are skipped on mimic - c.cephVersion.Name = cephv1.Mimic + c.clusterInfo.CephVersion = cephver.Mimic err = c.configureOrchestratorModules() assert.Nil(t, err) assert.False(t, orchestratorModuleEnabled) @@ -69,7 +74,7 @@ func TestOrchestratorModules(t *testing.T) { assert.False(t, rookBackendSet) // the modules are configured on nautilus - c.cephVersion.Name = cephv1.Nautilus + c.clusterInfo.CephVersion = cephver.Nautilus err = c.configureOrchestratorModules() assert.Nil(t, err) assert.True(t, orchestratorModuleEnabled) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index e4d539dd1..d6b8dcc10 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -21,7 +21,6 @@ import ( "strconv" "strings" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/config/keyring" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" @@ -59,7 +58,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *apps.Deployment { podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet } c.placement.ApplyToPodSpec(&podSpec.Spec) - if c.cephVersion.Name == cephv1.Luminous || c.cephVersion.Name == "" { + if c.clusterInfo.CephVersion.IsLuminous() { // prepend the keyring-copy workaround for luminous clusters podSpec.Spec.InitContainers = append( []v1.Container{c.makeCopyKeyringInitContainer(mgrConfig)}, @@ -119,14 +118,14 @@ func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule // N: config set mgr.a mgr//server_addr $(ROOK_CEPH__SERVER_ADDR) --force podIPEnvVar := "ROOK_POD_IP" cfgSetArgs := []string{"config", "set"} - if c.cephVersion.Name == cephv1.Luminous || c.cephVersion.Name == "" { + if c.clusterInfo.CephVersion.IsLuminous() { cfgSetArgs[0] = "config-key" } else { cfgSetArgs = append(cfgSetArgs, fmt.Sprintf("mgr.%s", mgrConfig.DaemonID)) } cfgPath := fmt.Sprintf("mgr/%s/server_addr", mgrModule) cfgSetArgs = append(cfgSetArgs, cfgPath, opspec.ContainerEnvVarReference(podIPEnvVar)) - if cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { cfgSetArgs = append(cfgSetArgs, "--force") } cfgSetArgs = append(cfgSetArgs, "--verbose") diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 0f1c1ffe2..6aecdca65 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -36,6 +36,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" opspec "github.com/rook/rook/pkg/operator/ceph/spec" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -147,7 +148,7 @@ func New(context *clusterd.Context, namespace, dataDirHostPath string, hostNetwo } // Start begins the process of running a cluster of Ceph mons. -func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string, spec cephv1.ClusterSpec) (*cephconfig.ClusterInfo, error) { +func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string, cephVersion cephver.CephVersion, spec cephv1.ClusterSpec) (*cephconfig.ClusterInfo, error) { // Only one goroutine can orchestrate the mons at a time c.acquireOrchestrationLock() @@ -171,7 +172,7 @@ func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string, logger.Infof("start running mons") logger.Debugf("establishing ceph cluster info") - if err := c.initClusterInfo(); err != nil { + if err := c.initClusterInfo(cephVersion); err != nil { return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err) } @@ -252,11 +253,11 @@ func (c *Cluster) ensureMonsRunning(mons []*monConfig, i int, requireAllInQuorum // initClusterInfo retrieves the ceph cluster info if it already exists. // If a new cluster, create new keys. -func (c *Cluster) initClusterInfo() error { +func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion) error { var err error // get the cluster info from secret c.clusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, c.Namespace, &c.ownerRef) - c.clusterInfo.CephVersionName = c.spec.CephVersion.Name + c.clusterInfo.CephVersion = cephVersion if err != nil { return fmt.Errorf("failed to get cluster info. %+v", err) diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 1083a378b..47135e00a 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -35,6 +35,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" cephtest "github.com/rook/rook/pkg/daemon/ceph/test" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -146,13 +147,13 @@ func TestStartMonPods(t *testing.T) { c := newCluster(context, namespace, false, true, v1.ResourceRequirements{}) // start a basic cluster - _, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) validateStart(t, c) // starting again should be a no-op, but still results in an error - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) validateStart(t, c) @@ -165,7 +166,7 @@ func TestOperatorRestart(t *testing.T) { c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - info, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + info, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) assert.True(t, info.IsInitialized()) @@ -174,7 +175,7 @@ func TestOperatorRestart(t *testing.T) { c = newCluster(context, namespace, false, true, v1.ResourceRequirements{}) // starting again should be a no-op, but will not result in an error - info, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + info, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) assert.True(t, info.IsInitialized()) @@ -191,7 +192,7 @@ func TestOperatorRestartHostNetwork(t *testing.T) { c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - info, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + info, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) assert.True(t, info.IsInitialized()) @@ -201,7 +202,7 @@ func TestOperatorRestartHostNetwork(t *testing.T) { c = newCluster(context, namespace, true, false, v1.ResourceRequirements{}) // starting again should be a no-op, but still results in an error - info, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + info, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) assert.True(t, info.IsInitialized(), info) diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index dc4286860..38c5a6376 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -22,6 +22,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -180,7 +181,7 @@ func TestHostNetworkSameNode(t *testing.T) { c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Error(t, err) } @@ -198,7 +199,7 @@ func TestPodMemory(t *testing.T) { c := newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err := c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err := c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Error(t, err) // Test REQUEST == LIMIT @@ -214,7 +215,7 @@ func TestPodMemory(t *testing.T) { c = newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Error(t, err) // Test LIMIT != REQUEST but obviously LIMIT > REQUEST @@ -230,7 +231,7 @@ func TestPodMemory(t *testing.T) { c = newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Error(t, err) // Test valid case where pod resource is set approprietly @@ -246,7 +247,7 @@ func TestPodMemory(t *testing.T) { c = newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) // Test no resources were specified on the pod @@ -254,7 +255,7 @@ func TestPodMemory(t *testing.T) { c = newCluster(context, namespace, false, true, r) c.clusterInfo = test.CreateConfigDir(1) // start a basic cluster - _, err = c.Start(c.clusterInfo, c.rookVersion, c.spec) + _, err = c.Start(c.clusterInfo, c.rookVersion, cephver.Mimic, c.spec) assert.Nil(t, err) } diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index 58d3a46ab..8218d92cd 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -20,7 +20,6 @@ import ( "fmt" "strconv" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -53,7 +52,7 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { } // If deploying Nautilus or newer we need a new port for the monitor service - if cephv1.VersionAtLeast(c.spec.CephVersion.Name, cephv1.Nautilus) { + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { addServicePort(s, "msgr2", Msgr2port) } @@ -76,7 +75,7 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { // mon endpoint are not actually like, they remain with the mgrs1 format // however it's interesting to show that monitors can be addressed via 2 different ports // in the end the service has msgr1 and msgr2 ports configured so it's not entirely wrong - if cephv1.VersionAtLeast(c.spec.CephVersion.Name, cephv1.Nautilus) { + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Infof("mon %s endpoint are [v2:%s:%s,v1:%s:%d]", mon.DaemonName, s.Spec.ClusterIP, strconv.Itoa(int(Msgr2port)), s.Spec.ClusterIP, mon.Port) } else { logger.Infof("mon %s endpoint is %s:%d", mon.DaemonName, s.Spec.ClusterIP, mon.Port) diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 97d85a7bd..5c3b58f3c 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -185,7 +185,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { } // If deploying Nautilus and newer we need a new port of the monitor container - if cephv1.VersionAtLeast(c.spec.CephVersion.Name, cephv1.Nautilus) { + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { addContainerPort(container, "msgr2", 3300) } diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 9b714125c..a6707cbc6 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -36,7 +36,7 @@ import ( ) func TestPodContainer(t *testing.T) { - cluster := &Cluster{Namespace: "myosd", rookVersion: "23", cephVersion: cephv1.CephVersionSpec{Name: "mimic"}} + cluster := &Cluster{Namespace: "myosd", rookVersion: "23", cephVersion: cephv1.CephVersionSpec{}} c, err := cluster.provisionPodTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "node", "", v1.RestartPolicyAlways) assert.NotNil(t, c) assert.Nil(t, err) diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index 47651f51a..68c038250 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -26,7 +26,6 @@ import ( "strings" "github.com/go-ini/ini" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" cephutil "github.com/rook/rook/pkg/daemon/ceph/util" @@ -169,7 +168,7 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephconfig.ClusterInfo msgr2Endpoint := net.JoinHostPort(monIP, monPorts[0]) msgr1Endpoint := net.JoinHostPort(monIP, monPorts[1]) - if cephv1.VersionAtLeast(clusterInfo.CephVersionName, cephv1.Nautilus) { + if clusterInfo.CephVersion.IsAtLeastNautilus() { hosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]" } else { hosts[i] = msgr1Endpoint diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index 163b75b63..82921b5f3 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/model" "github.com/rook/rook/pkg/operator/ceph/file/mds" "github.com/rook/rook/pkg/operator/ceph/pool" + cephver "github.com/rook/rook/pkg/operator/ceph/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -63,7 +64,7 @@ func createFilesystem( dataPools = append(dataPools, p.ToModel("")) } f := newFS(fs.Name, fs.Spec.MetadataPool.ToModel(""), dataPools, fs.Spec.MetadataServer.ActiveCount) - if err := f.doFilesystemCreate(context, cephVersion.Name, fs.Namespace); err != nil { + if err := f.doFilesystemCreate(context, clusterInfo.CephVersion, fs.Namespace); err != nil { return fmt.Errorf("failed to create filesystem %s: %+v", fs.Name, err) } } @@ -74,7 +75,7 @@ func createFilesystem( } // As of Nautilus, allow_standby_replay is a fs property so we need to apply it - if cephv1.VersionAtLeast(clusterInfo.CephVersionName, cephv1.Nautilus) { + if clusterInfo.CephVersion.IsAtLeastNautilus() { if fs.Spec.MetadataServer.ActiveStandby { if err = client.AllowStandbyReplay(context, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveStandby); err != nil { return fmt.Errorf("failed to set allow_standby_replay to filesystem %s: %v", fs.Name, err) @@ -84,7 +85,7 @@ func createFilesystem( // set the number of active mds instances if fs.Spec.MetadataServer.ActiveCount > 1 { - if err = client.SetNumMDSRanks(context, cephVersion.Name, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil { + if err = client.SetNumMDSRanks(context, clusterInfo.CephVersion, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil { logger.Warningf("failed setting active mds count to %d. %+v", fs.Spec.MetadataServer.ActiveCount, err) } } @@ -159,13 +160,13 @@ func newFS(name string, metadataPool *model.Pool, dataPools []*model.Pool, activ } // doFilesystemCreate starts the Ceph file daemons and creates the filesystem in Ceph. -func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, cephVersionName, clusterName string) error { +func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, cephVersion cephver.CephVersion, clusterName string) error { _, err := client.GetFilesystem(context, clusterName, f.Name) if err == nil { logger.Infof("filesystem %s already exists", f.Name) // Even if the fs already exists, the num active mdses may have changed - if err := client.SetNumMDSRanks(context, cephVersionName, clusterName, f.Name, f.activeMDSCount); err != nil { + if err := client.SetNumMDSRanks(context, cephVersion, clusterName, f.Name, f.activeMDSCount); err != nil { logger.Errorf( fmt.Sprintf("failed to set num mds ranks (max_mds) to %d for filesystem %s, still continuing. ", f.activeMDSCount, f.Name) + "this error is not critical, but mdses may not be as failure tolerant as desired. " + diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index f1ca4832e..ac3ba7d63 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -29,6 +29,7 @@ import ( cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -105,7 +106,7 @@ func (c *Cluster) Start() error { var fsPreparedForUpgrade = false defer func() { if fsPreparedForUpgrade { - if err := finishedWithDaemonUpgrade(c.context, c.cephVersion.Name, c.fs.Namespace, c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount); err != nil { + if err := finishedWithDaemonUpgrade(c.context, c.clusterInfo.CephVersion, c.fs.Namespace, c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount); err != nil { logger.Errorf("for filesystem %s, USER should make sure the Ceph fs max_mds property is set to %d: %+v", c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount, err) } @@ -238,7 +239,8 @@ func DeleteCluster(context *clusterd.Context, namespace, fsName string) error { // Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/ func prepareForDaemonUpgrade( context *clusterd.Context, - cephVersionName, clusterName, fsName string, + cephVersion cephver.CephVersion, + clusterName, fsName string, timeout time.Duration, ) error { logger.Infof("preparing filesystem %s for daemon upgrade", fsName) @@ -247,7 +249,7 @@ func prepareForDaemonUpgrade( // See more: https://ceph.com/releases/v13-2-0-mimic-released/ // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ // As of Oct. 2018, this is only necessary for Luminous and Mimic. - if err := client.SetNumMDSRanks(context, cephVersionName, clusterName, fsName, 1); err != nil { + if err := client.SetNumMDSRanks(context, cephVersion, clusterName, fsName, 1); err != nil { return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err) } if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil { @@ -263,7 +265,8 @@ func prepareForDaemonUpgrade( // ideal state following an upgrade of its daemon(s). func finishedWithDaemonUpgrade( context *clusterd.Context, - cephVersionName, clusterName, fsName string, + cephVersion cephver.CephVersion, + clusterName, fsName string, activeMDSCount int32, ) error { logger.Debugf("restoring filesystem %s from daemon upgrade", fsName) @@ -273,7 +276,7 @@ func finishedWithDaemonUpgrade( // See more: https://ceph.com/releases/v13-2-0-mimic-released/ // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ // TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic. - if err := client.SetNumMDSRanks(context, cephVersionName, clusterName, fsName, activeMDSCount); err != nil { + if err := client.SetNumMDSRanks(context, cephVersion, clusterName, fsName, activeMDSCount); err != nil { return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err) } // * End of noted section 1 return nil diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index b00c13f24..f86c7b65f 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -20,7 +20,6 @@ import ( "fmt" "strconv" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" @@ -89,7 +88,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { ) // These flags are obsoleted as of Nautilus - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { args = append( args, config.NewFlag("mds-standby-for-fscid", c.fsID), diff --git a/pkg/operator/ceph/file/mds/spec_test.go b/pkg/operator/ceph/file/mds/spec_test.go index 6241c9d58..97b7966e6 100644 --- a/pkg/operator/ceph/file/mds/spec_test.go +++ b/pkg/operator/ceph/file/mds/spec_test.go @@ -25,6 +25,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + cephver "github.com/rook/rook/pkg/operator/ceph/version" cephtest "github.com/rook/rook/pkg/operator/ceph/test" testop "github.com/rook/rook/pkg/operator/test" @@ -55,13 +56,16 @@ func testDeploymentObject(hostNetwork bool) *apps.Deployment { }, }, } - clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} + clusterInfo := &cephconfig.ClusterInfo{ + FSID: "myfsid", + CephVersion: cephver.Mimic, + } c := NewCluster( clusterInfo, &clusterd.Context{Clientset: testop.New(1)}, "rook/rook:myversion", - cephv1.CephVersionSpec{Image: "ceph/ceph:testversion", Name: "mimic"}, + cephv1.CephVersionSpec{Image: "ceph/ceph:testversion"}, hostNetwork, fs, &client.CephFilesystemDetails{ID: 15}, diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index 498b0dfa5..2b062e738 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -24,6 +24,7 @@ import ( opkit "github.com/rook/operator-kit" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" @@ -43,6 +44,7 @@ var CephNFSResource = opkit.CustomResource{ // NFSCephNFSController represents a controller for NFS custom resources type CephNFSController struct { + clusterInfo *cephconfig.ClusterInfo context *clusterd.Context rookImage string cephVersion cephv1.CephVersionSpec @@ -51,8 +53,9 @@ type CephNFSController struct { } // NewNFSCephNFSController create controller for watching NFS custom resources created -func NewCephNFSController(context *clusterd.Context, rookImage string, cephVersion cephv1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *CephNFSController { +func NewCephNFSController(clusterInfo *cephconfig.ClusterInfo, context *clusterd.Context, rookImage string, cephVersion cephv1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *CephNFSController { return &CephNFSController{ + clusterInfo: clusterInfo, context: context, rookImage: rookImage, cephVersion: cephVersion, @@ -79,7 +82,7 @@ func (c *CephNFSController) StartWatch(namespace string, stopCh chan struct{}) e func (c *CephNFSController) onAdd(obj interface{}) { nfs := obj.(*cephv1.CephNFS).DeepCopy() - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Errorf("Ceph NFS is only supported with Nautilus or newer. CRD %s will be ignored.", nfs.Name) return } @@ -93,7 +96,7 @@ func (c *CephNFSController) onAdd(obj interface{}) { func (c *CephNFSController) onUpdate(oldObj, newObj interface{}) { oldNFS := oldObj.(*cephv1.CephNFS).DeepCopy() newNFS := newObj.(*cephv1.CephNFS).DeepCopy() - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Errorf("Ceph NFS is only supported with Nautilus or newer. CRD %s will be ignored.", newNFS.Name) return } @@ -120,7 +123,7 @@ func (c *CephNFSController) onUpdate(oldObj, newObj interface{}) { func (c *CephNFSController) onDelete(obj interface{}) { nfs := obj.(*cephv1.CephNFS).DeepCopy() - if !cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) { + if !c.clusterInfo.CephVersion.IsAtLeastNautilus() { logger.Errorf("Ceph NFS is only supported with Nautilus or newer. CRD %s cleanup will be ignored.", nfs.Name) return } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 968ea3859..412df4266 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -23,6 +23,7 @@ import ( "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" cephtest "github.com/rook/rook/pkg/operator/ceph/test" + cephver "github.com/rook/rook/pkg/operator/ceph/version" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -43,13 +44,14 @@ func TestPodSpecs(t *testing.T) { }, } info := testop.CreateConfigDir(1) + info.CephVersion = cephver.Mimic data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default") c := &clusterConfig{ clusterInfo: info, store: store, rookVersion: "rook/rook:myversion", - cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, + cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1"}, hostNetwork: true, DataPathMap: data, } @@ -74,6 +76,7 @@ func TestSSLPodSpec(t *testing.T) { }, } info := testop.CreateConfigDir(1) + info.CephVersion = cephver.Mimic data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default") store.Spec.Gateway.SSLCertificateRef = "mycert" store.Spec.Gateway.SecurePort = 443 @@ -82,7 +85,7 @@ func TestSSLPodSpec(t *testing.T) { clusterInfo: info, store: store, rookVersion: "rook/rook:myversion", - cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, + cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1"}, hostNetwork: true, DataPathMap: data, } diff --git a/pkg/operator/ceph/version/version.go b/pkg/operator/ceph/version/version.go index 85bcbacab..589a473e7 100644 --- a/pkg/operator/ceph/version/version.go +++ b/pkg/operator/ceph/version/version.go @@ -90,7 +90,7 @@ func (v *CephVersion) IsLuminous() bool { return v.isRelease(Luminous) } -func (v *CephVersion) AtLeast(other CephVersion) bool { +func (v *CephVersion) IsAtLeast(other CephVersion) bool { if v.Major > other.Major { return true } else if v.Major < other.Major { @@ -109,10 +109,10 @@ func (v *CephVersion) AtLeast(other CephVersion) bool { return true } -func (v *CephVersion) AtLeastNautilus() bool { - return v.AtLeast(Nautilus) +func (v *CephVersion) IsAtLeastNautilus() bool { + return v.IsAtLeast(Nautilus) } -func (v *CephVersion) AtLeastMimic() bool { - return v.AtLeast(Mimic) +func (v *CephVersion) IsAtLeastMimic() bool { + return v.IsAtLeast(Mimic) } diff --git a/pkg/operator/ceph/version/version_test.go b/pkg/operator/ceph/version/version_test.go index d784b6f60..535f74ac0 100644 --- a/pkg/operator/ceph/version/version_test.go +++ b/pkg/operator/ceph/version/version_test.go @@ -113,30 +113,30 @@ func TestIsReleaseX(t *testing.T) { } func TestVersionAtLeast(t *testing.T) { - assert.True(t, Luminous.AtLeast(Luminous)) - assert.False(t, Luminous.AtLeast(Mimic)) - assert.False(t, Luminous.AtLeast(Nautilus)) - assert.True(t, Mimic.AtLeast(Luminous)) - assert.True(t, Mimic.AtLeast(Mimic)) - assert.False(t, Mimic.AtLeast(Nautilus)) - assert.True(t, Nautilus.AtLeast(Luminous)) - assert.True(t, Nautilus.AtLeast(Mimic)) - assert.True(t, Nautilus.AtLeast(Nautilus)) + assert.True(t, Luminous.IsAtLeast(Luminous)) + assert.False(t, Luminous.IsAtLeast(Mimic)) + assert.False(t, Luminous.IsAtLeast(Nautilus)) + assert.True(t, Mimic.IsAtLeast(Luminous)) + assert.True(t, Mimic.IsAtLeast(Mimic)) + assert.False(t, Mimic.IsAtLeast(Nautilus)) + assert.True(t, Nautilus.IsAtLeast(Luminous)) + assert.True(t, Nautilus.IsAtLeast(Mimic)) + assert.True(t, Nautilus.IsAtLeast(Nautilus)) - assert.True(t, (&CephVersion{1, 0, 0}).AtLeast(CephVersion{0, 0, 0})) - assert.False(t, (&CephVersion{0, 0, 0}).AtLeast(CephVersion{1, 0, 0})) - assert.True(t, (&CephVersion{1, 1, 0}).AtLeast(CephVersion{1, 0, 0})) - assert.False(t, (&CephVersion{1, 0, 0}).AtLeast(CephVersion{1, 1, 0})) - assert.True(t, (&CephVersion{1, 1, 1}).AtLeast(CephVersion{1, 1, 0})) - assert.False(t, (&CephVersion{1, 1, 0}).AtLeast(CephVersion{1, 1, 1})) - assert.True(t, (&CephVersion{1, 1, 1}).AtLeast(CephVersion{1, 1, 1})) + assert.True(t, (&CephVersion{1, 0, 0}).IsAtLeast(CephVersion{0, 0, 0})) + assert.False(t, (&CephVersion{0, 0, 0}).IsAtLeast(CephVersion{1, 0, 0})) + assert.True(t, (&CephVersion{1, 1, 0}).IsAtLeast(CephVersion{1, 0, 0})) + assert.False(t, (&CephVersion{1, 0, 0}).IsAtLeast(CephVersion{1, 1, 0})) + assert.True(t, (&CephVersion{1, 1, 1}).IsAtLeast(CephVersion{1, 1, 0})) + assert.False(t, (&CephVersion{1, 1, 0}).IsAtLeast(CephVersion{1, 1, 1})) + assert.True(t, (&CephVersion{1, 1, 1}).IsAtLeast(CephVersion{1, 1, 1})) } func TestVersionAtLeastX(t *testing.T) { - assert.True(t, Nautilus.AtLeastNautilus()) - assert.True(t, Nautilus.AtLeastMimic()) - assert.True(t, Mimic.AtLeastMimic()) - assert.False(t, Luminous.AtLeastMimic()) - assert.False(t, Luminous.AtLeastNautilus()) - assert.False(t, Mimic.AtLeastNautilus()) + assert.True(t, Nautilus.IsAtLeastNautilus()) + assert.True(t, Nautilus.IsAtLeastMimic()) + assert.True(t, Mimic.IsAtLeastMimic()) + assert.False(t, Luminous.IsAtLeastMimic()) + assert.False(t, Luminous.IsAtLeastNautilus()) + assert.False(t, Mimic.IsAtLeastNautilus()) } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 5e6575040..924debdc7 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -48,8 +48,8 @@ const ( ) var ( - LuminousVersion = cephv1.CephVersionSpec{Image: luminousTestImage, Name: cephv1.Luminous} - MimicVersion = cephv1.CephVersionSpec{Image: mimicTestImage, Name: cephv1.Mimic} + LuminousVersion = cephv1.CephVersionSpec{Image: luminousTestImage} + MimicVersion = cephv1.CephVersionSpec{Image: mimicTestImage} ) // CephInstaller wraps installing and uninstalling rook on a platform @@ -474,7 +474,7 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rook panic("failed to get kubectl client :" + err.Error()) } logger.Infof("Rook Version: %s", rookVersion) - logger.Infof("Ceph Version: %s (%s)", cephVersion.Image, cephVersion.Name) + logger.Infof("Ceph Version: %s", cephVersion.Image) h := &CephInstaller{ Manifests: NewCephManifests(rookVersion), k8shelper: k8shelp, diff --git a/tests/longhaul/base_block_test.go b/tests/longhaul/base_block_test.go index 853ba018f..acb922579 100644 --- a/tests/longhaul/base_block_test.go +++ b/tests/longhaul/base_block_test.go @@ -145,7 +145,7 @@ func StartLoadTestCluster(t func() *testing.T, namespace string) (LoadTestCluste kh, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster, cephv1.CephVersionSpec{Image: "ceph/ceph:v12.2.7", Name: "luminous"}) + i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster, cephv1.CephVersionSpec{Image: "ceph/ceph:v12.2.7"}) op := LoadTestCluster{i, kh, nil, t, namespace} op.Setup()