From 05d4c2776cf002afa057f1a9f679c0fe9cdeb955 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Fri, 26 Mar 2021 10:18:01 +0530 Subject: [PATCH] ceph: update cephCluster CR with ceph versions output update cephCluster CR with ceph versions output. this output will contain ceph version of the ceph daemons which will help with upgrade status. ceph versions command output ``` ceph versions { "mon": { "ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable)": 3 }, "mgr": { "ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable)": 1 }, "osd": { "ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable)": 1 }, "mds": { "ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable)": 2 }, "rbd-mirror": { "ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable)": 1 }, "rgw": { "ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable)": 1 }, "overall": { "ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable)": 9 } } ``` CR status ``` status: ceph: --- versions: mds: ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable): 2 mgr: ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable): 1 mon: ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable): 3 osd: ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable): 1 overall: ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable): 9 rbd-mirror: ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable): 1 rgw: ceph version 15.2.9 (357616cbf726abb779ca75a551e8d02568e15b17) octopus (stable): 1 ``` Signed-off-by: subhamkrai --- .../charts/rook-ceph/templates/resources.yaml | 44 +++++++++++ cluster/examples/kubernetes/ceph/crds.yaml | 45 +++++++++++ pkg/apis/ceph.rook.io/v1/types.go | 32 +++++++- .../ceph.rook.io/v1/zz_generated.deepcopy.go | 77 +++++++++++++++++++ pkg/daemon/ceph/client/upgrade.go | 18 +---- pkg/daemon/ceph/client/upgrade_test.go | 3 +- pkg/operator/ceph/cluster/cephstatus.go | 9 +++ pkg/operator/ceph/cluster/version.go | 2 +- pkg/operator/ceph/cluster/version_test.go | 11 ++- .../ceph/controller/controller_utils.go | 4 +- 10 files changed, 220 insertions(+), 25 deletions(-) diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index ff5058fcc..94e6852c5 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -4044,6 +4044,50 @@ spec: type: string previousHealth: type: string + versions: + description: CephDaemonsVersions show the current ceph version for different ceph daemons + properties: + cephfs-mirror: + additionalProperties: + type: integer + description: CephFSMirror shows CephFSMirror Ceph version + type: object + mds: + additionalProperties: + type: integer + description: Mds shows Mds Ceph version + type: object + mgr: + additionalProperties: + type: integer + description: Mgr shows Mgr Ceph version + type: object + mon: + additionalProperties: + type: integer + description: Mon shows Mon Ceph version + type: object + osd: + additionalProperties: + type: integer + description: Osd shows Osd Ceph version + type: object + overall: + additionalProperties: + type: integer + description: Overall shows overall Ceph version + type: object + rbd-mirror: + additionalProperties: + type: integer + description: RbdMirror shows RbdMirror Ceph version + type: object + rgw: + additionalProperties: + type: integer + description: Rgw shows Rgw Ceph version + type: object + type: object type: object conditions: items: diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 68cc796e7..7e6f4613c 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -6564,6 +6564,51 @@ spec: type: string previousHealth: type: string + versions: + description: CephDaemonsVersions show the current ceph version + for different ceph daemons + properties: + cephfs-mirror: + additionalProperties: + type: integer + description: CephFSMirror shows CephFSMirror Ceph version + type: object + mds: + additionalProperties: + type: integer + description: Mds shows Mds Ceph version + type: object + mgr: + additionalProperties: + type: integer + description: Mgr shows Mgr Ceph version + type: object + mon: + additionalProperties: + type: integer + description: Mon shows Mon Ceph version + type: object + osd: + additionalProperties: + type: integer + description: Osd shows Osd Ceph version + type: object + overall: + additionalProperties: + type: integer + description: Overall shows overall Ceph version + type: object + rbd-mirror: + additionalProperties: + type: integer + description: RbdMirror shows RbdMirror Ceph version + type: object + rgw: + additionalProperties: + type: integer + description: Rgw shows Rgw Ceph version + type: object + type: object type: object conditions: items: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 55831c251..9fba5c775 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at -    http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -299,6 +299,34 @@ type ClusterStatus struct { CephVersion *ClusterVersion `json:"version,omitempty"` } +// CephDaemonsVersions show the current ceph version for different ceph daemons +type CephDaemonsVersions struct { + // Mon shows Mon Ceph version + // +optional + Mon map[string]int `json:"mon,omitempty"` + // Mgr shows Mgr Ceph version + // +optional + Mgr map[string]int `json:"mgr,omitempty"` + // Osd shows Osd Ceph version + // +optional + Osd map[string]int `json:"osd,omitempty"` + // Rgw shows Rgw Ceph version + // +optional + Rgw map[string]int `json:"rgw,omitempty"` + // Mds shows Mds Ceph version + // +optional + Mds map[string]int `json:"mds,omitempty"` + // RbdMirror shows RbdMirror Ceph version + // +optional + RbdMirror map[string]int `json:"rbd-mirror,omitempty"` + // CephFSMirror shows CephFSMirror Ceph version + // +optional + CephFSMirror map[string]int `json:"cephfs-mirror,omitempty"` + // Overall shows overall Ceph version + // +optional + Overall map[string]int `json:"overall,omitempty"` +} + // CephStatus is the details health of a Ceph Cluster type CephStatus struct { Health string `json:"health,omitempty"` @@ -307,6 +335,8 @@ type CephStatus struct { LastChanged string `json:"lastChanged,omitempty"` PreviousHealth string `json:"previousHealth,omitempty"` Capacity Capacity `json:"capacity,omitempty"` + // +optional + Versions *CephDaemonsVersions `json:"versions,omitempty"` } // Capacity is the capacity information of a Ceph Cluster diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 883b5733a..22b9d2f4b 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -364,6 +364,78 @@ func (in *CephClusterList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CephDaemonsVersions) DeepCopyInto(out *CephDaemonsVersions) { + *out = *in + if in.Mon != nil { + in, out := &in.Mon, &out.Mon + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Mgr != nil { + in, out := &in.Mgr, &out.Mgr + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Osd != nil { + in, out := &in.Osd, &out.Osd + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Rgw != nil { + in, out := &in.Rgw, &out.Rgw + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Mds != nil { + in, out := &in.Mds, &out.Mds + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.RbdMirror != nil { + in, out := &in.RbdMirror, &out.RbdMirror + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.CephFSMirror != nil { + in, out := &in.CephFSMirror, &out.CephFSMirror + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Overall != nil { + in, out := &in.Overall, &out.Overall + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephDaemonsVersions. +func (in *CephDaemonsVersions) DeepCopy() *CephDaemonsVersions { + if in == nil { + return nil + } + out := new(CephDaemonsVersions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CephFilesystem) DeepCopyInto(out *CephFilesystem) { *out = *in @@ -976,6 +1048,11 @@ func (in *CephStatus) DeepCopyInto(out *CephStatus) { } } out.Capacity = in.Capacity + if in.Versions != nil { + in, out := &in.Versions, &out.Versions + *out = new(CephDaemonsVersions) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go index fcef5dce5..5e0af55b7 100644 --- a/pkg/daemon/ceph/client/upgrade.go +++ b/pkg/daemon/ceph/client/upgrade.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/util" @@ -33,17 +34,6 @@ const ( defaultOSDRetryDelay = 10 * time.Second ) -// CephDaemonsVersions is a structure that can be used to parsed the output of the 'ceph versions' command -type CephDaemonsVersions struct { - Mon map[string]int `json:"mon,omitempty"` - Mgr map[string]int `json:"mgr,omitempty"` - Osd map[string]int `json:"osd,omitempty"` - Rgw map[string]int `json:"rgw,omitempty"` - Mds map[string]int `json:"mds,omitempty"` - RbdMirror map[string]int `json:"rbd-mirror,omitempty"` - Overall map[string]int `json:"overall,omitempty"` -} - var ( // we don't perform any checks on these daemons // they don't have any "ok-to-stop" command implemented @@ -91,14 +81,14 @@ func GetCephMonVersion(context *clusterd.Context, clusterInfo *ClusterInfo) (*ce } // GetAllCephDaemonVersions reports the Ceph version of each daemon in the cluster -func GetAllCephDaemonVersions(context *clusterd.Context, clusterInfo *ClusterInfo) (*CephDaemonsVersions, error) { +func GetAllCephDaemonVersions(context *clusterd.Context, clusterInfo *ClusterInfo) (*cephv1.CephDaemonsVersions, error) { output, err := getAllCephDaemonVersionsString(context, clusterInfo) if err != nil { return nil, err } logger.Debug(output) - var cephVersionsResult CephDaemonsVersions + var cephVersionsResult cephv1.CephDaemonsVersions err = json.Unmarshal([]byte(output), &cephVersionsResult) if err != nil { return nil, errors.Wrap(err, "failed to retrieve ceph versions results") @@ -284,7 +274,7 @@ func findFSName(deployment string) string { return strings.TrimPrefix(deployment, "rook-ceph-mds-") } -func daemonMapEntry(versions *CephDaemonsVersions, daemonType string) (map[string]int, error) { +func daemonMapEntry(versions *cephv1.CephDaemonsVersions, daemonType string) (map[string]int, error) { switch daemonType { case "mon": return versions.Mon, nil diff --git a/pkg/daemon/ceph/client/upgrade_test.go b/pkg/daemon/ceph/client/upgrade_test.go index e6aaf1518..6491b50c2 100644 --- a/pkg/daemon/ceph/client/upgrade_test.go +++ b/pkg/daemon/ceph/client/upgrade_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -145,7 +146,7 @@ func TestDaemonMapEntry(t *testing.T) { } }`) - var dummyVersions CephDaemonsVersions + var dummyVersions cephv1.CephDaemonsVersions err := json.Unmarshal([]byte(dummyVersionsRaw), &dummyVersions) assert.NoError(t, err) diff --git a/pkg/operator/ceph/cluster/cephstatus.go b/pkg/operator/ceph/cluster/cephstatus.go index c8b0508af..d679979ae 100644 --- a/pkg/operator/ceph/cluster/cephstatus.go +++ b/pkg/operator/ceph/cluster/cephstatus.go @@ -165,6 +165,15 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus, cond // Update with Ceph Status cephCluster.Status.CephStatus = toCustomResourceStatus(cephCluster.Status, status) + // versions store the ceph version of all the ceph daemons and overall cluster version + versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.clusterInfo) + if err != nil { + logger.Errorf("failed to get ceph daemons versions. %v", err) + } else { + // Update status with Ceph versions + cephCluster.Status.CephStatus.Versions = versions + } + // Update condition logger.Debugf("updating ceph cluster %q status and condition to %+v, %v, %s, %s", clusterName.Namespace, status, conditionStatus, reason, message) opcontroller.UpdateClusterCondition(c.context, cephCluster, c.clusterInfo.NamespacedName(), condition, conditionStatus, reason, message, true) diff --git a/pkg/operator/ceph/cluster/version.go b/pkg/operator/ceph/cluster/version.go index b95ffb96f..297742c29 100644 --- a/pkg/operator/ceph/cluster/version.go +++ b/pkg/operator/ceph/cluster/version.go @@ -72,7 +72,7 @@ func (c *cluster) printOverallCephVersion() { // This function compare the Ceph spec image and the cluster running version // It returns true if the image is different and false if identical -func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, runningVersions client.CephDaemonsVersions) (bool, error) { +func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, runningVersions cephv1.CephDaemonsVersions) (bool, error) { numberOfCephVersions := len(runningVersions.Overall) if numberOfCephVersions == 0 { // let's return immediately diff --git a/pkg/operator/ceph/cluster/version_test.go b/pkg/operator/ceph/cluster/version_test.go index 20f43cf86..de274c4c1 100755 --- a/pkg/operator/ceph/cluster/version_test.go +++ b/pkg/operator/ceph/cluster/version_test.go @@ -23,7 +23,6 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" @@ -40,7 +39,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { "ceph version 14.2.0 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) nautilus (stable)": 2 } }`) - var dummyRunningVersions client.CephDaemonsVersions + var dummyRunningVersions cephv1.CephDaemonsVersions err := json.Unmarshal([]byte(fakeRunningVersions), &dummyRunningVersions) assert.NoError(t, err) @@ -56,7 +55,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { "ceph version 14.2.0 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) nautilus (stable)": 2 } }`) - var dummyRunningVersions2 client.CephDaemonsVersions + var dummyRunningVersions2 cephv1.CephDaemonsVersions err = json.Unmarshal([]byte(fakeRunningVersions), &dummyRunningVersions2) assert.NoError(t, err) @@ -71,7 +70,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { "ceph version 15.2.0 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) octopus (stable)": 2 } }`) - var dummyRunningVersions3 client.CephDaemonsVersions + var dummyRunningVersions3 cephv1.CephDaemonsVersions err = json.Unmarshal([]byte(fakeRunningVersions), &dummyRunningVersions3) assert.NoError(t, err) @@ -87,7 +86,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { "ceph version 13.2.0 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) mimic (stable)": 2 } }`) - var dummyRunningVersions4 client.CephDaemonsVersions + var dummyRunningVersions4 cephv1.CephDaemonsVersions err = json.Unmarshal([]byte(fakeRunningVersions), &dummyRunningVersions4) assert.NoError(t, err) @@ -103,7 +102,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { "ceph version 14.2.2 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) nautilus (stable)": 2 } }`) - var dummyRunningVersions5 client.CephDaemonsVersions + var dummyRunningVersions5 cephv1.CephDaemonsVersions err = json.Unmarshal([]byte(fakeRunningVersions), &dummyRunningVersions5) assert.NoError(t, err) diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index f3d6feb96..e749f5382 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -97,7 +97,7 @@ func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, control var allowedErrStatus = []string{"MDS_ALL_DOWN"} var ignoreHealthErr = len(healthErrKeys) == 1 && contains(allowedErrStatus, healthErrKeys[0]) if ignoreHealthErr { - logger.Debugf("%q: ignoring ceph status %q because only cause is %q (full status is %q)", controllerName, cephCluster.Status.CephStatus.Health, healthErrKeys[0], cephCluster.Status.CephStatus) + logger.Debugf("%q: ignoring ceph status %q because only cause is %q (full status is %+v)", controllerName, cephCluster.Status.CephStatus.Health, healthErrKeys[0], cephCluster.Status.CephStatus) } return ignoreHealthErr } @@ -138,7 +138,7 @@ func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, names if ok && len(details) == 1 && strings.Contains(message.Message, "Error initializing cluster client") { logger.Infof("%s: skipping reconcile since operator is still initializing", controllerName) } else { - logger.Infof("%s: CephCluster %q found but skipping reconcile since ceph health is %q", controllerName, cephCluster.Name, cephCluster.Status.CephStatus) + logger.Infof("%s: CephCluster %q found but skipping reconcile since ceph health is %+v", controllerName, cephCluster.Name, cephCluster.Status.CephStatus) } }