diff --git a/pkg/daemon/ceph/client/status.go b/pkg/daemon/ceph/client/status.go index d75f16672..7d765e698 100644 --- a/pkg/daemon/ceph/client/status.go +++ b/pkg/daemon/ceph/client/status.go @@ -52,6 +52,7 @@ type CephStatus struct { } `json:"osdmap"` PgMap PgMap `json:"pgmap"` MgrMap MgrMap `json:"mgrmap"` + Fsmap Fsmap `json:"fsmap"` } type HealthStatus struct { @@ -123,6 +124,23 @@ type PgStateEntry struct { Count int `json:"count"` } +// Fsmap is a struct representing the filesystem map +type Fsmap struct { + Epoch int `json:"epoch"` + ID int `json:"id"` + Up int `json:"up"` + In int `json:"in"` + Max int `json:"max"` + ByRank []struct { + FilesystemID int `json:"filesystem_id"` + Rank int `json:"rank"` + Name string `json:"name"` + Status string `json:"status"` + Gid int `json:"gid"` + } `json:"by_rank"` + UpStandby int `json:"up:standby"` +} + func Status(context *clusterd.Context, clusterName string, debug bool) (CephStatus, error) { args := []string{"status"} cmd := NewCephCommand(context, clusterName, args) @@ -171,3 +189,50 @@ func isClusterClean(status CephStatus) error { return fmt.Errorf("cluster is not fully clean. PGs: %+v", status.PgMap.PgsByState) } + +// getMDSRank returns the rank of a given MDS +func getMDSRank(status CephStatus, clusterName, fsName string) (int, error) { + // dummy rank + mdsRank := -1000 + for r := range status.Fsmap.ByRank { + if status.Fsmap.ByRank[r].Name == fsName { + mdsRank = r + } + } + // if the mds is not shown in the map one reason might be because it's in standby + // if not in standby there is something else going wron + if mdsRank < 0 && status.Fsmap.UpStandby < 1 { + // it might seem strange to log an error since this could be a warning too + // it is a warning until we reach the timeout, this should give enough time to the mds to transtion its state + // after the timeout we consider that the mds might be gone or the timeout was not long enough... + return mdsRank, fmt.Errorf("mds %s not found in fsmap, this likely means mdss are transitioning between active and standby states", fsName) + } + + return mdsRank, nil +} + +// MdsActiveOrStandbyReplay returns wether a given MDS is active or in standby +func MdsActiveOrStandbyReplay(context *clusterd.Context, clusterName, fsName string) error { + status, err := Status(context, clusterName, false) + if err != nil { + return fmt.Errorf("failed to get ceph status. %+v", err) + } + + mdsRank, err := getMDSRank(status, clusterName, fsName) + if err != nil { + return fmt.Errorf("%+v", err) + } + + // this MDS is in standby so let's return immediatly + if mdsRank < 0 { + logger.Infof("mds %s is in standby, nothing to check", fsName) + return nil + } + + if status.Fsmap.ByRank[mdsRank].Status == "up:active" || status.Fsmap.ByRank[mdsRank].Status == "up:standby-replay" || status.Fsmap.ByRank[mdsRank].Status == "up:standby" { + logger.Infof("mds %s is %s", fsName, status.Fsmap.ByRank[mdsRank].Status) + return nil + } + + return fmt.Errorf("mds %s is %s, bad state", fsName, status.Fsmap.ByRank[mdsRank].Status) +} diff --git a/pkg/daemon/ceph/client/status_test.go b/pkg/daemon/ceph/client/status_test.go index d544ab3b1..2a06e8a3c 100644 --- a/pkg/daemon/ceph/client/status_test.go +++ b/pkg/daemon/ceph/client/status_test.go @@ -27,6 +27,30 @@ const ( CephStatusResponseRaw = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` ) +var ( + // this JSON was generated from `ceph status -f json`, using Ceph Nautilus 14.2.1 + // It was chopped to only show what the tests are looking for + statusFakeRaw = []byte(`{ + "fsmap": { + "epoch": 13, + "id": 1, + "up": 1, + "in": 1, + "max": 1, + "by_rank": [ + { + "filesystem_id": 1, + "rank": 0, + "name": "myfs-b", + "status": "up:active", + "gid": 14716 + } + ], + "up:standby": 1 + } + }`) +) + func TestStatusMarshal(t *testing.T) { var status CephStatus err := json.Unmarshal([]byte(CephStatusResponseRaw), &status) @@ -85,3 +109,12 @@ func TestIsClusterClean(t *testing.T) { err = isClusterClean(status) assert.NotNil(t, err) } + +func TestGetMDSRank(t *testing.T) { + var statusFake CephStatus + json.Unmarshal(statusFakeRaw, &statusFake) + + mdsRankFake, err := getMDSRank(statusFake, "rook-ceph", "myfs-b") + assert.Nil(t, err) + assert.Equal(t, 0, mdsRankFake) +} diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go index 9208f9f70..6c6abb2d1 100644 --- a/pkg/daemon/ceph/client/upgrade.go +++ b/pkg/daemon/ceph/client/upgrade.go @@ -18,35 +18,72 @@ package client import ( "encoding/json" + "errors" "fmt" + "strings" + "time" "github.com/rook/rook/pkg/clusterd" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/util" ) // CephDaemonsVersions is a structure that can be used to parsed the output of the 'ceph versions' command type CephDaemonsVersions struct { - Mon map[string]int `json:"mon,omitempty"` - Osd map[string]int `json:"osd,omitempty"` - Mgr map[string]int `json:"mgr,omitempty"` - Mds map[string]int `json:"mds,omitempty"` - Overall map[string]int `json:"overall,omitempty"` + Mon map[string]int `json:"mon,omitempty"` + Mgr map[string]int `json:"mgr,omitempty"` + Osd map[string]int `json:"osd,omitempty"` + Rgw map[string]int `json:"rgw,omitempty"` + Mds map[string]int `json:"mds,omitempty"` + RbdMirror map[string]int `json:"rbd-mirror,omitempty"` + Overall map[string]int `json:"overall,omitempty"` } -func getCephMonVersionString(context *clusterd.Context) (string, error) { - output, err := context.Executor.ExecuteCommandWithOutput(false, "", "ceph", "version") +var ( + // we don't perform any checks on these daemons + // they don't have any "ok-to-stop" command implemented + daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs"} +) + +func getCephMonVersionString(context *clusterd.Context, clusterName string) (string, error) { + args := []string{"version"} + command, args := FinalizeCephCommandArgs("ceph", args, context.ConfigDir, clusterName) + + output, err := context.Executor.ExecuteCommandWithOutput(false, "", command, args...) if err != nil { - return "", fmt.Errorf("failed to run ceph version: %+v", err) + return "", fmt.Errorf("failed to run 'ceph version'. %+v", err) } logger.Debug(output) return output, nil } -func getCephVersionsString(context *clusterd.Context) (string, error) { - output, err := context.Executor.ExecuteCommandWithOutput(false, "", "ceph", "versions") +func getAllCephDaemonVersionsString(context *clusterd.Context, clusterName string) (string, error) { + args := []string{"versions"} + command, args := FinalizeCephCommandArgs("ceph", args, context.ConfigDir, clusterName) + + output, err := context.Executor.ExecuteCommandWithOutput(false, "", command, args...) if err != nil { - return "", fmt.Errorf("failed to run ceph versions: %+v", err) + return "", fmt.Errorf("failed to run 'ceph versions'. %+v", err) + } + logger.Debug(output) + + return output, nil +} + +func getCephDaemonVersionString(context *clusterd.Context, deployment, clusterName string) (string, error) { + daemonName, err := findDaemonName(deployment) + if err != nil { + return "", fmt.Errorf("%+v", err) + } + daemonID := findDaemonID(deployment) + daemon := daemonName + "." + daemonID + + args := []string{"tell", daemon, "version"} + command, args := FinalizeCephCommandArgs("ceph", args, context.ConfigDir, clusterName) + output, err := context.Executor.ExecuteCommandWithOutput(false, "", command, args...) + if err != nil { + return "", fmt.Errorf("failed to run ceph tell. %+v", err) } logger.Debug(output) @@ -54,10 +91,10 @@ func getCephVersionsString(context *clusterd.Context) (string, error) { } // GetCephMonVersion reports the Ceph version of all the monitors, or at least a majority with quorum -func GetCephMonVersion(context *clusterd.Context) (*cephver.CephVersion, error) { - output, err := getCephMonVersionString(context) +func GetCephMonVersion(context *clusterd.Context, clusterName string) (*cephver.CephVersion, error) { + output, err := getCephMonVersionString(context, clusterName) if err != nil { - return nil, fmt.Errorf("failed to run ceph version: %+v", err) + return nil, err } logger.Debug(output) @@ -69,11 +106,27 @@ func GetCephMonVersion(context *clusterd.Context) (*cephver.CephVersion, error) return v, nil } -// GetCephVersions reports the Ceph version of each daemon in the cluster -func GetCephVersions(context *clusterd.Context) (*CephDaemonsVersions, error) { - output, err := getCephVersionsString(context) +// GetCephDaemonVersion reports the Ceph version of a particular daemon +func GetCephDaemonVersion(context *clusterd.Context, deployment, clusterName string) (*cephver.CephVersion, error) { + output, err := getCephDaemonVersionString(context, deployment, clusterName) if err != nil { - return nil, fmt.Errorf("failed to run ceph versions: %+v", err) + return nil, fmt.Errorf("failed to run ceph tell. %+v", err) + } + logger.Debug(output) + + v, err := cephver.ExtractCephVersion(output) + if err != nil { + return nil, fmt.Errorf("failed to extract ceph version. %+v", err) + } + + return v, nil +} + +// GetAllCephDaemonVersions reports the Ceph version of each daemon in the cluster +func GetAllCephDaemonVersions(context *clusterd.Context, clusterName string) (*CephDaemonsVersions, error) { + output, err := getAllCephDaemonVersionsString(context, clusterName) + if err != nil { + return nil, err } logger.Debug(output) @@ -90,7 +143,7 @@ func GetCephVersions(context *clusterd.Context) (*CephDaemonsVersions, error) { func EnableMessenger2(context *clusterd.Context) error { _, err := context.Executor.ExecuteCommandWithOutput(false, "", "ceph", "mon", "enable-msgr2") if err != nil { - return fmt.Errorf("failed to enable msgr2 protocol: %+v", err) + return fmt.Errorf("failed to enable msgr2 protocol. %+v", err) } logger.Infof("successfully enabled msgr2 protocol") @@ -104,6 +157,197 @@ func EnableNautilusOSD(context *clusterd.Context) error { return fmt.Errorf("failed to disallow pre-nautilus osds and enable all new nautilus-only functionality: %+v", err) } logger.Infof("successfully disallowed pre-nautilus osds and enabled all new nautilus-only functionality") + return nil +} + +// OkToStop determines if it's ok to stop an upgrade +func OkToStop(context *clusterd.Context, namespace, deployment, clusterName string, cephVersion cephver.CephVersion) error { + daemonName, err := findDaemonName(deployment) + if err != nil { + logger.Warningf("%+v", err) + return nil + } + + // The ok-to-stop command for mon and mds landed on 14.2.1 + // so we return nil if that Ceph version is not satisfied + if !cephVersion.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 1}) { + if daemonName != "osd" { + return nil + } + } + + versions, err := GetAllCephDaemonVersions(context, clusterName) + if err != nil { + return fmt.Errorf("failed to get ceph daemons versions. %+v", err) + } + + switch daemonName { + // Trying to handle the case where a **single** mon is deployed and an upgrade is called + case "mon": + // if len(versions.Mon) > 1, this means we have different Ceph versions for some monitor(s). + // This is fine, we can run the upgrade checks + if len(versions.Mon) == 1 { + // now trying to parse and find how many mons are presents + // if we have less than 3 mons we skip the check and do best-effort + // we do less than 3 because during the initial bootstrap the mon sequence is updated too + // so running running the check on 2/3 mon fails + // versions.Mon looks like this map[ceph version 15.0.0-12-g6c8fb92 (6c8fb920cb1d862f36ee852ed849a15f9a50bd68) octopus (dev):1] + // now looping over a single element since we can't address the key directly (we don't know its name) + for _, monCount := range versions.Mon { + if monCount < 3 { + logger.Infof("the cluster has less than 3 monitors, not performing upgrade check, running in best-effort") + return nil + } + } + } + // Trying to handle the case where a **single** osd is deployed and an upgrade is called + case "osd": + // if len(versions.Osd) > 1, this means we have different Ceph versions osd(s) + // This is fine, we can run the upgrade checks + if len(versions.Osd) == 1 { + // now trying to parse and find how many osds are presents + // if we have less than 3 osds we skip the check and do best-effort + for _, osdCount := range versions.Osd { + if osdCount < 3 { + logger.Infof("the cluster has less than 3 OSDs, not performing upgrade check, running in best-effort") + return nil + } + } + } + } + // we don't implement any checks for mon, rgw and rbdmirror since: + // - mon: the is done in the monitor code since it ensures all the mons are always in quorum before continuing + // - rgw: the pod spec has a liveness probe so if the pod successfully start + // - rbdmirror: you can chain as many as you want like mdss but there is no ok-to-stop logic yet + err = okToStopDaemon(context, deployment, clusterName) + if err != nil { + return fmt.Errorf("failed to check if %s was ok to stop. %+v", deployment, err) + } return nil } + +// OkToContinue determines if it's ok to continue an upgrade +func OkToContinue(context *clusterd.Context, namespace, deployment string) error { + daemonName, err := findDaemonName(deployment) + if err != nil { + logger.Warningf("%+v", err) + return nil + } + + // the mon case is handled directly in the deployment where the mon checks for quorum + switch daemonName { + case "osd": + err := okToContinueOSDDaemon(context, namespace) + if err != nil { + return fmt.Errorf("failed to check if %s was ok to continue. %+v", deployment, err) + } + case "mds": + err := okToContinueMDSDaemon(context, namespace, deployment) + if err != nil { + return fmt.Errorf("failed to check if %s was ok to continue. %+v", deployment, err) + } + } + + return nil +} + +func okToStopDaemon(context *clusterd.Context, deployment, clusterName string) error { + daemonID := findDaemonID(deployment) + daemonName, err := findDaemonName(deployment) + if err != nil { + logger.Warningf("%+v", err) + return nil + } + + if !stringInSlice(daemonName, daemonNoCheck) { + args := []string{daemonName, "ok-to-stop", daemonID} + command, args := FinalizeCephCommandArgs("ceph", args, context.ConfigDir, clusterName) + + output, err := context.Executor.ExecuteCommandWithOutput(false, "", command, args...) + if err != nil { + return fmt.Errorf("deployment %s cannot be stopped. %+v", deployment, err) + } + logger.Debugf("deployment %s is ok to be updated. %s", deployment, output) + } + + logger.Debugf("deployment %s is ok to be updated.", deployment) + + return nil +} + +// okToContinueOSDDaemon determines whether it's fine to go to the next osd during an upgrade +// This basically makes sure all the PGs have settled +func okToContinueOSDDaemon(context *clusterd.Context, namespace string) error { + // Reconciliating PGs should not take too long so let's wait up to 10 minutes + err := util.Retry(10, 60*time.Second, func() error { + return IsClusterClean(context, namespace) + }) + if err != nil { + return err + } + + return nil +} + +// okToContinueMDSDaemon determines whether it's fine to go to the next mds during an upgrade +// mostly a placeholder function for the future but since we have standby mds this shouldn't be needed +func okToContinueMDSDaemon(context *clusterd.Context, namespace, deployment string) error { + // wait for the MDS to be active again or in standby-replay + err := util.Retry(10, 15*time.Second, func() error { + return MdsActiveOrStandbyReplay(context, namespace, findFSName(deployment, namespace)) + }) + if err != nil { + return err + } + + return nil +} + +func findFSName(deployment, namespace string) string { + mdsID := findDaemonID(deployment) + fsNameTrimSuffix := strings.TrimSuffix(deployment, "-"+mdsID) + return strings.TrimPrefix(fsNameTrimSuffix, namespace+"-mds-") +} + +func findDaemonID(deployment string) string { + daemonTrimPrefixSplit := strings.Split(deployment, "-") + return daemonTrimPrefixSplit[len(daemonTrimPrefixSplit)-1] +} + +func findDaemonName(deployment string) (string, error) { + err := errors.New("could not find daemon name, is this a new daemon?") + + if strings.Contains(deployment, "mds") { + return "mds", nil + } + if strings.Contains(deployment, "rgw") { + return "rgw", nil + } + if strings.Contains(deployment, "mon") { + return "mon", nil + } + if strings.Contains(deployment, "osd") { + return "osd", nil + } + if strings.Contains(deployment, "mgr") { + return "mgr", nil + } + if strings.Contains(deployment, "nfs") { + return "nfs", nil + } + if strings.Contains(deployment, "rbd-mirror") { + return "rbd-mirror", nil + } + + return "", fmt.Errorf("%+v from deployment %s", err, deployment) +} + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} diff --git a/pkg/daemon/ceph/client/upgrade_test.go b/pkg/daemon/ceph/client/upgrade_test.go index af2ede739..eb6e86bf6 100644 --- a/pkg/daemon/ceph/client/upgrade_test.go +++ b/pkg/daemon/ceph/client/upgrade_test.go @@ -17,9 +17,11 @@ limitations under the License. package client import ( + "fmt" "testing" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" ) @@ -28,12 +30,11 @@ func TestGetCephMonVersionString(t *testing.T) { executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { assert.Equal(t, "version", args[0]) - assert.Equal(t, 1, len(args)) return "", nil } context := &clusterd.Context{Executor: executor} - _, err := getCephMonVersionString(context) + _, err := getCephMonVersionString(context, "rook-ceph") assert.Nil(t, err) } @@ -41,12 +42,26 @@ func TestGetCephMonVersionsString(t *testing.T) { executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { assert.Equal(t, "versions", args[0]) - assert.Equal(t, 1, len(args)) return "", nil } context := &clusterd.Context{Executor: executor} - _, err := getCephVersionsString(context) + _, err := getAllCephDaemonVersionsString(context, "rook-ceph") + assert.Nil(t, err) +} + +func TestGetCephDaemonVersionString(t *testing.T) { + executor := &exectest.MockExecutor{} + deployment := "rook-ceph-mds-a" + executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { + assert.Equal(t, "tell", args[0]) + assert.Equal(t, "mds.a", args[1]) + assert.Equal(t, "version", args[2]) + return "", nil + } + context := &clusterd.Context{Executor: executor} + + _, err := getCephDaemonVersionString(context, deployment, "rook-ceph") assert.Nil(t, err) } @@ -55,7 +70,6 @@ func TestEnableMessenger2(t *testing.T) { executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { assert.Equal(t, "mon", args[0]) assert.Equal(t, "enable-msgr2", args[1]) - assert.Equal(t, 2, len(args)) return "", nil } context := &clusterd.Context{Executor: executor} @@ -69,8 +83,6 @@ func TestEnableNautilusOSD(t *testing.T) { executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { assert.Equal(t, "osd", args[0]) assert.Equal(t, "require-osd-release", args[1]) - assert.Equal(t, "nautilus", args[2]) - assert.Equal(t, 3, len(args)) return "", nil } context := &clusterd.Context{Executor: executor} @@ -78,3 +90,93 @@ func TestEnableNautilusOSD(t *testing.T) { err := EnableNautilusOSD(context) assert.Nil(t, err) } + +func TestOkToStopDaemon(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { + switch { + case args[0] == "create" && args[1] == "mon" && args[2] == "a": + assert.Equal(t, 2, len(args)) + return "", nil + } + return "", fmt.Errorf("unexpected ceph command '%v'", args) + } + context := &clusterd.Context{Executor: executor} + + deployment := "rook-ceph-mon-a" + err := okToStopDaemon(context, deployment, "rook-ceph") + assert.Nil(t, err) + + deployment = "rook-ceph-mgr-a" + err = okToStopDaemon(context, deployment, "rook-ceph") + assert.Nil(t, err) + + deployment = "rook-ceph-dummy-a" + err = okToStopDaemon(context, deployment, "rook-ceph") + assert.Nil(t, err) +} + +func TestFindDaemonName(t *testing.T) { + n, err := findDaemonName("rook-ceph-mon-a") + assert.Nil(t, err) + assert.Equal(t, "mon", n) + n, err = findDaemonName("rook-ceph-osd-0") + assert.Nil(t, err) + assert.Equal(t, "osd", n) + n, err = findDaemonName("rook-ceph-rgw-my-store-a") + assert.Nil(t, err) + assert.Equal(t, "rgw", n) + n, err = findDaemonName("rook-ceph-mds-myfs-a") + assert.Nil(t, err) + assert.Equal(t, "mds", n) + n, err = findDaemonName("rook-ceph-mgr-a") + assert.Nil(t, err) + assert.Equal(t, "mgr", n) + n, err = findDaemonName("rook-ceph-rbd-mirror-a") + assert.Nil(t, err) + assert.Equal(t, "rbd-mirror", n) + _, err = findDaemonName("rook-ceph-unknown-a") + assert.NotNil(t, err) +} + +func TestFindDaemonID(t *testing.T) { + id := findDaemonID("rook-ceph-mon-a") + assert.Equal(t, "a", id) + id = findDaemonID("rook-ceph-osd-0") + assert.Equal(t, "0", id) + id = findDaemonID("rook-ceph-rgw-my-super-store-a") + assert.Equal(t, "a", id) + id = findDaemonID("rook-ceph-mds-my-wonderful-fs-a") + assert.Equal(t, "a", id) + id = findDaemonID("rook-ceph-mgr-a") + assert.Equal(t, "a", id) + id = findDaemonID("rook.ceph.mgr.a") + assert.NotEqual(t, "a", id) +} + +func TestOkToContinue(t *testing.T) { + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Executor: executor} + + err := OkToContinue(context, "rook-ceph", "rook-ceph-mon-a") // mon is not checked on ok-to-continue so nil is expected + assert.Nil(t, err) +} + +func TestOkToStop(t *testing.T) { + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Executor: executor} + v := cephver.Nautilus + + err := OkToStop(context, "rook-ceph", "rook-ceph-mon-a", "rook-ceph", v) + assert.Nil(t, err) + + err = OkToStop(context, "rook-ceph", "rook-ceph-mds-a", "rook-ceph", v) + assert.Nil(t, err) +} + +func TestFindFSName(t *testing.T) { + fsName := findFSName("rook-ceph-mds-myfs-a", "rook-ceph") + assert.Equal(t, "myfs", fsName) + fsName = findFSName("rook-ceph-mds-my-super-fs-a", "rook-ceph") + assert.Equal(t, "my-super-fs", fsName) +} diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index e2653a471..07b1f0cd9 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -29,6 +29,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" @@ -106,7 +107,7 @@ func New( } } -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start begins the process of running a cluster of Ceph mgrs. func (c *Cluster) Start() error { @@ -151,7 +152,7 @@ func (c *Cluster) Start() error { return fmt.Errorf("failed to create mgr deployment %s. %+v", resourceName, err) } logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName) - if _, err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil { + if err := updateDeploymentAndWait(c.context, d, c.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { return fmt.Errorf("failed to update mgr deployment %s. %+v", resourceName, err) } } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index f715624e2..267f37f0e 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -219,12 +219,12 @@ func (c *Cluster) startMons(targetCount int) error { // Enable Ceph messenger 2 protocol on Nautilus if c.clusterInfo.CephVersion.IsAtLeastNautilus() { - v, err := client.GetCephMonVersion(c.context) + v, err := client.GetCephMonVersion(c.context, c.clusterInfo.Name) if err != nil { return fmt.Errorf("failed to get ceph mon version. %+v", err) } if v.IsAtLeastNautilus() { - versions, err := client.GetCephVersions(c.context) + versions, err := client.GetAllCephDaemonVersions(c.context, c.clusterInfo.Name) if err != nil { return fmt.Errorf("failed to get ceph daemons versions. %+v", err) } @@ -518,9 +518,28 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e if err != nil { return fmt.Errorf("failed to create mon %s. %+v", mons[i].DaemonName, err) } + // For the initial deployment (first creation) it's expected to not have all the monitors in quorum + // However, in an event of an update, it's crucial to proceed monitors by monitors + // At the end of the method we perform one last check where all the monitors must be in quorum + requireAllInQuorum := false + err = c.waitForMonsToJoin(mons, requireAllInQuorum) + if err != nil { + return fmt.Errorf("failed to check mon quorum %s. %+v", mons[i].DaemonName, err) + } } logger.Infof("mons created: %d", len(mons)) + // Final verification that **all** mons are in quorum + // Do not proceed if one monitor is still syncing + // Only do this when monitors versions are different so we don't block the orchestration if a mon is down. + versions, err := client.GetAllCephDaemonVersions(c.context, c.clusterInfo.Name) + if err != nil { + logger.Warningf("failed to get ceph daemons versions; this likely means there is no cluster yet. %+v", err) + } else { + if len(versions.Mon) != 1 { + requireAllInQuorum = true + } + } return c.waitForMonsToJoin(mons, requireAllInQuorum) } @@ -596,9 +615,10 @@ func (c *Cluster) saveMonConfig() error { return nil } -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var updateDeploymentAndWait = UpdateCephDeploymentAndWait func (c *Cluster) startMon(m *monConfig, hostname string) error { + d := c.makeDeployment(m, hostname) logger.Debugf("Starting mon: %+v", d.Name) _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d) @@ -607,7 +627,7 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error { return fmt.Errorf("failed to create mon deployment %s. %+v", m.ResourceName, err) } logger.Infof("deployment for mon %s already exists. updating if needed", m.ResourceName) - if _, err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil { + if err := updateDeploymentAndWait(c.context, d, c.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { return fmt.Errorf("failed to update mon deployment %s. %+v", m.ResourceName, err) } } @@ -620,7 +640,7 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [ // wait for monitors to establish quorum retryCount := 0 - retryMax := 20 + retryMax := 30 for { retryCount++ if retryCount > retryMax { diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 70f326369..5967100c5 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -21,9 +21,11 @@ import ( "os" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" - opspec "github.com/rook/rook/pkg/operator/ceph/spec" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -213,3 +215,29 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { return container } + +// UpdateCephDeploymentAndWait verifies a deployment can be stopped or continued +func UpdateCephDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace, clusterName string, cephVersion cephver.CephVersion) error { + callback := func(action string) error { + logger.Infof("checking if we can %s the deployment %s", action, deployment.Name) + + if action == "stop" { + err := client.OkToStop(context, namespace, deployment.Name, clusterName, cephVersion) + if err != nil { + return fmt.Errorf("failed to check if we can %s the deployment %s: %+v", action, deployment.Name, err) + } + } + + if action == "continue" { + err := client.OkToContinue(context, namespace, deployment.Name) + if err != nil { + return fmt.Errorf("failed to check if we can %s the deployment %s: %+v", action, deployment.Name, err) + } + } + + return nil + } + + _, err := k8sutil.UpdateDeploymentAndWait(context, deployment, namespace, callback) + return err +} diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 5ef63ce86..6d4dba915 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" osdconfig "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" cephver "github.com/rook/rook/pkg/operator/ceph/version" @@ -40,7 +41,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-osd") +var ( + logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-osd") + updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait +) const ( appName = "rook-ceph-osd" @@ -191,9 +195,9 @@ func (c *Cluster) Start() error { // The following block is used to apply any command(s) required by an upgrade // The block below handles the upgrade from Mimic to Nautilus. if c.clusterInfo.CephVersion.IsAtLeastNautilus() { - versions, err := client.GetCephVersions(c.context) + versions, err := client.GetAllCephDaemonVersions(c.context, c.clusterInfo.Name) if err != nil { - logger.Warningf("failed to get ceph daemons versions. this likely means there are no osds yet. %+v", err) + logger.Warningf("failed to get ceph daemons versions; this likely means there are no osds yet. %+v", err) } else { // If length is one, this clearly indicates that all the osds are running the same version logger.Infof("len of version.Osd is %d", len(versions.Osd)) @@ -321,7 +325,7 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig continue } logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID) - if _, err = k8sutil.UpdateDeploymentAndWait(c.context, dp, c.Namespace); err != nil { + if err = updateDeploymentAndWait(c.context, dp, c.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { config.addError(fmt.Sprintf("failed to update osd deployment %d. %+v", osd.ID, err)) } } diff --git a/pkg/operator/ceph/cluster/rbd/mirror.go b/pkg/operator/ceph/cluster/rbd/mirror.go index be3805633..b5884f401 100644 --- a/pkg/operator/ceph/cluster/rbd/mirror.go +++ b/pkg/operator/ceph/cluster/rbd/mirror.go @@ -25,6 +25,7 @@ import ( rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" @@ -87,7 +88,7 @@ func New( } } -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start begins the process of running rbd mirroring daemons. func (m *Mirroring) Start() error { @@ -119,7 +120,7 @@ func (m *Mirroring) Start() error { return fmt.Errorf("failed to create %s deployment. %+v", resourceName, err) } logger.Infof("deployment for rbd-mirror %s already exists. updating if needed", resourceName) - if _, err := updateDeploymentAndWait(m.context, d, m.Namespace); err != nil { + if err := updateDeploymentAndWait(m.context, d, m.Namespace, m.ClusterInfo.Name, m.ClusterInfo.CephVersion); err != nil { // fail could be an issue updating label selector (immutable), so try del and recreate logger.Debugf("updateDeploymentAndWait failed for rbd-mirror %s. Attempting del-and-recreate. %+v", resourceName, err) err = m.context.Clientset.AppsV1().Deployments(m.Namespace).Delete(d.Name, &metav1.DeleteOptions{}) diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index 35e2ffe02..ba6cc2ea6 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -27,6 +27,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" cephver "github.com/rook/rook/pkg/operator/ceph/version" @@ -94,7 +95,7 @@ func NewCluster( } // UpdateDeploymentAndWait can be overridden for unit tests. Do not alter this for runtime operation. -var UpdateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var UpdateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start starts or updates a Ceph mds cluster in Kubernetes. func (c *Cluster) Start() error { @@ -155,7 +156,7 @@ func (c *Cluster) Start() error { // keyring must be generated before update-and-wait since no keyring will prevent the // deployment from reaching ready state if createErr != nil && errors.IsAlreadyExists(createErr) { - if _, err = UpdateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil { + if err = UpdateDeploymentAndWait(c.context, d, c.fs.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { return fmt.Errorf("failed to update mds deployment %s. %+v", d.Name, err) } } diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index c2de02853..a84e66ae4 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -36,7 +36,7 @@ const ( ganeshaRadosGraceCmd = "ganesha-rados-grace" ) -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var updateDeploymentAndWait = opmon.UpdateCephDeploymentAndWait // Create the ganesha server func (c *CephNFSController) upCephNFS(n cephv1.CephNFS, oldActive int) error { @@ -68,7 +68,7 @@ func (c *CephNFSController) upCephNFS(n cephv1.CephNFS, oldActive int) error { return fmt.Errorf("failed to create ganesha deployment. %+v", err) } logger.Infof("ganesha deployment %s already exists. updating if needed", deployment.Name) - if _, err := updateDeploymentAndWait(c.context, deployment, n.Namespace); err != nil { + if err := updateDeploymentAndWait(c.context, deployment, n.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { return fmt.Errorf("failed to update ganesha deployment %s. %+v", deployment.Name, err) } } else { diff --git a/pkg/operator/edgefs/cluster/mgr/mgr.go b/pkg/operator/edgefs/cluster/mgr/mgr.go index b26acfc72..17b602ee4 100644 --- a/pkg/operator/edgefs/cluster/mgr/mgr.go +++ b/pkg/operator/edgefs/cluster/mgr/mgr.go @@ -131,7 +131,13 @@ func (c *Cluster) Start(rookImage string) error { return fmt.Errorf("failed to create %s deployment. %+v", appName, err) } logger.Infof("deployment for mgr %s already exists. updating if needed", appName) - if _, err := k8sutil.UpdateDeploymentAndWait(c.context, deployment, c.Namespace); err != nil { + + // placeholder for a verify callback + // see comments on k8sutil.UpdateDeploymentAndWait's definition to understand its purpose + callback := func(action string) error { + return nil + } + if _, err := k8sutil.UpdateDeploymentAndWait(c.context, deployment, c.Namespace, callback); err != nil { return fmt.Errorf("failed to update mgr deployment %s. %+v", appName, err) } } else { diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index c5244f52c..3c3c93b78 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -21,8 +21,9 @@ import ( "time" "github.com/rook/rook/pkg/clusterd" - "k8s.io/api/apps/v1" + "github.com/rook/rook/pkg/util" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -36,6 +37,7 @@ func GetDeploymentImage(clientset kubernetes.Interface, namespace, name, contain return GetDeploymentSpecImage(clientset, *d, container, false) } +// GetDeploymentSpecImage returns the image name from the spec func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, container string, initContainer bool) (string, error) { image, err := GetSpecContainerImage(d.Spec.Template.Spec, container, initContainer) if err != nil { @@ -47,12 +49,27 @@ func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, c // UpdateDeploymentAndWait updates a deployment and waits until it is running to return. It will // error if the deployment does not exist to be updated or if it takes too long. -func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error) { +// This method has a generic callback function that each backend can rely on +// It serves two purposes: +// 1. verify that a resource can be stopped +// 2. verify that we can continue the update procedure +// Basically, we go one resource by one and check if we can stop and then if the ressource has been successfully updated +// we check if we can go ahead and move to the next one. +func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string, verifyCallback func(action string) error) (*v1.Deployment, error) { original, err := context.Clientset.AppsV1().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err) } + // Let's verify the deployment can be stopped + // retry for 5 times, every minute + err = util.Retry(5, 60*time.Second, func() error { + return verifyCallback("stop") + }) + if err != nil { + return nil, fmt.Errorf("failed to check if deployment %s can be updated: %+v", deployment.Name, err) + } + logger.Infof("updating deployment %s", deployment.Name) if _, err := context.Clientset.AppsV1().Deployments(namespace).Update(deployment); err != nil { return nil, fmt.Errorf("failed to update deployment %s. %+v", deployment.Name, err) @@ -73,9 +90,15 @@ func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deploym } if d.Status.ObservedGeneration != original.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 { logger.Infof("finished waiting for updated deployment %s", d.Name) + + // Now we check if we can go to the next daemon + err = verifyCallback("continue") + if err != nil { + return nil, fmt.Errorf("failed to check if deployment %s can be updated: %+v", deployment.Name, err) + } + return d, nil } - logger.Debugf("deployment %s status=%+v", d.Name, d.Status) time.Sleep(time.Duration(sleepTime) * time.Second) } diff --git a/pkg/operator/k8sutil/test/deployment.go b/pkg/operator/k8sutil/test/deployment.go index e4d5c5b8e..b3b4fd336 100644 --- a/pkg/operator/k8sutil/test/deployment.go +++ b/pkg/operator/k8sutil/test/deployment.go @@ -2,8 +2,8 @@ package test import ( "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/version" apps "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // UpdateDeploymentAndWaitStub returns a stub replacement for the UpdateDeploymentAndWait function @@ -15,13 +15,13 @@ import ( // returns a pointer to this slice which the calling func may use to verify the expected contents of // deploymentsUpdated based on expected behavior. func UpdateDeploymentAndWaitStub() ( - stubFunc func(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*apps.Deployment, error), + stubFunc func(context *clusterd.Context, deployment *apps.Deployment, namespace, clusterName string, cephVersion version.CephVersion) error, deploymentsUpdated *[]*apps.Deployment, ) { deploymentsUpdated = &[]*apps.Deployment{} - stubFunc = func(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*apps.Deployment, error) { + stubFunc = func(context *clusterd.Context, deployment *apps.Deployment, namespace, clusterName string, cephVersion version.CephVersion) error { *deploymentsUpdated = append(*deploymentsUpdated, deployment) - return &apps.Deployment{ObjectMeta: metav1.ObjectMeta{UID: "stub-deployment-uid"}}, nil + return nil } return stubFunc, deploymentsUpdated }