Merge pull request #8583 from leseb/cancel-retry

ceph: do not check ok-to-stop when OSDs are in CLBO
This commit is contained in:
Sébastien Han authored and GitHub committed 2021-08-25 17:38:06 +02:00
commit 9744f0f2c6
2 files changed
+26 -2

No files matched your search

+19 -2
View File
@@ -37,7 +37,8 @@ const (
var (
// we don't perform any checks on these daemons
// they don't have any "ok-to-stop" command implemented
daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"}
daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"}
errNoHostInCRUSH = errors.New("no host in crush map yet?")
)
func getCephMonVersionString(context *clusterd.Context, clusterInfo *ClusterInfo) (string, error) {
@@ -311,7 +312,7 @@ func allOSDsSameHost(context *clusterd.Context, clusterInfo *ClusterInfo) (bool,
hostOsdNodes := len(hostOsdTree.Nodes)
if hostOsdNodes == 0 {
return false, errors.New("no host in crush map yet?")
return false, errNoHostInCRUSH
}
// If the number of OSD node is 1, chances are this is simple setup with all OSDs on it
@@ -369,6 +370,10 @@ func OSDUpdateShouldCheckOkToStop(context *clusterd.Context, clusterInfo *Cluste
// aio means all in one
aio, err := allOSDsSameHost(context, clusterInfo)
if err != nil {
if errors.Is(err, errNoHostInCRUSH) {
logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks")
return false
}
logger.Warningf("failed to determine if all osds are running on the same host. will check if OSDs are ok-to-stop. if all OSDs are running on one host %s. %v", userIntervention, err)
return true
}
@@ -401,6 +406,18 @@ func osdDoNothing(context *clusterd.Context, clusterInfo *ClusterInfo) bool {
// aio means all in one
aio, err := allOSDsSameHost(context, clusterInfo)
if err != nil {
// We return true so that we can continue without a retry and subsequently not test if the
// osd can be stopped This handles the scenario where the OSDs have been created but not yet
// started due to a wrong CR configuration For instance, when OSDs are encrypted and Vault
// is used to store encryption keys, if the KV version is incorrect during the cluster
// initialization the OSDs will fail to start and stay in CLBO until the CR is updated again
// with the correct KV version so that it can start For this scenario we don't need to go
// through the path where the check whether the OSD can be stopped or not, so it will always
// fail and make us wait for nothing
if errors.Is(err, errNoHostInCRUSH) {
logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks")
return true
}
logger.Warningf("failed to determine if all osds are running on the same host, performing upgrade check anyways. %v", err)
return false
}
+7
View File
@@ -357,4 +357,11 @@ func TestOSDUpdateShouldCheckOkToStop(t *testing.T) {
treeOutput = fake.OsdTreeOutput(0, 0)
assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo))
})
// degraded case, OSDs are failing to start so they haven't registered in the CRUSH map yet
t.Run("0 nodes with down OSDs", func(t *testing.T) {
lsOutput = fake.OsdLsOutput(3)
treeOutput = fake.OsdTreeOutput(0, 1)
assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo))
})
}