feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

ceph: do not check ok-to-stop when OSDs are in CLBO

Browse Source 
This handles the scenario where the OSDs have been created but not yet
started due to a wrong CR configuration.
For instance, when OSDs are encrypted and Vault is used to store
encryption keys, if the KV version is incorrect during the cluster
initialization the OSDs will fail to start and stay in CLBO until the
CR is updated again with the correct KV version so that it can start.

For this scenario, if the CRUSH map has no host registered yet it's fair
to assume the initialization broke and we need to fix it. So when don't
need to call ok-to-stop since it will always fail and eventually force
pass but let's not wait for nothing.

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han
2021-08-25 09:30:38 +02:00
parent 6582bac8e7
commit 88048cc4e8
2 changed files with 26 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/daemon/ceph/client/upgrade.go
+19 -2
View File
@@ -37,7 +37,8 @@ const (
var (
// we don't perform any checks on these daemons
// they don't have any "ok-to-stop" command implemented
daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"}
daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"}
errNoHostInCRUSH = errors.New("no host in crush map yet?")
)
func getCephMonVersionString(context *clusterd.Context, clusterInfo *ClusterInfo) (string, error) {
@@ -311,7 +312,7 @@ func allOSDsSameHost(context *clusterd.Context, clusterInfo *ClusterInfo) (bool,
hostOsdNodes := len(hostOsdTree.Nodes)
if hostOsdNodes == 0 {
return false, errors.New("no host in crush map yet?")
return false, errNoHostInCRUSH
}
// If the number of OSD node is 1, chances are this is simple setup with all OSDs on it
@@ -369,6 +370,10 @@ func OSDUpdateShouldCheckOkToStop(context *clusterd.Context, clusterInfo *Cluste
// aio means all in one
aio, err := allOSDsSameHost(context, clusterInfo)
if err != nil {
if errors.Is(err, errNoHostInCRUSH) {
logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks")
return false
}
logger.Warningf("failed to determine if all osds are running on the same host. will check if OSDs are ok-to-stop. if all OSDs are running on one host %s. %v", userIntervention, err)
return true
}
@@ -401,6 +406,18 @@ func osdDoNothing(context *clusterd.Context, clusterInfo *ClusterInfo) bool {
// aio means all in one
aio, err := allOSDsSameHost(context, clusterInfo)
if err != nil {
// We return true so that we can continue without a retry and subsequently not test if the
// osd can be stopped This handles the scenario where the OSDs have been created but not yet
// started due to a wrong CR configuration For instance, when OSDs are encrypted and Vault
// is used to store encryption keys, if the KV version is incorrect during the cluster
// initialization the OSDs will fail to start and stay in CLBO until the CR is updated again
// with the correct KV version so that it can start For this scenario we don't need to go
// through the path where the check whether the OSD can be stopped or not, so it will always
// fail and make us wait for nothing
if errors.Is(err, errNoHostInCRUSH) {
logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks")
return true
}
logger.Warningf("failed to determine if all osds are running on the same host, performing upgrade check anyways. %v", err)
return false
}
pkg/daemon/ceph/client/upgrade_test.go
+7
View File
@@ -357,4 +357,11 @@ func TestOSDUpdateShouldCheckOkToStop(t *testing.T) {
treeOutput = fake.OsdTreeOutput(0, 0)
assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo))
})
// degraded case, OSDs are failing to start so they haven't registered in the CRUSH map yet
t.Run("0 nodes with down OSDs", func(t *testing.T) {
lsOutput = fake.OsdLsOutput(3)
treeOutput = fake.OsdTreeOutput(0, 1)
assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo))
})
}