diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go index 4dd380a48..5fee97e7b 100644 --- a/pkg/daemon/ceph/client/upgrade.go +++ b/pkg/daemon/ceph/client/upgrade.go @@ -37,7 +37,8 @@ const ( var ( // we don't perform any checks on these daemons // they don't have any "ok-to-stop" command implemented - daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"} + daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"} + errNoHostInCRUSH = errors.New("no host in crush map yet?") ) func getCephMonVersionString(context *clusterd.Context, clusterInfo *ClusterInfo) (string, error) { @@ -311,7 +312,7 @@ func allOSDsSameHost(context *clusterd.Context, clusterInfo *ClusterInfo) (bool, hostOsdNodes := len(hostOsdTree.Nodes) if hostOsdNodes == 0 { - return false, errors.New("no host in crush map yet?") + return false, errNoHostInCRUSH } // If the number of OSD node is 1, chances are this is simple setup with all OSDs on it @@ -369,6 +370,10 @@ func OSDUpdateShouldCheckOkToStop(context *clusterd.Context, clusterInfo *Cluste // aio means all in one aio, err := allOSDsSameHost(context, clusterInfo) if err != nil { + if errors.Is(err, errNoHostInCRUSH) { + logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks") + return false + } logger.Warningf("failed to determine if all osds are running on the same host. will check if OSDs are ok-to-stop. if all OSDs are running on one host %s. %v", userIntervention, err) return true } @@ -401,6 +406,18 @@ func osdDoNothing(context *clusterd.Context, clusterInfo *ClusterInfo) bool { // aio means all in one aio, err := allOSDsSameHost(context, clusterInfo) if err != nil { + // We return true so that we can continue without a retry and subsequently not test if the + // osd can be stopped This handles the scenario where the OSDs have been created but not yet + // started due to a wrong CR configuration For instance, when OSDs are encrypted and Vault + // is used to store encryption keys, if the KV version is incorrect during the cluster + // initialization the OSDs will fail to start and stay in CLBO until the CR is updated again + // with the correct KV version so that it can start For this scenario we don't need to go + // through the path where the check whether the OSD can be stopped or not, so it will always + // fail and make us wait for nothing + if errors.Is(err, errNoHostInCRUSH) { + logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks") + return true + } logger.Warningf("failed to determine if all osds are running on the same host, performing upgrade check anyways. %v", err) return false } diff --git a/pkg/daemon/ceph/client/upgrade_test.go b/pkg/daemon/ceph/client/upgrade_test.go index aceefadd3..2035d84bb 100644 --- a/pkg/daemon/ceph/client/upgrade_test.go +++ b/pkg/daemon/ceph/client/upgrade_test.go @@ -357,4 +357,11 @@ func TestOSDUpdateShouldCheckOkToStop(t *testing.T) { treeOutput = fake.OsdTreeOutput(0, 0) assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo)) }) + + // degraded case, OSDs are failing to start so they haven't registered in the CRUSH map yet + t.Run("0 nodes with down OSDs", func(t *testing.T) { + lsOutput = fake.OsdLsOutput(3) + treeOutput = fake.OsdTreeOutput(0, 1) + assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo)) + }) }