From 1c1d6cf261fd79cb1dee5aaaf03b0573c428645f Mon Sep 17 00:00:00 2001 From: Santosh Date: Mon, 11 May 2026 15:36:36 +0530 Subject: [PATCH] core: skip crush rule cleanup if no pools exist yet During the initial cluster setup, CleanUnusedCrushRules() runs after MGR startup but before any pools are created (if user does not create the .mgr pool along with the cephCluster manifest). It deletes the default replicated_pool since no pool references it. This leaves the cluster in a state if Zero crush rules. When the MGR tries to start the device_health_metris during this window, thne it fails to find the crush rule and crashes. Skip the cleanup when no pools exists yet, since there is nothing meaninful to cleanup. Signed-off-by: Santosh --- pkg/daemon/ceph/client/pool.go | 13 +++++++++++++ pkg/daemon/ceph/client/pool_test.go | 25 ++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pkg/daemon/ceph/client/pool.go b/pkg/daemon/ceph/client/pool.go index 1c732e8f4..6e2958a8b 100644 --- a/pkg/daemon/ceph/client/pool.go +++ b/pkg/daemon/ceph/client/pool.go @@ -607,6 +607,15 @@ func CleanupUnusedCrushRules(context *clusterd.Context, clusterInfo *ClusterInfo return err } + // Skip cleanup when no pools exist yet. During initial cluster setup, this function + // runs after MGR startup but before pools are created. Deleting the default + // "replicated_rule" in this window leaves the cluster with zero CRUSH rules, causing + // Ceph internal modules (e.g. devicehealth) to crash when they try to create pools. + if len(usedRules) == 0 { + logger.Infof("no crush rules are in use yet, skipping cleanup") + return nil + } + crushMap, err := GetCrushMap(context, clusterInfo) if err != nil { return err @@ -618,6 +627,10 @@ func CleanupUnusedCrushRules(context *clusterd.Context, clusterInfo *ClusterInfo logger.Debugf("crush rule %q is still in use", rule.Name) continue } + if rule.Name == "replicated_rule" { + logger.Debugf("skipping deletion of default crush rule %q", rule.Name) + continue + } logger.Infof("crush rule %q is unused and will be deleted", rule.Name) if err := deleteCrushRule(context, clusterInfo, rule.Name); err != nil { logger.Warningf("failed to delete unused crush rule %q, continuing with remaining rules. %v", rule.Name, err) diff --git a/pkg/daemon/ceph/client/pool_test.go b/pkg/daemon/ceph/client/pool_test.go index b17ed113e..12301e10b 100644 --- a/pkg/daemon/ceph/client/pool_test.go +++ b/pkg/daemon/ceph/client/pool_test.go @@ -397,7 +397,8 @@ func TestCleanupUnusedCrushRules(t *testing.T) { {"rule_name":"mypool_zone","steps":[{},{"type":"zone"}]}, {"rule_name":"mypool_zone_ssd","steps":[{},{"type":"zone","item_name":"default~ssd"}]}, {"rule_name":"shared_rule","steps":[{"item_name":"default"},{"type":"host"}]}, - {"rule_name":"custom_rule","steps":[{"item_name":"default"},{"type":"zone"}]} + {"rule_name":"custom_rule","steps":[{"item_name":"default"},{"type":"zone"}]}, + {"rule_name":"replicated_rule","steps":[{"item_name":"default"},{"type":"host"}]} ]}`, nil } if command == "ceph" && args[1] == "crush" && args[2] == "rule" && args[3] == "rm" { @@ -413,6 +414,28 @@ func TestCleanupUnusedCrushRules(t *testing.T) { slices.Sort(removedRules) expected := []string{"custom_rule", "mypool", "mypool_zone"} assert.Equal(t, expected, removedRules) + assert.NotContains(t, removedRules, "replicated_rule") +} + +func TestCleanupUnusedCrushRulesNoPools(t *testing.T) { + removedRules := []string{} + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Executor: executor} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + if command == "ceph" && args[1] == "lspools" { + return `[]`, nil + } + if command == "ceph" && args[1] == "crush" && args[2] == "rule" && args[3] == "rm" { + removedRules = append(removedRules, args[4]) + return "", nil + } + return "", errors.Errorf("unexpected ceph command %q", args) + } + + err := CleanupUnusedCrushRules(context, AdminTestClusterInfo("mycluster")) + assert.NoError(t, err) + assert.Empty(t, removedRules) } func TestExtractPoolDetails(t *testing.T) {