mon: mute MON_NETSPLIT health warning in stretch clusters

Ceph incorrectly raises the MON_NETSPLIT health warning in stretch cluster configurations. Added a call to MuteHealthWarning() in postMonStartupActions() which runs `ceph health mute MON_NETSPLIT --sticky` to suppress the warning during reconcile. All errors are logged as warnings and the reconcile proceeds.

Signed-off-by: Prabhala Tara Aasrita <taraasrita@ibm.com>
(cherry picked from commit eccc8128af)

pkg/daemon/ceph/client/status.go

@@ -315,3 +315,11 @@ func isCephHealthy(status CephStatus) bool {
 
 	return false
 }
+
+func MuteHealthWarning(context *clusterd.Context, clusterInfo *ClusterInfo, warning string) {
+	args := []string{"health", "mute", warning, "--sticky"}
+	_, err := NewCephCommand(context, clusterInfo, args).Run()
+	if err != nil {
+		logger.Warningf("failed to mute health warning %q: %v", warning, err)
+	}
+}

pkg/daemon/ceph/client/status_test.go

@@ -20,6 +20,9 @@ import (
 	"regexp"
 	"testing"
 
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -151,3 +154,46 @@ func TestIsCephHealthy(t *testing.T) {
 	s = isCephHealthy(statusFake)
 	assert.False(t, s)
 }
+
+func TestMuteHealthWarning(t *testing.T) {
+	tests := []struct {
+		name           string
+		warning        string
+		mockError      error
+		expectsWarning bool
+	}{
+		{
+			name:           "successful mute",
+			warning:        "MON_NETSPLIT",
+			mockError:      nil,
+			expectsWarning: false,
+		},
+		{
+			name:           "command failure logs warning",
+			warning:        "MON_NETSPLIT",
+			mockError:      errors.New("command execution failed"),
+			expectsWarning: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			executor := &exectest.MockExecutor{}
+			executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+				assert.Equal(t, "ceph", command)
+				assert.Equal(t, "health", args[0])
+				assert.Equal(t, "mute", args[1])
+				assert.Equal(t, tt.warning, args[2])
+				assert.Equal(t, "--sticky", args[3])
+				return "", tt.mockError
+			}
+
+			context := &clusterd.Context{Executor: executor}
+			clusterInfo := AdminTestClusterInfo("mycluster")
+
+			assert.NotPanics(t, func() {
+				MuteHealthWarning(context, clusterInfo, tt.warning)
+			})
+		})
+	}
+}

pkg/operator/ceph/cluster/cluster.go

@@ -533,6 +533,13 @@ func (c *cluster) postMonStartupActions() error {
 		return errors.Wrap(err, "failed to configure storage settings")
 	}
 
+	// Mute the MON_NETSPLIT health warning in stretch clusters since ceph raises it incorrectly.
+	// This can be removed once ceph fixes the health warning: https://tracker.ceph.com/issues/71344
+	if c.Spec.IsStretchCluster() {
+		logger.Debugf("stretch cluster detected, muting MON_NETSPLIT health warning")
+		client.MuteHealthWarning(c.context, c.ClusterInfo, "MON_NETSPLIT")
+	}
+
 	crushRoot := client.GetCrushRootFromSpec(c.Spec)
 	if crushRoot != "default" {
 		// Remove the root=default and replicated_rule which are created by