From 771e377cc7e46166d4eb065b18e9873009e33756 Mon Sep 17 00:00:00 2001 From: Prabhala Tara Aasrita Date: Tue, 5 May 2026 12:39:07 +0530 Subject: [PATCH] mon: check node existence on every health check iteration Previously, shouldFailoverMonImmediately() was called only when a mon first went out of quorum. If the node gets deleted after the timeout starts, it does not get detected until the timeout expires. Moved the call outside the timeout initialization block so it runs on every health check iteration and detects node deletion immediately. Fixes #15953 Signed-off-by: Prabhala Tara Aasrita (cherry picked from commit aeae61e5bea3e7c6210685af25038425462caac6) --- pkg/operator/ceph/cluster/mon/health.go | 7 ++++--- pkg/operator/ceph/cluster/mon/health_test.go | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 2434935b1..1338532af 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -293,11 +293,12 @@ func (c *Cluster) checkHealth(ctx context.Context) error { rescheduleImmediately := false if _, ok := c.monTimeoutList[mon.Name]; !ok { c.monTimeoutList[mon.Name] = time.Now() - - // If the assigned node is not found, skip the timeout wait - rescheduleImmediately = c.shouldFailoverMonImmediately(ctx, mon.Name) } + // Check if the assigned node exists on every health check iteration + // to detect if a node is deleted during the timeout + rescheduleImmediately = c.shouldFailoverMonImmediately(ctx, mon.Name) + // when the timeout for the mon has been reached, continue to the // normal failover mon pod part of the code if !rescheduleImmediately { diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 06e67317b..7cd5db7c5 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -241,6 +241,11 @@ func TestScheduleFailoverImmediately(t *testing.T) { assert.False(t, c.shouldFailoverMonImmediately((context.TODO()), "b")) // mon c is assigned to a non-existent node, so it should failover immediately assert.True(t, c.shouldFailoverMonImmediately((context.TODO()), "c")) + // Test that the function detects node deletion when called repeatedly + err = clientset.CoreV1().Nodes().Delete(context.TODO(), "a", metav1.DeleteOptions{}) + assert.NoError(t, err) + // node a has been deleted, so mon a should failover immediately + assert.True(t, c.shouldFailoverMonImmediately((context.TODO()), "a")) } func TestTrackMonsOutOfQuorum(t *testing.T) {