diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 2434935b1..1338532af 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -293,11 +293,12 @@ func (c *Cluster) checkHealth(ctx context.Context) error { rescheduleImmediately := false if _, ok := c.monTimeoutList[mon.Name]; !ok { c.monTimeoutList[mon.Name] = time.Now() - - // If the assigned node is not found, skip the timeout wait - rescheduleImmediately = c.shouldFailoverMonImmediately(ctx, mon.Name) } + // Check if the assigned node exists on every health check iteration + // to detect if a node is deleted during the timeout + rescheduleImmediately = c.shouldFailoverMonImmediately(ctx, mon.Name) + // when the timeout for the mon has been reached, continue to the // normal failover mon pod part of the code if !rescheduleImmediately { diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 06e67317b..7cd5db7c5 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -241,6 +241,11 @@ func TestScheduleFailoverImmediately(t *testing.T) { assert.False(t, c.shouldFailoverMonImmediately((context.TODO()), "b")) // mon c is assigned to a non-existent node, so it should failover immediately assert.True(t, c.shouldFailoverMonImmediately((context.TODO()), "c")) + // Test that the function detects node deletion when called repeatedly + err = clientset.CoreV1().Nodes().Delete(context.TODO(), "a", metav1.DeleteOptions{}) + assert.NoError(t, err) + // node a has been deleted, so mon a should failover immediately + assert.True(t, c.shouldFailoverMonImmediately((context.TODO()), "a")) } func TestTrackMonsOutOfQuorum(t *testing.T) {