Merge pull request #17490 from rook/mergify/bp/release-1.20/pr-17484

mon: check node existence on every health check iteration (backport #17484)

pkg/operator/ceph/cluster/mon/health.go

@@ -293,11 +293,12 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
 		rescheduleImmediately := false
 		if _, ok := c.monTimeoutList[mon.Name]; !ok {
 			c.monTimeoutList[mon.Name] = time.Now()
-
-			// If the assigned node is not found, skip the timeout wait
-			rescheduleImmediately = c.shouldFailoverMonImmediately(ctx, mon.Name)
 		}
 
+		// Check if the assigned node exists on every health check iteration
+		// to detect if a node is deleted during the timeout
+		rescheduleImmediately = c.shouldFailoverMonImmediately(ctx, mon.Name)
+
 		// when the timeout for the mon has been reached, continue to the
 		// normal failover mon pod part of the code
 		if !rescheduleImmediately {

pkg/operator/ceph/cluster/mon/health_test.go

@@ -241,6 +241,11 @@ func TestScheduleFailoverImmediately(t *testing.T) {
 	assert.False(t, c.shouldFailoverMonImmediately((context.TODO()), "b"))
 	// mon c is assigned to a non-existent node, so it should failover immediately
 	assert.True(t, c.shouldFailoverMonImmediately((context.TODO()), "c"))
+	// Test that the function detects node deletion when called repeatedly
+	err = clientset.CoreV1().Nodes().Delete(context.TODO(), "a", metav1.DeleteOptions{})
+	assert.NoError(t, err)
+	// node a has been deleted, so mon a should failover immediately
+	assert.True(t, c.shouldFailoverMonImmediately((context.TODO()), "a"))
 }
 
 func TestTrackMonsOutOfQuorum(t *testing.T) {