From ab9475220c69156c7f0949e81cecbd9eeb78e36b Mon Sep 17 00:00:00 2001 From: Elias Carter Date: Wed, 2 Jul 2025 18:41:24 +0000 Subject: [PATCH] osd: treat non existing OSD nodes as drained If the node for an OSD does not exist within kubernetes, the PDB reconciliation logic will fail to make progress. This can be problematic as new OSDs added to the cluster may not be protected by OSD PDBs after they are added since the PDB reconcilation logic is failing. The solution is to treat non existing OSD nodes as drained which is logically consistent with treating unschedulable OSD nodes as drained as you cannot schedule a pod onto a node which does not exist. Fixes: #16086 Signed-off-by: Elias Carter (cherry picked from commit 5f2d6c01d770a7088e26af33c3e0f3dd953e4e31) --- .../ceph/disruption/clusterdisruption/osd.go | 3 ++ .../disruption/clusterdisruption/osd_test.go | 33 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd.go b/pkg/operator/ceph/disruption/clusterdisruption/osd.go index 5a4979e47..42e46d18e 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/osd.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd.go @@ -490,6 +490,9 @@ func (r *ReconcileClusterDisruption) getOSDFailureDomains(clusterInfo *cephclien // hasOSDNodeDrained returns true if OSD pod is not assigned to any node or if the OSD node is not schedulable func hasOSDNodeDrained(ctx context.Context, c client.Client, osdNodeName string) (bool, error) { node, err := getNode(ctx, c, osdNodeName) + if apierrors.IsNotFound(err) { + return true, nil + } if err != nil { return false, errors.Wrapf(err, "failed to get node %q", osdNodeName) } diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go b/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go index 754e6edfd..e1ffa441f 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go @@ -181,6 +181,30 @@ func TestGetOSDFailureDomains(t *testing.T) { expectedDrainingFailureDomains: []string{"zone-3"}, expectedDownOSDs: []int{3}, }, + { + name: "case 6: osd in zone-3 is pending and the osd node does not exist", + osds: []appsv1.Deployment{ + fakeOSDDeployment(1, 1), fakeOSDDeployment(2, 1), + fakeOSDDeployment(3, 0), + }, + nodes: []corev1.Node{*getNodeObject("node-1", false), *getNodeObject("node-2", false)}, + expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, + expectedOsdDownFailureDomains: []string{"zone-3"}, + expectedDrainingFailureDomains: []string{"zone-3"}, + expectedDownOSDs: []int{3}, + }, + { + name: "case 7: osd in zone-1 and zone-2 are pending and node does not exist", + osds: []appsv1.Deployment{ + fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 0), + fakeOSDDeployment(3, 1), + }, + nodes: []corev1.Node{*getNodeObject("node-3", false)}, + expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, + expectedOsdDownFailureDomains: []string{"zone-1", "zone-2"}, + expectedDrainingFailureDomains: []string{"zone-1", "zone-2"}, + expectedDownOSDs: []int{1, 2}, + }, } for _, tc := range testcases { @@ -426,11 +450,18 @@ func TestHasNodeDrained(t *testing.T) { assert.False(t, expected) // Expecting node drain because OSD pod is assigned to an unschedulable node - osdDeployment = fakeOSDDeployment(2, 2) + osdDeployment = fakeOSDDeployment(2, 0) r = getFakeReconciler(t, getNodeObject("node-2", true), osdDeployment.DeepCopy(), &corev1.ConfigMap{}) expected, err = hasOSDNodeDrained(ctx, r.client, "node-2") assert.NoError(t, err) assert.True(t, expected) + + // Expecting node drain because OSD pod is assigned to a non existent node + osdDeployment = fakeOSDDeployment(3, 0) + r = getFakeReconciler(t, osdDeployment.DeepCopy(), &corev1.ConfigMap{}) + expected, err = hasOSDNodeDrained(ctx, r.client, "node-3") + assert.NoError(t, err) + assert.True(t, expected) } func TestSetPDBConfig(t *testing.T) {