Merge pull request #17179 from rook/mergify/bp/release-1.19/pr-17165

exporter: delete orphaned ceph-exporter deployments on reconcile (backport #17165)

pkg/operator/ceph/cluster/nodedaemon/exporter.go

@@ -34,8 +34,11 @@ import (
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
@@ -300,6 +303,42 @@ func applyCephExporterLabels(cephCluster cephv1.CephCluster, serviceMonitor *mon
 	}
 }
 
+// deleteOrphanedExporterDeployments lists all ceph-exporter deployments in the given namespace
+// and deletes any whose target node (stored in the node_name label) no longer exists in the cluster.
+// This cleans up stale Pending exporter pods that were left behind when a node was removed while
+// the operator was not running.
+func (r *ReconcileNode) deleteOrphanedExporterDeployments(namespace string) error {
+	deploymentList := &appsv1.DeploymentList{}
+	err := r.client.List(r.opManagerContext, deploymentList,
+		client.MatchingLabels{k8sutil.AppAttr: cephExporterAppName},
+		client.InNamespace(namespace),
+	)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list exporter deployments in namespace %q", namespace)
+	}
+
+	for i := range deploymentList.Items {
+		d := deploymentList.Items[i]
+		nodeName, ok := d.Labels[NodeNameLabel]
+		if !ok {
+			continue
+		}
+		node := &corev1.Node{}
+		err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: nodeName}, node)
+		if err != nil {
+			if kerrors.IsNotFound(err) {
+				log.NamespacedInfo(namespace, logger, "deleting orphaned ceph-exporter deployment %q: target node %q no longer exists", d.Name, nodeName)
+				if delErr := r.deleteDeployment(d); delErr != nil {
+					return errors.Wrapf(delErr, "failed to delete orphaned exporter deployment %q in namespace %q", d.Name, namespace)
+				}
+			} else {
+				return errors.Wrapf(err, "failed to check existence of node %q for exporter deployment %q", nodeName, d.Name)
+			}
+		}
+	}
+	return nil
+}
+
 func applyPrometheusAnnotations(cephCluster cephv1.CephCluster, objectMeta *metav1.ObjectMeta) {
 	if len(cephv1.GetCephExporterAnnotations(cephCluster.Spec.Annotations)) == 0 {
 		t := cephv1.Annotations{

pkg/operator/ceph/cluster/nodedaemon/exporter_test.go

@@ -36,6 +36,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
@@ -484,3 +485,101 @@ func TestApplyCephExporterLabels(t *testing.T) {
 	applyCephExporterLabels(cephCluster, sm)
 	assert.Nil(t, sm.Spec.Endpoints[0].RelabelConfigs)
 }
+
+func TestDeleteOrphanedExporterDeployments(t *testing.T) {
+	const namespace = "rook-ceph"
+	ctx := context.TODO()
+
+	s := scheme.Scheme
+	err := appsv1.AddToScheme(s)
+	assert.NoError(t, err)
+	err = corev1.AddToScheme(s)
+	assert.NoError(t, err)
+
+	// helper to build a ceph-exporter Deployment with an optional node_name label
+	makeExporterDeployment := func(name, nodeName string) *appsv1.Deployment {
+		labels := map[string]string{
+			k8sutil.AppAttr: cephExporterAppName,
+		}
+		if nodeName != "" {
+			labels[NodeNameLabel] = nodeName
+		}
+		return &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: namespace,
+				Labels:    labels,
+			},
+		}
+	}
+
+	makeNode := func(name string) *corev1.Node {
+		return &corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{Name: name},
+		}
+	}
+
+	t.Run("no deployments - no error", func(t *testing.T) {
+		r := &ReconcileNode{
+			client:           fake.NewClientBuilder().WithScheme(s).Build(),
+			opManagerContext: ctx,
+		}
+		assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace))
+	})
+
+	t.Run("deployment whose node still exists is not deleted", func(t *testing.T) {
+		deploy := makeExporterDeployment("exporter-existing", "node1")
+		node := makeNode("node1")
+		r := &ReconcileNode{
+			client:           fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(deploy, node).Build(),
+			opManagerContext: ctx,
+		}
+		assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace))
+
+		remaining := &appsv1.DeploymentList{}
+		assert.NoError(t, r.client.List(ctx, remaining, client.InNamespace(namespace)))
+		assert.Len(t, remaining.Items, 1)
+	})
+
+	t.Run("deployment whose node no longer exists is deleted", func(t *testing.T) {
+		deploy := makeExporterDeployment("exporter-orphaned", "gone-node")
+		r := &ReconcileNode{
+			client:           fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(deploy).Build(),
+			opManagerContext: ctx,
+		}
+		assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace))
+
+		remaining := &appsv1.DeploymentList{}
+		assert.NoError(t, r.client.List(ctx, remaining, client.InNamespace(namespace)))
+		assert.Empty(t, remaining.Items)
+	})
+
+	t.Run("deployment without node_name label is skipped", func(t *testing.T) {
+		deploy := makeExporterDeployment("exporter-no-label", "")
+		r := &ReconcileNode{
+			client:           fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(deploy).Build(),
+			opManagerContext: ctx,
+		}
+		assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace))
+
+		remaining := &appsv1.DeploymentList{}
+		assert.NoError(t, r.client.List(ctx, remaining, client.InNamespace(namespace)))
+		assert.Len(t, remaining.Items, 1)
+	})
+
+	t.Run("mixed: one orphaned one healthy deployment", func(t *testing.T) {
+		deployOrphaned := makeExporterDeployment("exporter-orphaned", "gone-node")
+		deployHealthy := makeExporterDeployment("exporter-healthy", "live-node")
+		node := makeNode("live-node")
+		r := &ReconcileNode{
+			client:           fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(deployOrphaned, deployHealthy, node).Build(),
+			opManagerContext: ctx,
+		}
+		assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace))
+
+		remaining := &appsv1.DeploymentList{}
+		assert.NoError(t, r.client.List(ctx, remaining, client.InNamespace(namespace)))
+		assert.Len(t, remaining.Items, 1)
+		assert.Equal(t, "exporter-healthy", remaining.Items[0].Name)
+	})
+}

pkg/operator/ceph/cluster/nodedaemon/reconcile.go

@@ -59,6 +59,13 @@ var (
 
 	// wait for secret "rook-ceph-crash-collector-keyring" to be created
 	waitForRequeueIfSecretNotCreated = reconcile.Result{Requeue: true, RequeueAfter: 30 * time.Second}
+
+	// exporterOrphanCheckDone tracks namespaces where the one-time orphaned
+	// exporter deployment cleanup has already run. The check only needs to
+	// happen once per operator lifetime – stale deployments can only
+	// accumulate while the operator is offline, so a single pass on startup
+	// is sufficient.
+	exporterOrphanCheckDone = map[string]bool{}
 )
 
 // ReconcileNode reconciles ReplicaSets
@@ -178,6 +185,18 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
 			}
 		}
 
+		// Clean up any exporter deployments whose target node no longer exists.
+		// This only needs to run once per operator lifetime: stale deployments
+		// can only accumulate while the operator is offline, so a single pass
+		// on startup is sufficient. Skip subsequent reconciles to avoid the
+		// extra API calls, especially in large clusters.
+		if !exporterOrphanCheckDone[namespace] {
+			if err := r.deleteOrphanedExporterDeployments(namespace); err != nil {
+				log.NamespacedError(namespace, logger, "failed to clean up orphaned ceph-exporter deployments. %v", err)
+			}
+			exporterOrphanCheckDone[namespace] = true
+		}
+
 		// If the node has Ceph pods we create the daemons
 		tolerations := uniqueTolerations.ToList()
 		if hasCephPods {