diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index bd4b944e8..272c77087 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -34,8 +34,11 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -300,6 +303,42 @@ func applyCephExporterLabels(cephCluster cephv1.CephCluster, serviceMonitor *mon } } +// deleteOrphanedExporterDeployments lists all ceph-exporter deployments in the given namespace +// and deletes any whose target node (stored in the node_name label) no longer exists in the cluster. +// This cleans up stale Pending exporter pods that were left behind when a node was removed while +// the operator was not running. +func (r *ReconcileNode) deleteOrphanedExporterDeployments(namespace string) error { + deploymentList := &appsv1.DeploymentList{} + err := r.client.List(r.opManagerContext, deploymentList, + client.MatchingLabels{k8sutil.AppAttr: cephExporterAppName}, + client.InNamespace(namespace), + ) + if err != nil { + return errors.Wrapf(err, "failed to list exporter deployments in namespace %q", namespace) + } + + for i := range deploymentList.Items { + d := deploymentList.Items[i] + nodeName, ok := d.Labels[NodeNameLabel] + if !ok { + continue + } + node := &corev1.Node{} + err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: nodeName}, node) + if err != nil { + if kerrors.IsNotFound(err) { + log.NamespacedInfo(namespace, logger, "deleting orphaned ceph-exporter deployment %q: target node %q no longer exists", d.Name, nodeName) + if delErr := r.deleteDeployment(d); delErr != nil { + return errors.Wrapf(delErr, "failed to delete orphaned exporter deployment %q in namespace %q", d.Name, namespace) + } + } else { + return errors.Wrapf(err, "failed to check existence of node %q for exporter deployment %q", nodeName, d.Name) + } + } + } + return nil +} + func applyPrometheusAnnotations(cephCluster cephv1.CephCluster, objectMeta *metav1.ObjectMeta) { if len(cephv1.GetCephExporterAnnotations(cephCluster.Spec.Annotations)) == 0 { t := cephv1.Annotations{ diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter_test.go b/pkg/operator/ceph/cluster/nodedaemon/exporter_test.go index bc3ed4a5c..935479cf0 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter_test.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter_test.go @@ -36,6 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -484,3 +485,101 @@ func TestApplyCephExporterLabels(t *testing.T) { applyCephExporterLabels(cephCluster, sm) assert.Nil(t, sm.Spec.Endpoints[0].RelabelConfigs) } + +func TestDeleteOrphanedExporterDeployments(t *testing.T) { + const namespace = "rook-ceph" + ctx := context.TODO() + + s := scheme.Scheme + err := appsv1.AddToScheme(s) + assert.NoError(t, err) + err = corev1.AddToScheme(s) + assert.NoError(t, err) + + // helper to build a ceph-exporter Deployment with an optional node_name label + makeExporterDeployment := func(name, nodeName string) *appsv1.Deployment { + labels := map[string]string{ + k8sutil.AppAttr: cephExporterAppName, + } + if nodeName != "" { + labels[NodeNameLabel] = nodeName + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + } + } + + makeNode := func(name string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + } + } + + t.Run("no deployments - no error", func(t *testing.T) { + r := &ReconcileNode{ + client: fake.NewClientBuilder().WithScheme(s).Build(), + opManagerContext: ctx, + } + assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace)) + }) + + t.Run("deployment whose node still exists is not deleted", func(t *testing.T) { + deploy := makeExporterDeployment("exporter-existing", "node1") + node := makeNode("node1") + r := &ReconcileNode{ + client: fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(deploy, node).Build(), + opManagerContext: ctx, + } + assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace)) + + remaining := &appsv1.DeploymentList{} + assert.NoError(t, r.client.List(ctx, remaining, client.InNamespace(namespace))) + assert.Len(t, remaining.Items, 1) + }) + + t.Run("deployment whose node no longer exists is deleted", func(t *testing.T) { + deploy := makeExporterDeployment("exporter-orphaned", "gone-node") + r := &ReconcileNode{ + client: fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(deploy).Build(), + opManagerContext: ctx, + } + assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace)) + + remaining := &appsv1.DeploymentList{} + assert.NoError(t, r.client.List(ctx, remaining, client.InNamespace(namespace))) + assert.Empty(t, remaining.Items) + }) + + t.Run("deployment without node_name label is skipped", func(t *testing.T) { + deploy := makeExporterDeployment("exporter-no-label", "") + r := &ReconcileNode{ + client: fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(deploy).Build(), + opManagerContext: ctx, + } + assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace)) + + remaining := &appsv1.DeploymentList{} + assert.NoError(t, r.client.List(ctx, remaining, client.InNamespace(namespace))) + assert.Len(t, remaining.Items, 1) + }) + + t.Run("mixed: one orphaned one healthy deployment", func(t *testing.T) { + deployOrphaned := makeExporterDeployment("exporter-orphaned", "gone-node") + deployHealthy := makeExporterDeployment("exporter-healthy", "live-node") + node := makeNode("live-node") + r := &ReconcileNode{ + client: fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(deployOrphaned, deployHealthy, node).Build(), + opManagerContext: ctx, + } + assert.NoError(t, r.deleteOrphanedExporterDeployments(namespace)) + + remaining := &appsv1.DeploymentList{} + assert.NoError(t, r.client.List(ctx, remaining, client.InNamespace(namespace))) + assert.Len(t, remaining.Items, 1) + assert.Equal(t, "exporter-healthy", remaining.Items[0].Name) + }) +} diff --git a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go index 5dbf0f343..478fcfb5e 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go +++ b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go @@ -59,6 +59,13 @@ var ( // wait for secret "rook-ceph-crash-collector-keyring" to be created waitForRequeueIfSecretNotCreated = reconcile.Result{Requeue: true, RequeueAfter: 30 * time.Second} + + // exporterOrphanCheckDone tracks namespaces where the one-time orphaned + // exporter deployment cleanup has already run. The check only needs to + // happen once per operator lifetime – stale deployments can only + // accumulate while the operator is offline, so a single pass on startup + // is sufficient. + exporterOrphanCheckDone = map[string]bool{} ) // ReconcileNode reconciles ReplicaSets @@ -178,6 +185,18 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, } } + // Clean up any exporter deployments whose target node no longer exists. + // This only needs to run once per operator lifetime: stale deployments + // can only accumulate while the operator is offline, so a single pass + // on startup is sufficient. Skip subsequent reconciles to avoid the + // extra API calls, especially in large clusters. + if !exporterOrphanCheckDone[namespace] { + if err := r.deleteOrphanedExporterDeployments(namespace); err != nil { + log.NamespacedError(namespace, logger, "failed to clean up orphaned ceph-exporter deployments. %v", err) + } + exporterOrphanCheckDone[namespace] = true + } + // If the node has Ceph pods we create the daemons tolerations := uniqueTolerations.ToList() if hasCephPods {