Merge pull request #16821 from rook/mergify/bp/release-1.19/pr-16786

operator: watch cephConfigFromSecret changes (backport #16786)

pkg/operator/ceph/cluster/controller.go

@@ -143,7 +143,19 @@ func watchOwnedCoreObject[T client.Object](c controller.Controller, mgr manager.
 	)
 }
 
-func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context, opConfig opcontroller.OperatorConfig) error {
+// isSecretRefFromCluster checks if the secret name is referenced in the CephCluster cephConfigFromSecret field.
+func isSecretRefFromCluster(secretName string, clusterSpec cephv1.ClusterSpec) bool {
+	for _, secretKeyMap := range clusterSpec.CephConfigFromSecret {
+		for _, keySelector := range secretKeyMap {
+			if secretName == keySelector.Name {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, clusterdContext *clusterd.Context, opConfig opcontroller.OperatorConfig) error {
 	concurrentClusters := os.Getenv("ROOK_RECONCILE_CONCURRENT_CLUSTERS")
 	concurrentReconciles := 1
 	if i, err := strconv.Atoi(concurrentClusters); err == nil && i > 1 {
@@ -207,7 +219,43 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco
 			mgr.GetCache(),
 			&corev1.Node{TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: corev1.SchemeGroupVersion.String()}},
 			handler.TypedEnqueueRequestsFromMapFunc(nodeHandler),
-			predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context, opConfig.OperatorNamespace),
+			predicateForNodeWatcher(opManagerContext, mgr.GetClient(), clusterdContext, opConfig.OperatorNamespace),
+		),
+	)
+	if err != nil {
+		return err
+	}
+
+	// Watch for changes to secrets referenced in ClusterSpec.CephConfigFromSecret
+	err = c.Watch(
+		source.Kind(
+			mgr.GetCache(),
+			&corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: corev1.SchemeGroupVersion.String()}},
+			handler.TypedEnqueueRequestsFromMapFunc(
+				func(ctx context.Context, secret *corev1.Secret) []reconcile.Request {
+					clusterList := cephv1.CephClusterList{}
+					err := mgr.GetClient().List(ctx, &clusterList)
+					if err != nil {
+						return nil
+					}
+					requests := []reconcile.Request{}
+					for _, clusterResource := range clusterList.Items {
+						// No more than 1 cluster can exist in a namespace, and all secrets referenced by a
+						// CephConfigFromSecret must be in the same namespace as the cluster. We only need to
+						// trigger a reconcile once, so we only need to find one match per cluster.
+						if secret.GetNamespace() == clusterResource.GetNamespace() && isSecretRefFromCluster(secret.GetName(), clusterResource.Spec) {
+							requests = append(requests, reconcile.Request{
+								NamespacedName: client.ObjectKey{
+									Namespace: clusterResource.GetNamespace(),
+									Name:      clusterResource.GetName(),
+								},
+							})
+						}
+					}
+					return requests
+				},
+			),
+			changedOrDeleted,
 		),
 	)
 	if err != nil {

pkg/operator/ceph/cluster/predicate.go

@@ -36,6 +36,18 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
 
+// Always trigger a reconcile when a secret is deleted. This will cause a
+// reconciliation failure to happen immediately in hopes of alerting the end
+// user to the configuration problem.
+var changedOrDeleted = predicate.Or(
+	predicate.TypedResourceVersionChangedPredicate[*corev1.Secret]{},
+	predicate.TypedFuncs[*corev1.Secret]{
+		DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Secret]) bool {
+			return true
+		},
+	},
+)
+
 func shouldReconcileChangedNode(objOld, objNew *corev1.Node) bool {
 	// do not watch node if only resourceversion got changed
 	resourceQtyComparer := cmpopts.IgnoreFields(v1.ObjectMeta{}, "ResourceVersion")