From cf13deee6f0aaa1ff854d57493bf91df70f8f436 Mon Sep 17 00:00:00 2001 From: Oded Viner Date: Thu, 17 Jul 2025 13:06:51 +0300 Subject: [PATCH] core: log panics in controller reconcile functions add RecoverAndLogException() helper to log panics with stack trace. added defer call in all rook controller Reconcile() methods for better error visibility in operator logs Signed-off-by: Oded Viner --- pkg/operator/ceph/client/controller.go | 1 + pkg/operator/ceph/cluster/controller.go | 1 + pkg/operator/ceph/cluster/nodedaemon/reconcile.go | 1 + pkg/operator/ceph/cluster/rbd/controller.go | 1 + pkg/operator/ceph/controller.go | 1 + pkg/operator/ceph/controller/controller_utils.go | 9 +++++++++ pkg/operator/ceph/csi/controller.go | 1 + .../ceph/disruption/clusterdisruption/reconcile.go | 2 ++ pkg/operator/ceph/file/controller.go | 1 + pkg/operator/ceph/file/mirror/controller.go | 1 + pkg/operator/ceph/file/subvolumegroup/controller.go | 1 + pkg/operator/ceph/nfs/controller.go | 1 + pkg/operator/ceph/object/bucket/controller.go | 1 + pkg/operator/ceph/object/controller.go | 1 + pkg/operator/ceph/object/cosi/controller.go | 1 + pkg/operator/ceph/object/notification/controller.go | 1 + .../ceph/object/notification/obc_label_controller.go | 1 + pkg/operator/ceph/object/realm/controller.go | 1 + pkg/operator/ceph/object/topic/controller.go | 1 + pkg/operator/ceph/object/user/controller.go | 1 + pkg/operator/ceph/object/zone/controller.go | 1 + pkg/operator/ceph/object/zonegroup/controller.go | 1 + pkg/operator/ceph/pool/controller.go | 1 + pkg/operator/ceph/pool/radosnamespace/controller.go | 1 + 24 files changed, 33 insertions(+) diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index 500132ef7..8b6703ca4 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -138,6 +138,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephClient) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephClient, err := r.reconcile(request) return reporting.ReportReconcileResult(logger, r.recorder, request, &cephClient, reconcileResponse, err) diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 0d3151a7a..43efca52f 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -244,6 +244,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephCluster) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephCluster, err := r.reconcile(request) diff --git a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go index 45ec02ac1..5fe180f2a 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go +++ b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go @@ -73,6 +73,7 @@ type ReconcileNode struct { // The Controller will requeue the Request to be processed again if an error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileNode) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface result, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 051b223bb..d8a1f7622 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -159,6 +159,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephRBDMirror, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/controller.go b/pkg/operator/ceph/controller.go index 19d1f206d..6d2f92a5b 100644 --- a/pkg/operator/ceph/controller.go +++ b/pkg/operator/ceph/controller.go @@ -93,6 +93,7 @@ func add(ctx context.Context, context *clusterd.Context, mgr manager.Manager, r // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileConfig) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index b40f8233e..4947a3b31 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "runtime/debug" "slices" "strconv" "strings" @@ -281,3 +282,11 @@ var ClusterResource = k8sutil.CustomResource{ Kind: reflect.TypeOf(cephv1.CephCluster{}).Name(), APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), } + +// RecoverAndLogException handles and logs panics from a controller Reconcile loop. +func RecoverAndLogException() { + if r := recover(); r != nil { + logger.Errorf("Panic: %v", r) + logger.Errorf("Stack trace:\n%s", string(debug.Stack())) + } +} diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index e94cd49e4..ccb9d392a 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -128,6 +128,7 @@ func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler, opCon // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go index 5a8aeb046..0751b03e5 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" "github.com/rook/rook/pkg/operator/k8sutil" ) @@ -68,6 +69,7 @@ type ReconcileClusterDisruption struct { // The Controller will requeue the Request to be processed again if an error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileClusterDisruption) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // wrapping reconcile because the rook logging mechanism is not compatible with the controller-runtime logging interface result, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index 76e7c2764..17cfe851d 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -180,6 +180,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephFilesystem) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephFilesystem, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/mirror/controller.go b/pkg/operator/ceph/file/mirror/controller.go index 2afc008ef..2d4d94bbe 100644 --- a/pkg/operator/ceph/file/mirror/controller.go +++ b/pkg/operator/ceph/file/mirror/controller.go @@ -150,6 +150,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileFilesystemMirror) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephFilesystemMirror, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index 4a62440b0..2afc76e32 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -138,6 +138,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephFilesystemSubVolumeGroup) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index ad121d4a1..8e19eb745 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -152,6 +152,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephNFS) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephNFS, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/bucket/controller.go b/pkg/operator/ceph/object/bucket/controller.go index 6cdd3365e..1daaadbb3 100644 --- a/pkg/operator/ceph/object/bucket/controller.go +++ b/pkg/operator/ceph/object/bucket/controller.go @@ -112,6 +112,7 @@ func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler) error // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileBucket) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 42e7cab82..6e91d1a73 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -269,6 +269,7 @@ func isObjStoreSpecContainsSecret(spec *cephv1.ObjectStoreSpec, secret *corev1.S // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephObjectStore) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, objectStore, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/cosi/controller.go b/pkg/operator/ceph/object/cosi/controller.go index 175fd4254..d2aacec7b 100644 --- a/pkg/operator/ceph/object/cosi/controller.go +++ b/pkg/operator/ceph/object/cosi/controller.go @@ -125,6 +125,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // Reconcile reads that state of the cluster for a CephCOSIDriver object and makes changes based on the state read // and what is in the CephCOSIDriver.Spec func (r *ReconcileCephCOSIDriver) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() reconcileResponse, cephCOSIDriver, err := r.reconcile(request) return reporting.ReportReconcileResult(logger, r.recorder, request, &cephCOSIDriver, reconcileResponse, err) diff --git a/pkg/operator/ceph/object/notification/controller.go b/pkg/operator/ceph/object/notification/controller.go index ec398d366..5577df881 100644 --- a/pkg/operator/ceph/object/notification/controller.go +++ b/pkg/operator/ceph/object/notification/controller.go @@ -121,6 +121,7 @@ func addNotificationReconciler(mgr manager.Manager, r reconcile.Reconciler) erro // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileNotifications) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() reconcileResponse, notification, err := r.reconcile(request) if err != nil { logger.Errorf("failed to reconcile %v", err) diff --git a/pkg/operator/ceph/object/notification/obc_label_controller.go b/pkg/operator/ceph/object/notification/obc_label_controller.go index 6c68c2732..8c7340166 100644 --- a/pkg/operator/ceph/object/notification/obc_label_controller.go +++ b/pkg/operator/ceph/object/notification/obc_label_controller.go @@ -130,6 +130,7 @@ func addOBCLabelReconciler(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileOBCLabels) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 966e7e58e..270b4e713 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -126,6 +126,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectRealm) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectRealm, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/topic/controller.go b/pkg/operator/ceph/object/topic/controller.go index 84d4d6a71..bba983c90 100644 --- a/pkg/operator/ceph/object/topic/controller.go +++ b/pkg/operator/ceph/object/topic/controller.go @@ -188,6 +188,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileBucketTopic) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index dd3d03483..a07e6ee4b 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -217,6 +217,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectStoreUser) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectStoreUser, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go index df70cb466..f5a3a9434 100644 --- a/pkg/operator/ceph/object/zone/controller.go +++ b/pkg/operator/ceph/object/zone/controller.go @@ -134,6 +134,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectZone) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectZone, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/zonegroup/controller.go b/pkg/operator/ceph/object/zonegroup/controller.go index 60420c956..083ba4fb0 100644 --- a/pkg/operator/ceph/object/zonegroup/controller.go +++ b/pkg/operator/ceph/object/zonegroup/controller.go @@ -117,6 +117,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectZoneGroup) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index 03e2c3047..7b44c840b 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -160,6 +160,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephBlockPool) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephBlockPool, err := r.reconcile(request) diff --git a/pkg/operator/ceph/pool/radosnamespace/controller.go b/pkg/operator/ceph/pool/radosnamespace/controller.go index 378b6d8b9..4543448eb 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller.go @@ -149,6 +149,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // processed again if the returned error is non-nil or Result.Requeue is true, // otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephBlockPoolRadosNamespace) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, radosNamespace, err := r.reconcile(request) if err != nil {