From ee0bae1a6cefbe22359e07a2c7fb6a3d791645bb Mon Sep 17 00:00:00 2001 From: Oded Viner Date: Thu, 17 Jul 2025 13:06:51 +0300 Subject: [PATCH] core: log panics in controller reconcile functions add RecoverAndLogException() helper to log panics with stack trace. added defer call in all rook controller Reconcile() methods for better error visibility in operator logs Signed-off-by: Oded Viner (cherry picked from commit cf13deee6f0aaa1ff854d57493bf91df70f8f436) --- pkg/operator/ceph/client/controller.go | 1 + pkg/operator/ceph/cluster/controller.go | 1 + pkg/operator/ceph/cluster/nodedaemon/reconcile.go | 1 + pkg/operator/ceph/cluster/rbd/controller.go | 1 + pkg/operator/ceph/controller.go | 1 + pkg/operator/ceph/controller/controller_utils.go | 9 +++++++++ pkg/operator/ceph/csi/controller.go | 1 + .../ceph/disruption/clusterdisruption/reconcile.go | 2 ++ pkg/operator/ceph/file/controller.go | 1 + pkg/operator/ceph/file/mirror/controller.go | 1 + pkg/operator/ceph/file/subvolumegroup/controller.go | 1 + pkg/operator/ceph/nfs/controller.go | 1 + pkg/operator/ceph/object/bucket/controller.go | 1 + pkg/operator/ceph/object/controller.go | 1 + pkg/operator/ceph/object/cosi/controller.go | 1 + pkg/operator/ceph/object/notification/controller.go | 1 + .../ceph/object/notification/obc_label_controller.go | 1 + pkg/operator/ceph/object/realm/controller.go | 1 + pkg/operator/ceph/object/topic/controller.go | 1 + pkg/operator/ceph/object/user/controller.go | 1 + pkg/operator/ceph/object/zone/controller.go | 1 + pkg/operator/ceph/object/zonegroup/controller.go | 1 + pkg/operator/ceph/pool/controller.go | 1 + pkg/operator/ceph/pool/radosnamespace/controller.go | 1 + 24 files changed, 33 insertions(+) diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index 1c36a46a7..86707ffdc 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -135,6 +135,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephClient) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephClient, err := r.reconcile(request) return reporting.ReportReconcileResult(logger, r.recorder, request, &cephClient, reconcileResponse, err) diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 7ed598cfa..a3498205d 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -244,6 +244,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephCluster) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephCluster, err := r.reconcile(request) diff --git a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go index 45ec02ac1..5fe180f2a 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go +++ b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go @@ -73,6 +73,7 @@ type ReconcileNode struct { // The Controller will requeue the Request to be processed again if an error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileNode) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface result, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 051b223bb..d8a1f7622 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -159,6 +159,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephRBDMirror, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/controller.go b/pkg/operator/ceph/controller.go index 19d1f206d..6d2f92a5b 100644 --- a/pkg/operator/ceph/controller.go +++ b/pkg/operator/ceph/controller.go @@ -93,6 +93,7 @@ func add(ctx context.Context, context *clusterd.Context, mgr manager.Manager, r // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileConfig) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index b40f8233e..4947a3b31 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "runtime/debug" "slices" "strconv" "strings" @@ -281,3 +282,11 @@ var ClusterResource = k8sutil.CustomResource{ Kind: reflect.TypeOf(cephv1.CephCluster{}).Name(), APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), } + +// RecoverAndLogException handles and logs panics from a controller Reconcile loop. +func RecoverAndLogException() { + if r := recover(); r != nil { + logger.Errorf("Panic: %v", r) + logger.Errorf("Stack trace:\n%s", string(debug.Stack())) + } +} diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index ffebd3464..66787dc78 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -128,6 +128,7 @@ func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler, opCon // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go index 5a8aeb046..0751b03e5 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" "github.com/rook/rook/pkg/operator/k8sutil" ) @@ -68,6 +69,7 @@ type ReconcileClusterDisruption struct { // The Controller will requeue the Request to be processed again if an error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileClusterDisruption) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // wrapping reconcile because the rook logging mechanism is not compatible with the controller-runtime logging interface result, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index 76e7c2764..17cfe851d 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -180,6 +180,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephFilesystem) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephFilesystem, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/mirror/controller.go b/pkg/operator/ceph/file/mirror/controller.go index 2afc008ef..2d4d94bbe 100644 --- a/pkg/operator/ceph/file/mirror/controller.go +++ b/pkg/operator/ceph/file/mirror/controller.go @@ -150,6 +150,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileFilesystemMirror) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephFilesystemMirror, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index 4a62440b0..2afc76e32 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -138,6 +138,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephFilesystemSubVolumeGroup) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index ad121d4a1..8e19eb745 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -152,6 +152,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephNFS) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephNFS, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/bucket/controller.go b/pkg/operator/ceph/object/bucket/controller.go index 6cdd3365e..1daaadbb3 100644 --- a/pkg/operator/ceph/object/bucket/controller.go +++ b/pkg/operator/ceph/object/bucket/controller.go @@ -112,6 +112,7 @@ func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler) error // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileBucket) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index e81899926..282ef1519 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -268,6 +268,7 @@ func isObjStoreSpecContainsSecret(spec *cephv1.ObjectStoreSpec, secret *corev1.S // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephObjectStore) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, objectStore, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/cosi/controller.go b/pkg/operator/ceph/object/cosi/controller.go index 175fd4254..d2aacec7b 100644 --- a/pkg/operator/ceph/object/cosi/controller.go +++ b/pkg/operator/ceph/object/cosi/controller.go @@ -125,6 +125,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // Reconcile reads that state of the cluster for a CephCOSIDriver object and makes changes based on the state read // and what is in the CephCOSIDriver.Spec func (r *ReconcileCephCOSIDriver) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() reconcileResponse, cephCOSIDriver, err := r.reconcile(request) return reporting.ReportReconcileResult(logger, r.recorder, request, &cephCOSIDriver, reconcileResponse, err) diff --git a/pkg/operator/ceph/object/notification/controller.go b/pkg/operator/ceph/object/notification/controller.go index ec398d366..5577df881 100644 --- a/pkg/operator/ceph/object/notification/controller.go +++ b/pkg/operator/ceph/object/notification/controller.go @@ -121,6 +121,7 @@ func addNotificationReconciler(mgr manager.Manager, r reconcile.Reconciler) erro // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileNotifications) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() reconcileResponse, notification, err := r.reconcile(request) if err != nil { logger.Errorf("failed to reconcile %v", err) diff --git a/pkg/operator/ceph/object/notification/obc_label_controller.go b/pkg/operator/ceph/object/notification/obc_label_controller.go index 6c68c2732..8c7340166 100644 --- a/pkg/operator/ceph/object/notification/obc_label_controller.go +++ b/pkg/operator/ceph/object/notification/obc_label_controller.go @@ -130,6 +130,7 @@ func addOBCLabelReconciler(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileOBCLabels) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 966e7e58e..270b4e713 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -126,6 +126,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectRealm) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectRealm, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/topic/controller.go b/pkg/operator/ceph/object/topic/controller.go index 84d4d6a71..bba983c90 100644 --- a/pkg/operator/ceph/object/topic/controller.go +++ b/pkg/operator/ceph/object/topic/controller.go @@ -188,6 +188,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileBucketTopic) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index dd3d03483..a07e6ee4b 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -217,6 +217,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectStoreUser) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectStoreUser, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go index df70cb466..f5a3a9434 100644 --- a/pkg/operator/ceph/object/zone/controller.go +++ b/pkg/operator/ceph/object/zone/controller.go @@ -134,6 +134,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectZone) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectZone, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/zonegroup/controller.go b/pkg/operator/ceph/object/zonegroup/controller.go index 60420c956..083ba4fb0 100644 --- a/pkg/operator/ceph/object/zonegroup/controller.go +++ b/pkg/operator/ceph/object/zonegroup/controller.go @@ -117,6 +117,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectZoneGroup) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index efeaf6580..c398f28e5 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -160,6 +160,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephBlockPool) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephBlockPool, err := r.reconcile(request) diff --git a/pkg/operator/ceph/pool/radosnamespace/controller.go b/pkg/operator/ceph/pool/radosnamespace/controller.go index 378b6d8b9..4543448eb 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller.go @@ -149,6 +149,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // processed again if the returned error is non-nil or Result.Requeue is true, // otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephBlockPoolRadosNamespace) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, radosNamespace, err := r.reconcile(request) if err != nil {