diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index 1c36a46a7..86707ffdc 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -135,6 +135,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephClient) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephClient, err := r.reconcile(request) return reporting.ReportReconcileResult(logger, r.recorder, request, &cephClient, reconcileResponse, err) diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 7ed598cfa..a3498205d 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -244,6 +244,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephCluster) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephCluster, err := r.reconcile(request) diff --git a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go index 45ec02ac1..5fe180f2a 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go +++ b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go @@ -73,6 +73,7 @@ type ReconcileNode struct { // The Controller will requeue the Request to be processed again if an error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileNode) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface result, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 051b223bb..d8a1f7622 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -159,6 +159,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephRBDMirror, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/controller.go b/pkg/operator/ceph/controller.go index 19d1f206d..6d2f92a5b 100644 --- a/pkg/operator/ceph/controller.go +++ b/pkg/operator/ceph/controller.go @@ -93,6 +93,7 @@ func add(ctx context.Context, context *clusterd.Context, mgr manager.Manager, r // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileConfig) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index b40f8233e..4947a3b31 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "runtime/debug" "slices" "strconv" "strings" @@ -281,3 +282,11 @@ var ClusterResource = k8sutil.CustomResource{ Kind: reflect.TypeOf(cephv1.CephCluster{}).Name(), APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), } + +// RecoverAndLogException handles and logs panics from a controller Reconcile loop. +func RecoverAndLogException() { + if r := recover(); r != nil { + logger.Errorf("Panic: %v", r) + logger.Errorf("Stack trace:\n%s", string(debug.Stack())) + } +} diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index ffebd3464..66787dc78 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -128,6 +128,7 @@ func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler, opCon // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go index 5a8aeb046..0751b03e5 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" "github.com/rook/rook/pkg/operator/k8sutil" ) @@ -68,6 +69,7 @@ type ReconcileClusterDisruption struct { // The Controller will requeue the Request to be processed again if an error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileClusterDisruption) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // wrapping reconcile because the rook logging mechanism is not compatible with the controller-runtime logging interface result, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index 76e7c2764..17cfe851d 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -180,6 +180,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephFilesystem) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephFilesystem, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/mirror/controller.go b/pkg/operator/ceph/file/mirror/controller.go index 2afc008ef..2d4d94bbe 100644 --- a/pkg/operator/ceph/file/mirror/controller.go +++ b/pkg/operator/ceph/file/mirror/controller.go @@ -150,6 +150,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileFilesystemMirror) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephFilesystemMirror, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index 4a62440b0..2afc76e32 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -138,6 +138,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephFilesystemSubVolumeGroup) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index ad121d4a1..8e19eb745 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -152,6 +152,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephNFS) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephNFS, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/bucket/controller.go b/pkg/operator/ceph/object/bucket/controller.go index 6cdd3365e..1daaadbb3 100644 --- a/pkg/operator/ceph/object/bucket/controller.go +++ b/pkg/operator/ceph/object/bucket/controller.go @@ -112,6 +112,7 @@ func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler) error // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileBucket) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index e81899926..282ef1519 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -268,6 +268,7 @@ func isObjStoreSpecContainsSecret(spec *cephv1.ObjectStoreSpec, secret *corev1.S // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephObjectStore) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, objectStore, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/cosi/controller.go b/pkg/operator/ceph/object/cosi/controller.go index 175fd4254..d2aacec7b 100644 --- a/pkg/operator/ceph/object/cosi/controller.go +++ b/pkg/operator/ceph/object/cosi/controller.go @@ -125,6 +125,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // Reconcile reads that state of the cluster for a CephCOSIDriver object and makes changes based on the state read // and what is in the CephCOSIDriver.Spec func (r *ReconcileCephCOSIDriver) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() reconcileResponse, cephCOSIDriver, err := r.reconcile(request) return reporting.ReportReconcileResult(logger, r.recorder, request, &cephCOSIDriver, reconcileResponse, err) diff --git a/pkg/operator/ceph/object/notification/controller.go b/pkg/operator/ceph/object/notification/controller.go index ec398d366..5577df881 100644 --- a/pkg/operator/ceph/object/notification/controller.go +++ b/pkg/operator/ceph/object/notification/controller.go @@ -121,6 +121,7 @@ func addNotificationReconciler(mgr manager.Manager, r reconcile.Reconciler) erro // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileNotifications) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() reconcileResponse, notification, err := r.reconcile(request) if err != nil { logger.Errorf("failed to reconcile %v", err) diff --git a/pkg/operator/ceph/object/notification/obc_label_controller.go b/pkg/operator/ceph/object/notification/obc_label_controller.go index 6c68c2732..8c7340166 100644 --- a/pkg/operator/ceph/object/notification/obc_label_controller.go +++ b/pkg/operator/ceph/object/notification/obc_label_controller.go @@ -130,6 +130,7 @@ func addOBCLabelReconciler(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileOBCLabels) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 966e7e58e..270b4e713 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -126,6 +126,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectRealm) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectRealm, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/topic/controller.go b/pkg/operator/ceph/object/topic/controller.go index 84d4d6a71..bba983c90 100644 --- a/pkg/operator/ceph/object/topic/controller.go +++ b/pkg/operator/ceph/object/topic/controller.go @@ -188,6 +188,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileBucketTopic) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index dd3d03483..a07e6ee4b 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -217,6 +217,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectStoreUser) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectStoreUser, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go index df70cb466..f5a3a9434 100644 --- a/pkg/operator/ceph/object/zone/controller.go +++ b/pkg/operator/ceph/object/zone/controller.go @@ -134,6 +134,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectZone) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephObjectZone, err := r.reconcile(request) diff --git a/pkg/operator/ceph/object/zonegroup/controller.go b/pkg/operator/ceph/object/zonegroup/controller.go index 60420c956..083ba4fb0 100644 --- a/pkg/operator/ceph/object/zonegroup/controller.go +++ b/pkg/operator/ceph/object/zonegroup/controller.go @@ -117,6 +117,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileObjectZoneGroup) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index efeaf6580..c398f28e5 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -160,6 +160,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephBlockPool) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephBlockPool, err := r.reconcile(request) diff --git a/pkg/operator/ceph/pool/radosnamespace/controller.go b/pkg/operator/ceph/pool/radosnamespace/controller.go index 378b6d8b9..4543448eb 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller.go @@ -149,6 +149,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // processed again if the returned error is non-nil or Result.Requeue is true, // otherwise upon completion it will remove the work from the queue. func (r *ReconcileCephBlockPoolRadosNamespace) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, radosNamespace, err := r.reconcile(request) if err != nil {