From c39c1c7ddfa96b0296fedf3f6b50690f62b23ea6 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 1 Jul 2021 17:12:56 -0600 Subject: [PATCH] ceph: retry reconcile immediately after cancellation If the reconcile is cancelled due to a CR update, we want to retry the next reconcile immediately rather than wait for the exponential backoff timeout if the reconcile was already failing. The wait can easily be minutes if the reconcile was in this state, which makes it appear the operator is ignoring the request to start a new reconcile. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/controller.go | 5 +++++ pkg/operator/ceph/controller/controller_utils.go | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 9411c69df..7769f4d88 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "os" + "strings" "sync" "time" @@ -224,6 +225,10 @@ func add(mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context) func (r *ReconcileCephCluster) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephCluster, err := r.reconcile(request) + if err != nil && strings.Contains(err.Error(), opcontroller.CancellingOrchestrationMessage) { + logger.Infof("Cluster update requested. %s", opcontroller.CancellingOrchestrationMessage) + return opcontroller.ImmediateRetryResultNoBackoff, nil + } return reporting.ReportReconcileResult(logger, r.clusterController.recorder, cephCluster, reconcileResponse, err) diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index e27dab61d..7556e539b 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -43,12 +43,19 @@ const ( // OperatorNotInitializedMessage is the message we print when the Operator is not ready to reconcile, typically the ceph.conf has not been generated yet OperatorNotInitializedMessage = "skipping reconcile since operator is still initializing" + + // CancellingOrchestrationMessage is the message to indicate a reconcile was cancelled + CancellingOrchestrationMessage = "CANCELLING CURRENT ORCHESTRATION" ) var ( // ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object. ImmediateRetryResult = reconcile.Result{Requeue: true} + // ImmediateRetryResultNoBackoff Return this for a immediate retry of the reconciliation loop with the same request object. + // Override the exponential backoff behavior by setting the RequeueAfter time explicitly. + ImmediateRetryResultNoBackoff = reconcile.Result{Requeue: true, RequeueAfter: time.Second} + // WaitForRequeueIfCephClusterNotReady waits for the CephCluster to be ready WaitForRequeueIfCephClusterNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} @@ -80,7 +87,7 @@ func CheckForCancelledOrchestration(context *clusterd.Context) error { // Check whether we need to cancel the orchestration if context.RequestCancelOrchestration.IsSet() { - return errors.New("CANCELLING CURRENT ORCHESTRATION") + return errors.New(CancellingOrchestrationMessage) } return nil