From 394b98cb0b0f04879d4d1b8706c3458a67ce0cf1 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 15 May 2020 12:24:33 -0600 Subject: [PATCH] ceph: cancel the host cleanup if pvc deletion pending During cluster delection, the finalizer will not be removed until all the PVCs are removed. Since this could wait for a long time, as long as the deletion event is re-queued, we need to cancel the goroutine that is watching for cluster deletion so it can avoid running multiple goroutines as the event runs again and again. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/cleanup.go | 33 ++++++++++++++----------- pkg/operator/ceph/cluster/controller.go | 4 ++- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/pkg/operator/ceph/cluster/cleanup.go b/pkg/operator/ceph/cluster/cleanup.go index 0b46f8f5b..d67fe3fa4 100644 --- a/pkg/operator/ceph/cluster/cleanup.go +++ b/pkg/operator/ceph/cluster/cleanup.go @@ -49,9 +49,9 @@ var ( monitorSecret = "ROOK_MON_SECRET" ) -func (c *ClusterController) startClusterCleanUp(cluster *cephv1.CephCluster, cephHosts []string, monSecret string) { +func (c *ClusterController) startClusterCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, cephHosts []string, monSecret string) { logger.Infof("starting clean up for cluster %q", cluster.Name) - err := c.waitForCephDaemonCleanUp(cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second) + err := c.waitForCephDaemonCleanUp(stopCleanupCh, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second) if err != nil { logger.Errorf("failed to wait till ceph daemons are destroyed. %v", err) return @@ -129,25 +129,28 @@ func (c *ClusterController) cleanUpJobTemplateSpec(cluster *cephv1.CephCluster, return podSpec } -func (c *ClusterController) waitForCephDaemonCleanUp(cluster *cephv1.CephCluster, retryInterval time.Duration) error { +func (c *ClusterController) waitForCephDaemonCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, retryInterval time.Duration) error { logger.Infof("waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace) for { - cephHosts, err := c.getCephHosts(cluster.Namespace) - if err != nil { - return errors.Wrap(err, "failed to list ceph daemon nodes") - } + select { + case <-time.After(retryInterval): + cephHosts, err := c.getCephHosts(cluster.Namespace) + if err != nil { + return errors.Wrap(err, "failed to list ceph daemon nodes") + } - if len(cephHosts) == 0 { - logger.Info("all ceph daemons are cleaned up") + if len(cephHosts) == 0 { + logger.Info("all ceph daemons are cleaned up") + return nil + } + + logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q", + cluster.Namespace, retryInterval.String()) break + case <-stopCleanupCh: + return errors.New("cancelling the host cleanup job") } - - logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q", - cluster.Namespace, retryInterval.String()) - <-time.After(retryInterval) } - - return nil } func (c *ClusterController) getCephHosts(namespace string) ([]string, error) { diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 150a22026..b1225a81b 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -265,6 +265,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R logger.Infof("deleting ceph cluster %q", cephCluster.Name) // Start cluster clean up only if cleanupPolicy is applied to the ceph cluster + stopCleanupCh := make(chan struct{}) if hasCleanupPolicy(cephCluster) { monSecret, err := r.clusterController.getMonSecret(cephCluster.Namespace) if err != nil { @@ -274,7 +275,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to find valid ceph hosts in the cluster %q", cephCluster.Namespace) } - go r.clusterController.startClusterCleanUp(cephCluster, cephHosts, monSecret) + go r.clusterController.startClusterCleanUp(stopCleanupCh, cephCluster, cephHosts, monSecret) } // Run delete sequence @@ -282,6 +283,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R if !ok { // If the cluster cannot be deleted, requeue the request for deletion to see if the conditions // will eventually be satisfied such as the volumes being removed + close(stopCleanupCh) return response, nil }