ceph: cancel the host cleanup if pvc deletion pending

During cluster delection, the finalizer will not be removed until
all the PVCs are removed. Since this could wait for a long time,
as long as the deletion event is re-queued, we need to cancel
the goroutine that is watching for cluster deletion so it can
avoid running multiple goroutines as the event runs again
and again.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2020-05-18 13:07:35 -06:00
1 parent 4173bee7c1
commit 394b98cb0b
2 files changed
+21 -16

No files matched your search

+18 -15
View File
@@ -49,9 +49,9 @@ var (
monitorSecret = "ROOK_MON_SECRET"
)
func (c *ClusterController) startClusterCleanUp(cluster *cephv1.CephCluster, cephHosts []string, monSecret string) {
func (c *ClusterController) startClusterCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, cephHosts []string, monSecret string) {
logger.Infof("starting clean up for cluster %q", cluster.Name)
err := c.waitForCephDaemonCleanUp(cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second)
err := c.waitForCephDaemonCleanUp(stopCleanupCh, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second)
if err != nil {
logger.Errorf("failed to wait till ceph daemons are destroyed. %v", err)
return
@@ -129,25 +129,28 @@ func (c *ClusterController) cleanUpJobTemplateSpec(cluster *cephv1.CephCluster,
return podSpec
}
func (c *ClusterController) waitForCephDaemonCleanUp(cluster *cephv1.CephCluster, retryInterval time.Duration) error {
func (c *ClusterController) waitForCephDaemonCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, retryInterval time.Duration) error {
logger.Infof("waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace)
for {
cephHosts, err := c.getCephHosts(cluster.Namespace)
if err != nil {
return errors.Wrap(err, "failed to list ceph daemon nodes")
}
select {
case <-time.After(retryInterval):
cephHosts, err := c.getCephHosts(cluster.Namespace)
if err != nil {
return errors.Wrap(err, "failed to list ceph daemon nodes")
}
if len(cephHosts) == 0 {
logger.Info("all ceph daemons are cleaned up")
if len(cephHosts) == 0 {
logger.Info("all ceph daemons are cleaned up")
return nil
}
logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
cluster.Namespace, retryInterval.String())
break
case <-stopCleanupCh:
return errors.New("cancelling the host cleanup job")
}
logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
cluster.Namespace, retryInterval.String())
<-time.After(retryInterval)
}
return nil
}
func (c *ClusterController) getCephHosts(namespace string) ([]string, error) {
+3 -1
View File
@@ -265,6 +265,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
logger.Infof("deleting ceph cluster %q", cephCluster.Name)
// Start cluster clean up only if cleanupPolicy is applied to the ceph cluster
stopCleanupCh := make(chan struct{})
if hasCleanupPolicy(cephCluster) {
monSecret, err := r.clusterController.getMonSecret(cephCluster.Namespace)
if err != nil {
@@ -274,7 +275,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
if err != nil {
return reconcile.Result{}, errors.Wrapf(err, "failed to find valid ceph hosts in the cluster %q", cephCluster.Namespace)
}
go r.clusterController.startClusterCleanUp(cephCluster, cephHosts, monSecret)
go r.clusterController.startClusterCleanUp(stopCleanupCh, cephCluster, cephHosts, monSecret)
}
// Run delete sequence
@@ -282,6 +283,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
if !ok {
// If the cluster cannot be deleted, requeue the request for deletion to see if the conditions
// will eventually be satisfied such as the volumes being removed
close(stopCleanupCh)
return response, nil
}