ceph: cancel the host cleanup if pvc deletion pending
During cluster delection, the finalizer will not be removed until all the PVCs are removed. Since this could wait for a long time, as long as the deletion event is re-queued, we need to cancel the goroutine that is watching for cluster deletion so it can avoid running multiple goroutines as the event runs again and again. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
2 files changed
+21
-16
No files matched your search
@@ -49,9 +49,9 @@ var (
|
||||
monitorSecret = "ROOK_MON_SECRET"
|
||||
)
|
||||
|
||||
func (c *ClusterController) startClusterCleanUp(cluster *cephv1.CephCluster, cephHosts []string, monSecret string) {
|
||||
func (c *ClusterController) startClusterCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, cephHosts []string, monSecret string) {
|
||||
logger.Infof("starting clean up for cluster %q", cluster.Name)
|
||||
err := c.waitForCephDaemonCleanUp(cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second)
|
||||
err := c.waitForCephDaemonCleanUp(stopCleanupCh, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to wait till ceph daemons are destroyed. %v", err)
|
||||
return
|
||||
@@ -129,25 +129,28 @@ func (c *ClusterController) cleanUpJobTemplateSpec(cluster *cephv1.CephCluster,
|
||||
return podSpec
|
||||
}
|
||||
|
||||
func (c *ClusterController) waitForCephDaemonCleanUp(cluster *cephv1.CephCluster, retryInterval time.Duration) error {
|
||||
func (c *ClusterController) waitForCephDaemonCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, retryInterval time.Duration) error {
|
||||
logger.Infof("waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace)
|
||||
for {
|
||||
cephHosts, err := c.getCephHosts(cluster.Namespace)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to list ceph daemon nodes")
|
||||
}
|
||||
select {
|
||||
case <-time.After(retryInterval):
|
||||
cephHosts, err := c.getCephHosts(cluster.Namespace)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to list ceph daemon nodes")
|
||||
}
|
||||
|
||||
if len(cephHosts) == 0 {
|
||||
logger.Info("all ceph daemons are cleaned up")
|
||||
if len(cephHosts) == 0 {
|
||||
logger.Info("all ceph daemons are cleaned up")
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
|
||||
cluster.Namespace, retryInterval.String())
|
||||
break
|
||||
case <-stopCleanupCh:
|
||||
return errors.New("cancelling the host cleanup job")
|
||||
}
|
||||
|
||||
logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
|
||||
cluster.Namespace, retryInterval.String())
|
||||
<-time.After(retryInterval)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) getCephHosts(namespace string) ([]string, error) {
|
||||
|
||||
@@ -265,6 +265,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
logger.Infof("deleting ceph cluster %q", cephCluster.Name)
|
||||
|
||||
// Start cluster clean up only if cleanupPolicy is applied to the ceph cluster
|
||||
stopCleanupCh := make(chan struct{})
|
||||
if hasCleanupPolicy(cephCluster) {
|
||||
monSecret, err := r.clusterController.getMonSecret(cephCluster.Namespace)
|
||||
if err != nil {
|
||||
@@ -274,7 +275,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
if err != nil {
|
||||
return reconcile.Result{}, errors.Wrapf(err, "failed to find valid ceph hosts in the cluster %q", cephCluster.Namespace)
|
||||
}
|
||||
go r.clusterController.startClusterCleanUp(cephCluster, cephHosts, monSecret)
|
||||
go r.clusterController.startClusterCleanUp(stopCleanupCh, cephCluster, cephHosts, monSecret)
|
||||
}
|
||||
|
||||
// Run delete sequence
|
||||
@@ -282,6 +283,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
if !ok {
|
||||
// If the cluster cannot be deleted, requeue the request for deletion to see if the conditions
|
||||
// will eventually be satisfied such as the volumes being removed
|
||||
close(stopCleanupCh)
|
||||
return response, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user