From 5e8375e92162428906941149600a598d7ad41677 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Tue, 30 Apr 2019 18:19:56 +0530 Subject: [PATCH] ceph: Delete and recreate rook-ceph-detect-version job The ceph controller will delete and recreate the detec-version job, when an already existing job is found. This helps avoid cluster creation from getting hung up on waiting for an already existing/stale job to finish. Fixes #3081. Signed-off-by: Kaushal M --- pkg/operator/ceph/cluster/cluster.go | 2 +- pkg/operator/ceph/cluster/osd/osd.go | 2 +- pkg/operator/ceph/nfs/nfs.go | 2 +- pkg/operator/k8sutil/job.go | 7 ++++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 9d9cf87a9..e68a3fa25 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -119,7 +119,7 @@ func (c *cluster) detectCephVersion(image string, timeout time.Duration) (*cephv k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &job.ObjectMeta, &c.ownerRef) // run the job to detect the version - if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil { + if err := k8sutil.RunReplaceableJob(c.context.Clientset, job, true); err != nil { return nil, fmt.Errorf("failed to start version job. %+v", err) } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 460670732..e9f6e44b7 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -256,7 +256,7 @@ func (c *Cluster) startProvisioning(config *provisionConfig) { } func (c *Cluster) runJob(job *batch.Job, nodeName string, config *provisionConfig, action string) bool { - if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil { + if err := k8sutil.RunReplaceableJob(c.context.Clientset, job, false); err != nil { if !errors.IsAlreadyExists(err) { // we failed to create job, update the orchestration status for this node message := fmt.Sprintf("failed to create %s job for node %s. %+v", action, nodeName, err) diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index cc70365ed..5f1deed3b 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -173,7 +173,7 @@ func (c *CephNFSController) runGaneshaRadosGraceJob(n cephv1.CephNFS, name, acti k8sutil.SetOwnerRef(c.context.Clientset, n.Namespace, &job.ObjectMeta, &c.ownerRef) // run the job to detect the version - if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil { + if err := k8sutil.RunReplaceableJob(c.context.Clientset, job, false); err != nil { return fmt.Errorf("failed to start job %s. %+v", job.Name, err) } diff --git a/pkg/operator/k8sutil/job.go b/pkg/operator/k8sutil/job.go index c38028ff5..c7909aa9d 100644 --- a/pkg/operator/k8sutil/job.go +++ b/pkg/operator/k8sutil/job.go @@ -29,14 +29,15 @@ import ( "k8s.io/client-go/kubernetes" ) -func RunReplaceableJob(clientset kubernetes.Interface, job *batch.Job) error { +func RunReplaceableJob(clientset kubernetes.Interface, job *batch.Job, deleteIfFound bool) error { // check if the job was already created and what its status is existingJob, err := clientset.BatchV1().Jobs(job.Namespace).Get(job.Name, metav1.GetOptions{}) if err != nil && !errors.IsNotFound(err) { logger.Warningf("failed to detect job %s. %+v", job.Name, err) } else if err == nil { - // if the job is still running, allow it to continue to completion - if existingJob.Status.Active > 0 { + // if the job is still running, and the caller has not asked for deletion, + // allow it to continue to completion + if existingJob.Status.Active > 0 && !deleteIfFound { logger.Infof("Found previous job %s. Status=%+v", job.Name, existingJob.Status) return nil }