From acc5ba1a6b69ec6ec3c8f691dccd79d901256a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 13 Aug 2020 11:02:06 +0200 Subject: [PATCH] ceph: check for gone node on crash collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a node is gone, we try to remove crash collector deployments so they don't stay stale or pending. We must ensure the node is gone and not just return an error even if we successfully deployment the orphan deployment. Signed-off-by: Sébastien Han --- pkg/operator/ceph/cluster/crash/reconcile.go | 31 +++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/pkg/operator/ceph/cluster/crash/reconcile.go b/pkg/operator/ceph/cluster/crash/reconcile.go index 236e67032..cad073b40 100644 --- a/pkg/operator/ceph/cluster/crash/reconcile.go +++ b/pkg/operator/ceph/cluster/crash/reconcile.go @@ -78,23 +78,26 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: request.Name}} err := r.client.Get(context.TODO(), request.NamespacedName, node) if err != nil { - //if a node is not present, check if there are any crashcollector deployment for that node and delete it. - deploymentList := &appsv1.DeploymentList{} - namespaceListOpts := client.InNamespace(request.Namespace) - err := r.client.List(context.TODO(), deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName, NodeNameLabel: request.Name}, namespaceListOpts) - if err != nil { - logger.Errorf("failed to list crash collector deployments, delete it/them manually. %v", err) - } - for _, d := range deploymentList.Items { - logger.Infof("deleting deployment %q for deleted node %q", d.ObjectMeta.Name, request.Name) - err := r.deleteCrashCollector(d) + if kerrors.IsNotFound(err) { + // if a node is not present, check if there are any crashcollector deployment for that node and delete it. + deploymentList := &appsv1.DeploymentList{} + namespaceListOpts := client.InNamespace(request.Namespace) + err := r.client.List(context.TODO(), deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName, NodeNameLabel: request.Name}, namespaceListOpts) if err != nil { - logger.Errorf("failed to delete crash collector deployment %q, delete it manually. %v", d.Name, err) - continue + logger.Errorf("failed to list crash collector deployments, delete it/them manually. %v", err) } - logger.Infof("crash collector deployment %q successfully removed", d.Name) + for _, d := range deploymentList.Items { + logger.Infof("deleting deployment %q for deleted node %q", d.ObjectMeta.Name, request.Name) + err := r.deleteCrashCollector(d) + if err != nil { + logger.Errorf("failed to delete crash collector deployment %q, delete it manually. %v", d.Name, err) + continue + } + logger.Infof("crash collector deployment %q successfully removed from dead node %q", d.Name, request.Name) + } + } else { + return reconcile.Result{}, errors.Wrapf(err, "could not get node %q", request.Name) } - return reconcile.Result{}, errors.Errorf("could not get node %q", request.NamespacedName) } // Get the list of all the Ceph pods