From bed1d01e9c300a86c8d9e2217f2cb26889663aa4 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 5 Jul 2018 13:32:26 +0100 Subject: [PATCH] Fix retry of OSD deletion Previously, if this failed mid-way it would then throw an exception at the first operation when retrying (because it's trying to reweight an OSD that might not be in the map any more) Signed-off-by: John Spray --- pkg/daemon/ceph/client/crush.go | 5 +---- pkg/operator/ceph/cluster/osd/remove.go | 29 ++++++++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pkg/daemon/ceph/client/crush.go b/pkg/daemon/ceph/client/crush.go index a70ff8fe1..2c80c1845 100644 --- a/pkg/daemon/ceph/client/crush.go +++ b/pkg/daemon/ceph/client/crush.go @@ -164,11 +164,8 @@ func SetCrushTunables(context *clusterd.Context, clusterName, profile string) (s func CrushReweight(context *clusterd.Context, clusterName string, id int, weight float64) (string, error) { args := []string{"osd", "crush", "reweight", fmt.Sprintf("osd.%d", id), fmt.Sprintf("%.1f", weight)} buf, err := ExecuteCephCommand(context, clusterName, args) - if err != nil { - return "", fmt.Errorf("failed to crush reweight: %+v, %s", err, string(buf)) - } - return string(buf), nil + return string(buf), err } func CrushRemove(context *clusterd.Context, clusterName, name string) (string, error) { diff --git a/pkg/operator/ceph/cluster/osd/remove.go b/pkg/operator/ceph/cluster/osd/remove.go index 5d743c6d6..95d84315c 100644 --- a/pkg/operator/ceph/cluster/osd/remove.go +++ b/pkg/operator/ceph/cluster/osd/remove.go @@ -19,6 +19,7 @@ package osd import ( "fmt" + "syscall" "time" "github.com/rook/rook/pkg/clusterd" @@ -26,6 +27,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" + "github.com/rook/rook/pkg/util/exec" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -40,18 +42,29 @@ func removeOSD(context *clusterd.Context, namespace, deploymentName string, id i // first reweight the OSD to be 0.0, which will begin the data migration o, err := client.CrushReweight(context, namespace, id, 0.0) + alreadyPurged := false if err != nil { - return fmt.Errorf("failed to reweight osd.%d to 0.0: %+v. %s", id, err, o) + // Special error handling for this initial step, to pick up + // the case where the OSD is already removed from Ceph cluster, + // and skip the next few steps if so. + cmdErr, ok := err.(*exec.CommandError) + if ok && cmdErr.ExitStatus() == int(syscall.ENOENT) { + alreadyPurged = true + } else { + return fmt.Errorf("failed to reweight osd.%d to 0.0: %+v. %s", id, err, o) + } } - // mark the OSD as out - if err := markOSDOut(context, namespace, id); err != nil { - return fmt.Errorf("failed to mark osd.%d out: %+v", id, err) - } + if !alreadyPurged { + // mark the OSD as out + if err := markOSDOut(context, namespace, id); err != nil { + return fmt.Errorf("failed to mark osd.%d out: %+v", id, err) + } - // wait for the OSDs data to be migrated - if err := waitForRebalance(context, namespace, id, initialUsage); err != nil { - return fmt.Errorf("failed to wait for cluster rebalancing after removing osd.%d: %+v", id, err) + // wait for the OSDs data to be migrated + if err := waitForRebalance(context, namespace, id, initialUsage); err != nil { + return fmt.Errorf("failed to wait for cluster rebalancing after removing osd.%d: %+v", id, err) + } } // data is migrated off the osd, we can delete the deployment now