Fix retry of OSD deletion

Previously, if this failed mid-way it would then throw
an exception at the first operation when retrying (because it's
trying to reweight an OSD that might not be in the map any more)

Signed-off-by: John Spray <john.spray@redhat.com>
This commit is contained in:
John Spray committed 2018-07-17 14:08:23 +01:00
1 parent dff99d5532
commit bed1d01e9c
2 files changed
+22 -12

No files matched your search

+1 -4
View File
@@ -164,11 +164,8 @@ func SetCrushTunables(context *clusterd.Context, clusterName, profile string) (s
func CrushReweight(context *clusterd.Context, clusterName string, id int, weight float64) (string, error) {
args := []string{"osd", "crush", "reweight", fmt.Sprintf("osd.%d", id), fmt.Sprintf("%.1f", weight)}
buf, err := ExecuteCephCommand(context, clusterName, args)
if err != nil {
return "", fmt.Errorf("failed to crush reweight: %+v, %s", err, string(buf))
}
return string(buf), nil
return string(buf), err
}
func CrushRemove(context *clusterd.Context, clusterName, name string) (string, error) {
+21 -8
View File
@@ -19,6 +19,7 @@ package osd
import (
"fmt"
"syscall"
"time"
"github.com/rook/rook/pkg/clusterd"
@@ -26,6 +27,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/exec"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
@@ -40,18 +42,29 @@ func removeOSD(context *clusterd.Context, namespace, deploymentName string, id i
// first reweight the OSD to be 0.0, which will begin the data migration
o, err := client.CrushReweight(context, namespace, id, 0.0)
alreadyPurged := false
if err != nil {
return fmt.Errorf("failed to reweight osd.%d to 0.0: %+v. %s", id, err, o)
// Special error handling for this initial step, to pick up
// the case where the OSD is already removed from Ceph cluster,
// and skip the next few steps if so.
cmdErr, ok := err.(*exec.CommandError)
if ok && cmdErr.ExitStatus() == int(syscall.ENOENT) {
alreadyPurged = true
} else {
return fmt.Errorf("failed to reweight osd.%d to 0.0: %+v. %s", id, err, o)
}
}
// mark the OSD as out
if err := markOSDOut(context, namespace, id); err != nil {
return fmt.Errorf("failed to mark osd.%d out: %+v", id, err)
}
if !alreadyPurged {
// mark the OSD as out
if err := markOSDOut(context, namespace, id); err != nil {
return fmt.Errorf("failed to mark osd.%d out: %+v", id, err)
}
// wait for the OSDs data to be migrated
if err := waitForRebalance(context, namespace, id, initialUsage); err != nil {
return fmt.Errorf("failed to wait for cluster rebalancing after removing osd.%d: %+v", id, err)
// wait for the OSDs data to be migrated
if err := waitForRebalance(context, namespace, id, initialUsage); err != nil {
return fmt.Errorf("failed to wait for cluster rebalancing after removing osd.%d: %+v", id, err)
}
}
// data is migrated off the osd, we can delete the deployment now