From 89b6a6028c67ee0ee3f23638dc8f49d2ffda911c Mon Sep 17 00:00:00 2001 From: Satoru Takeuchi Date: Tue, 27 Jul 2021 16:43:23 +0000 Subject: [PATCH] ceph: add an option to preserve pvc in osd purge job Sometimes we want to investigate a PVC for removed OSD. Signed-off-by: Satoru Takeuchi --- Documentation/ceph-osd-mgmt.md | 10 +++--- .../examples/kubernetes/ceph/osd-purge.yaml | 3 +- cmd/rook/ceph/osd.go | 4 ++- pkg/daemon/ceph/osd/remove.go | 33 ++++++++++++++----- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/Documentation/ceph-osd-mgmt.md b/Documentation/ceph-osd-mgmt.md index 3d88c6a84..d12f7eb07 100644 --- a/Documentation/ceph-osd-mgmt.md +++ b/Documentation/ceph-osd-mgmt.md @@ -119,13 +119,15 @@ If you want to remove OSDs by hand, continue with the following sections. Howeve If the OSD purge job fails or you need fine-grained control of the removal, here are the individual commands that can be run from the toolbox. -1. Mark the OSD as `out` if not already marked as such by Ceph. This signals Ceph to start moving (backfilling) the data that was on that OSD to another OSD. +1. Detach the OSD PVC from Rook + - `kubectl -n rook-ceph label pvc ceph.rook.io/DeviceSetPVCId-` +2. Mark the OSD as `out` if not already marked as such by Ceph. This signals Ceph to start moving (backfilling) the data that was on that OSD to another OSD. - `ceph osd out osd.` (for example if the OSD ID is 23 this would be `ceph osd out osd.23`) -2. Wait for the data to finish backfilling to other OSDs. +3. Wait for the data to finish backfilling to other OSDs. - `ceph status` will indicate the backfilling is done when all of the PGs are `active+clean`. If desired, it's safe to remove the disk after that. -3. Remove the OSD from the Ceph cluster +4. Remove the OSD from the Ceph cluster - `ceph osd purge --yes-i-really-mean-it` -4. Verify the OSD is removed from the node in the CRUSH map +5. Verify the OSD is removed from the node in the CRUSH map - `ceph osd tree` The operator can automatically remove OSD deployments that are considered "safe-to-destroy" by Ceph. diff --git a/cluster/examples/kubernetes/ceph/osd-purge.yaml b/cluster/examples/kubernetes/ceph/osd-purge.yaml index a472ce71a..3acc2405b 100644 --- a/cluster/examples/kubernetes/ceph/osd-purge.yaml +++ b/cluster/examples/kubernetes/ceph/osd-purge.yaml @@ -28,7 +28,8 @@ spec: image: rook/ceph:master # TODO: Insert the OSD ID in the last parameter that is to be removed # The OSD IDs are a comma-separated list. For example: "0" or "0,2". - args: ["ceph", "osd", "remove", "--osd-ids", ""] + # If you want to preserve the OSD PVCs, set `--preserve-pvc true`. + args: ["ceph", "osd", "remove", "--preserve-pvc", "false", "--osd-ids", ""] env: - name: POD_NAMESPACE valueFrom: diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 0cdeb8a5e..436b4b4e7 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -70,6 +70,7 @@ var ( blockPath string lvBackedPV bool osdIDsToRemove string + preservePVC bool ) func addOSDFlags(command *cobra.Command) { @@ -98,6 +99,7 @@ func addOSDFlags(command *cobra.Command) { // flags for removing OSDs that are unhealthy or otherwise should be purged from the cluster osdRemoveCmd.Flags().StringVar(&osdIDsToRemove, "osd-ids", "", "OSD IDs to remove from the cluster") + osdRemoveCmd.Flags().BoolVar(&preservePVC, "preserve-pvc", false, "Whether PVCs for OSDs will be deleted") // add the subcommands to the parent osd command osdCmd.AddCommand(osdConfigCmd, @@ -260,7 +262,7 @@ func removeOSDs(cmd *cobra.Command, args []string) error { context := createContext() // Run OSD remove sequence - err := osddaemon.RemoveOSDs(context, &clusterInfo, strings.Split(osdIDsToRemove, ",")) + err := osddaemon.RemoveOSDs(context, &clusterInfo, strings.Split(osdIDsToRemove, ","), preservePVC) if err != nil { rook.TerminateFatal(err) } diff --git a/pkg/daemon/ceph/osd/remove.go b/pkg/daemon/ceph/osd/remove.go index 26c6a940b..a4f032666 100644 --- a/pkg/daemon/ceph/osd/remove.go +++ b/pkg/daemon/ceph/osd/remove.go @@ -32,7 +32,7 @@ import ( ) // RemoveOSDs purges a list of OSDs from the cluster -func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string) error { +func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string, preservePVC bool) error { // Generate the ceph config for running ceph commands similar to the operator if err := client.WriteCephConfig(context, clusterInfo); err != nil { @@ -61,13 +61,13 @@ func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osds continue } logger.Infof("osd.%d is marked 'DOWN'. Removing it", osdID) - removeOSD(context, clusterInfo, osdID) + removeOSD(context, clusterInfo, osdID, preservePVC) } return nil } -func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInfo, osdID int) { +func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInfo, osdID int, preservePVC bool) { ctx := context.TODO() // Get the host where the OSD is found hostName, err := client.GetCrushHostName(clusterdContext, clusterInfo, osdID) @@ -111,12 +111,27 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf } } } - // Remove the OSD PVC - logger.Infof("removing the OSD PVC %q", pvcName) - if err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil { - if err != nil { - // Continue deleting the OSD PVC even if PVC deletion fails - logger.Errorf("failed to delete pvc for OSD %q. %v", pvcName, err) + if preservePVC { + // Detach the OSD PVC from Rook. We will continue OSD deletion even if failed to remove PVC label + logger.Infof("detach the OSD PVC %q from Rook", pvcName) + if pvc, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Get(ctx, pvcName, metav1.GetOptions{}); err != nil { + logger.Errorf("failed to get pvc for OSD %q. %v", pvcName, err) + } else { + labels := pvc.GetLabels() + delete(labels, osd.CephDeviceSetPVCIDLabelKey) + pvc.SetLabels(labels) + if _, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Update(ctx, pvc, metav1.UpdateOptions{}); err != nil { + logger.Errorf("failed to remove label %q from pvc for OSD %q. %v", osd.CephDeviceSetPVCIDLabelKey, pvcName, err) + } + } + } else { + // Remove the OSD PVC + logger.Infof("removing the OSD PVC %q", pvcName) + if err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil { + if err != nil { + // Continue deleting the OSD PVC even if PVC deletion fails + logger.Errorf("failed to delete pvc for OSD %q. %v", pvcName, err) + } } } } else {