Merge pull request #8403 from cybozu-go/ceph-dont-remove-pvc-in-osd-purge-job

ceph: add an option to preserve pvc in osd purge job

Documentation/ceph-osd-mgmt.md

@@ -119,13 +119,15 @@ If you want to remove OSDs by hand, continue with the following sections. Howeve
 
 If the OSD purge job fails or you need fine-grained control of the removal, here are the individual commands that can be run from the toolbox.
 
-1. Mark the OSD as `out` if not already marked as such by Ceph. This signals Ceph to start moving (backfilling) the data that was on that OSD to another OSD.
+1. Detach the OSD PVC from Rook
+   - `kubectl -n rook-ceph label pvc <orphaned-pvc> ceph.rook.io/DeviceSetPVCId-`
+2. Mark the OSD as `out` if not already marked as such by Ceph. This signals Ceph to start moving (backfilling) the data that was on that OSD to another OSD.
    - `ceph osd out osd.<ID>` (for example if the OSD ID is 23 this would be `ceph osd out osd.23`)
-2. Wait for the data to finish backfilling to other OSDs.
+3. Wait for the data to finish backfilling to other OSDs.
    - `ceph status` will indicate the backfilling is done when all of the PGs are `active+clean`. If desired, it's safe to remove the disk after that.
-3. Remove the OSD from the Ceph cluster
+4. Remove the OSD from the Ceph cluster
    - `ceph osd purge <ID> --yes-i-really-mean-it`
-4. Verify the OSD is removed from the node in the CRUSH map
+5. Verify the OSD is removed from the node in the CRUSH map
    - `ceph osd tree`
 
 The operator can automatically remove OSD deployments that are considered "safe-to-destroy" by Ceph.

cluster/examples/kubernetes/ceph/osd-purge.yaml

@@ -28,7 +28,8 @@ spec:
           image: rook/ceph:master
           # TODO: Insert the OSD ID in the last parameter that is to be removed
           # The OSD IDs are a comma-separated list. For example: "0" or "0,2".
-          args: ["ceph", "osd", "remove", "--osd-ids", "<OSD-IDs>"]
+          # If you want to preserve the OSD PVCs, set `--preserve-pvc true`.
+          args: ["ceph", "osd", "remove", "--preserve-pvc", "false", "--osd-ids", "<OSD-IDs>"]
           env:
             - name: POD_NAMESPACE
               valueFrom:

cmd/rook/ceph/osd.go

@@ -70,6 +70,7 @@ var (
 	blockPath               string
 	lvBackedPV              bool
 	osdIDsToRemove          string
+	preservePVC             bool
 )
 
 func addOSDFlags(command *cobra.Command) {
@@ -98,6 +99,7 @@ func addOSDFlags(command *cobra.Command) {
 
 	// flags for removing OSDs that are unhealthy or otherwise should be purged from the cluster
 	osdRemoveCmd.Flags().StringVar(&osdIDsToRemove, "osd-ids", "", "OSD IDs to remove from the cluster")
+	osdRemoveCmd.Flags().BoolVar(&preservePVC, "preserve-pvc", false, "Whether PVCs for OSDs will be deleted")
 
 	// add the subcommands to the parent osd command
 	osdCmd.AddCommand(osdConfigCmd,
@@ -260,7 +262,7 @@ func removeOSDs(cmd *cobra.Command, args []string) error {
 	context := createContext()
 
 	// Run OSD remove sequence
-	err := osddaemon.RemoveOSDs(context, &clusterInfo, strings.Split(osdIDsToRemove, ","))
+	err := osddaemon.RemoveOSDs(context, &clusterInfo, strings.Split(osdIDsToRemove, ","), preservePVC)
 	if err != nil {
 		rook.TerminateFatal(err)
 	}

pkg/daemon/ceph/osd/remove.go

@@ -32,7 +32,7 @@ import (
 )
 
 // RemoveOSDs purges a list of OSDs from the cluster
-func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string) error {
+func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string, preservePVC bool) error {
 
 	// Generate the ceph config for running ceph commands similar to the operator
 	if err := client.WriteCephConfig(context, clusterInfo); err != nil {
@@ -61,13 +61,13 @@ func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osds
 			continue
 		}
 		logger.Infof("osd.%d is marked 'DOWN'. Removing it", osdID)
-		removeOSD(context, clusterInfo, osdID)
+		removeOSD(context, clusterInfo, osdID, preservePVC)
 	}
 
 	return nil
 }
 
-func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInfo, osdID int) {
+func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInfo, osdID int, preservePVC bool) {
 	ctx := context.TODO()
 	// Get the host where the OSD is found
 	hostName, err := client.GetCrushHostName(clusterdContext, clusterInfo, osdID)
@@ -111,12 +111,27 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf
 					}
 				}
 			}
-			// Remove the OSD PVC
-			logger.Infof("removing the OSD PVC %q", pvcName)
-			if err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil {
-				if err != nil {
-					// Continue deleting the OSD PVC even if PVC deletion fails
-					logger.Errorf("failed to delete pvc for OSD %q. %v", pvcName, err)
+			if preservePVC {
+				// Detach the OSD PVC from Rook. We will continue OSD deletion even if failed to remove PVC label
+				logger.Infof("detach the OSD PVC %q from Rook", pvcName)
+				if pvc, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Get(ctx, pvcName, metav1.GetOptions{}); err != nil {
+					logger.Errorf("failed to get pvc for OSD %q. %v", pvcName, err)
+				} else {
+					labels := pvc.GetLabels()
+					delete(labels, osd.CephDeviceSetPVCIDLabelKey)
+					pvc.SetLabels(labels)
+					if _, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Update(ctx, pvc, metav1.UpdateOptions{}); err != nil {
+						logger.Errorf("failed to remove label %q from pvc for OSD %q. %v", osd.CephDeviceSetPVCIDLabelKey, pvcName, err)
+					}
+				}
+			} else {
+				// Remove the OSD PVC
+				logger.Infof("removing the OSD PVC %q", pvcName)
+				if err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil {
+					if err != nil {
+						// Continue deleting the OSD PVC even if PVC deletion fails
+						logger.Errorf("failed to delete pvc for OSD %q. %v", pvcName, err)
+					}
 				}
 			}
 		} else {