Merge pull request #14904 from rook/mergify/bp/release-1.15/pr-14895

core: Cleanup blockpool with annotation (backport #14895)

Documentation/Storage-Configuration/ceph-teardown.md

@@ -205,3 +205,4 @@ This cleanup is supported only for the following custom resources:
 | --------                             | ------- |
 | CephFilesystemSubVolumeGroup         | CSI stored RADOS OMAP details for pvc/volumesnapshots, subvolume snapshots, subvolume clones, subvolumes |
 | CephBlockPoolRadosNamespace          | Images and snapshots in the RADOS namespace|
+| CephBlockPool                        | Images and snapshots in the BlockPool|

cmd/rook/ceph/cleanup.go

@@ -62,6 +62,12 @@ var cleanUpRadosNamespaceCmd = &cobra.Command{
 	Short: "Starts the cleanup process for a CephBlockPoolRadosNamespace",
 }
 
+var cleanUpBlockPoolCmd = &cobra.Command{
+	// the subcommand matches CRD kind of the custom resource to be cleaned up
+	Use:   "CephBlockPool",
+	Short: "Starts the cleanup process for a CephBlockPool",
+}
+
 func init() {
 	cleanUpHostCmd.Flags().StringVar(&dataDirHostPath, "data-dir-host-path", "", "dataDirHostPath on the node")
 	cleanUpHostCmd.Flags().StringVar(&namespaceDir, "namespace-dir", "", "dataDirHostPath on the node")
@@ -74,11 +80,12 @@ func init() {
 	flags.SetFlagsFromEnv(cleanUpHostCmd.Flags(), rook.RookEnvVarPrefix)
 	flags.SetFlagsFromEnv(cleanUpSubVolumeGroupCmd.Flags(), rook.RookEnvVarPrefix)
 
-	cleanUpCmd.AddCommand(cleanUpHostCmd, cleanUpSubVolumeGroupCmd, cleanUpRadosNamespaceCmd)
+	cleanUpCmd.AddCommand(cleanUpHostCmd, cleanUpSubVolumeGroupCmd, cleanUpRadosNamespaceCmd, cleanUpBlockPoolCmd)
 
 	cleanUpHostCmd.RunE = startHostCleanUp
 	cleanUpSubVolumeGroupCmd.RunE = startSubVolumeGroupCleanUp
 	cleanUpRadosNamespaceCmd.RunE = startRadosNamespaceCleanup
+	cleanUpBlockPoolCmd.RunE = startBlockPoolCleanup
 }
 
 func startHostCleanUp(cmd *cobra.Command, args []string) error {
@@ -174,3 +181,25 @@ func startRadosNamespaceCleanup(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func startBlockPoolCleanup(cmd *cobra.Command, args []string) error {
+	rook.SetLogLevel()
+	rook.LogStartupInfo(cleanUpRadosNamespaceCmd.Flags())
+
+	ctx := cmd.Context()
+	context := createContext()
+	namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
+	clusterInfo := client.AdminClusterInfo(ctx, namespace, "")
+
+	poolName := os.Getenv(opcontroller.CephBlockPoolNameEnv)
+	if poolName == "" {
+		rook.TerminateFatal(fmt.Errorf("cephblockpool name is not available in the pod environment variables"))
+	}
+
+	err := cleanup.BlockPoolCleanup(context, clusterInfo, poolName)
+	if err != nil {
+		rook.TerminateFatal(fmt.Errorf("failed to cleanup cephBlockPool %q resource %v", poolName, err))
+	}
+
+	return nil
+}

pkg/daemon/ceph/cleanup/radosnamespace.go

@@ -17,6 +17,8 @@ limitations under the License.
 package cleanup
 
 import (
+	"fmt"
+
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
@@ -26,46 +28,67 @@ import (
 func RadosNamespaceCleanup(context *clusterd.Context, clusterInfo *client.ClusterInfo, poolName, radosNamespace string) error {
 	logger.Infof("starting clean up of CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName)
 
+	err := cleanupImages(context, clusterInfo, poolName, radosNamespace)
+	if err != nil {
+		logger.Errorf("failed to clean up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName)
+		return err
+	}
+
+	logger.Infof("successfully cleaned up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName)
+	return nil
+}
+
+func cleanupImages(context *clusterd.Context, clusterInfo *client.ClusterInfo, poolName, radosNamespace string) error {
+	msg := fmt.Sprintf("cephblockpool %q", poolName)
+	if radosNamespace != "" {
+		msg = fmt.Sprintf("%s in rados namespace %q", msg, radosNamespace)
+	}
 	images, err := cephclient.ListImagesInRadosNamespace(context, clusterInfo, poolName, radosNamespace)
 	if err != nil {
-		return errors.Wrapf(err, "failed to list images in cephblockpool %q in rados namespace %q", poolName, radosNamespace)
+		return errors.Wrapf(err, "failed to list images in %s", msg)
 	}
 
 	var retErr error
 	for _, image := range images {
 		snaps, err := cephclient.ListSnapshotsInRadosNamespace(context, clusterInfo, poolName, image.Name, radosNamespace)
 		if err != nil {
-			retErr = errors.Wrapf(err, "failed to list snapshots for the image %q in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace)
+			retErr = errors.Wrapf(err, "failed to list snapshots for the image %q in %s", image.Name, msg)
 			logger.Error(retErr)
 		}
 
 		for _, snap := range snaps {
 			err := cephclient.DeleteSnapshotInRadosNamespace(context, clusterInfo, poolName, image.Name, snap.Name, radosNamespace)
 			if err != nil {
-				retErr = errors.Wrapf(err, "failed to delete snapshot %q of the image %q in cephblockpool %q in rados namespace %q.", snap.Name, image.Name, poolName, radosNamespace)
+				retErr = errors.Wrapf(err, "failed to delete snapshot %q of the image %q in %s", snap.Name, image.Name, msg)
 				logger.Error(retErr)
 			} else {
-				logger.Infof("successfully deleted snapshot %q of image %q in cephblockpool %q in rados namespace %q", snap.Name, image.Name, poolName, radosNamespace)
+				logger.Infof("successfully deleted snapshot %q of image %q in %s", snap.Name, image.Name, msg)
 			}
 		}
 
 		err = cephclient.MoveImageToTrashInRadosNamespace(context, clusterInfo, poolName, image.Name, radosNamespace)
 		if err != nil {
-			retErr = errors.Wrapf(err, "failed to move image %q to trash in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace)
+			retErr = errors.Wrapf(err, "failed to move image %q to trash in %s", image.Name, msg)
 			logger.Error(retErr)
 		}
 		err = cephclient.DeleteImageFromTrashInRadosNamespace(context, clusterInfo, poolName, image.ID, radosNamespace)
 		if err != nil {
-			retErr = errors.Wrapf(err, "failed to add task to remove image %q from trash in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace)
+			retErr = errors.Wrapf(err, "failed to add task to remove image %q from trash in %s", image.Name, msg)
 			logger.Error(retErr)
 		}
 	}
+	return retErr
+}
 
-	if retErr != nil {
-		logger.Errorf("failed to clean up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName)
-		return retErr
+func BlockPoolCleanup(context *clusterd.Context, clusterInfo *client.ClusterInfo, poolName string) error {
+	logger.Infof("starting clean up of CephBlockPool %q resource", poolName)
+
+	err := cleanupImages(context, clusterInfo, poolName, "")
+	if err != nil {
+		logger.Errorf("failed to clean up CephBlockPool %q resource", poolName)
+		return err
 	}
 
-	logger.Infof("successfully cleaned up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName)
+	logger.Infof("successfully cleaned up CephBlockPool %q resource", poolName)
 	return nil
 }

pkg/daemon/ceph/cleanup/radosnamespace_test.go

@@ -90,3 +90,56 @@ func TestRadosNamespace(t *testing.T) {
 		assert.NoError(t, err)
 	})
 }
+
+func TestBlockPoolCleanup(t *testing.T) {
+	clusterInfo := cephclient.AdminTestClusterInfo("mycluster")
+	poolName := "test-pool"
+
+	t.Run("no images in blockpool", func(t *testing.T) {
+		executor := &exectest.MockExecutor{}
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			logger.Infof("Command: %s %v", command, args)
+			if args[0] == "ls" && args[1] == "-l" {
+				assert.Equal(t, poolName, args[2])
+				return "", nil
+			}
+			return "", errors.New("unknown command")
+		}
+		context := &clusterd.Context{Executor: executor}
+		err := BlockPoolCleanup(context, clusterInfo, poolName)
+		assert.NoError(t, err)
+	})
+
+	t.Run("images with snapshots available in blockpool", func(t *testing.T) {
+		executor := &exectest.MockExecutor{}
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			logger.Infof("Command: %s %v", command, args)
+			// list all subvolumes in subvolumegroup
+			if args[0] == "ls" && args[1] == "-l" {
+				assert.Equal(t, poolName, args[2])
+				return mockImageLSResponse, nil
+			}
+			if args[0] == "snap" && args[1] == "ls" {
+				assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d", args[2])
+				return mockSnapshotsResponse, nil
+			}
+			if args[0] == "snap" && args[1] == "rm" {
+				assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d@snap1", args[2])
+				return "", nil
+			}
+			if args[0] == "trash" && args[1] == "mv" {
+				assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d", args[2])
+				return "", nil
+			}
+			if args[0] == "rbd" && args[1] == "task" && args[2] == "add" && args[3] == "trash" {
+				// pool-name/image-id
+				assert.Equal(t, "test-pool/16e35cfa56a7", args[5])
+				return "", nil
+			}
+			return "", errors.New("unknown command")
+		}
+		context := &clusterd.Context{Executor: executor}
+		err := BlockPoolCleanup(context, clusterInfo, poolName)
+		assert.NoError(t, err)
+	})
+}

pkg/daemon/ceph/client/image.go

@@ -85,7 +85,10 @@ func ListImagesInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterI
 // ListSnapshotsInRadosNamespace lists all the snapshots created for an image in a cephblockpool in a given rados namespace
 func ListSnapshotsInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, namespace string) ([]CephBlockImageSnapshot, error) {
 	snapshots := []CephBlockImageSnapshot{}
-	args := []string{"snap", "ls", getImageSpec(imageName, poolName), "--namespace", namespace}
+	args := []string{"snap", "ls", getImageSpec(imageName, poolName)}
+	if namespace != "" {
+		args = append(args, "--namespace", namespace)
+	}
 	cmd := NewRBDCommand(context, clusterInfo, args)
 	cmd.JsonOutput = true
 	buf, err := cmd.Run()
@@ -101,7 +104,10 @@ func ListSnapshotsInRadosNamespace(context *clusterd.Context, clusterInfo *Clust
 
 // DeleteSnapshotInRadosNamespace deletes a image snapshot created in block pool in a given rados namespace
 func DeleteSnapshotInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, snapshot, namespace string) error {
-	args := []string{"snap", "rm", getImageSnapshotSpec(poolName, imageName, snapshot), "--namespace", namespace}
+	args := []string{"snap", "rm", getImageSnapshotSpec(poolName, imageName, snapshot)}
+	if namespace != "" {
+		args = append(args, "--namespace", namespace)
+	}
 	cmd := NewRBDCommand(context, clusterInfo, args)
 	_, err := cmd.Run()
 	if err != nil {
@@ -112,7 +118,10 @@ func DeleteSnapshotInRadosNamespace(context *clusterd.Context, clusterInfo *Clus
 
 // MoveImageToTrashInRadosNamespace moves the cephblockpool image to trash in the rados namespace
 func MoveImageToTrashInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, namespace string) error {
-	args := []string{"trash", "mv", getImageSpec(imageName, poolName), "--namespace", namespace}
+	args := []string{"trash", "mv", getImageSpec(imageName, poolName)}
+	if namespace != "" {
+		args = append(args, "--namespace", namespace)
+	}
 	cmd := NewRBDCommand(context, clusterInfo, args)
 	_, err := cmd.Run()
 	if err != nil {
@@ -268,8 +277,12 @@ func getImageSpec(name, poolName string) string {
 }
 
 func getImageSpecInRadosNamespace(poolName, namespace, imageID string) string {
+	if namespace == "" {
+		return fmt.Sprintf("%s/%s", poolName, imageID)
+	}
 	return fmt.Sprintf("%s/%s/%s", poolName, namespace, imageID)
 }
+
 func getImageSnapshotSpec(poolName, imageName, snapshot string) string {
 	return fmt.Sprintf("%s/%s@%s", poolName, imageName, snapshot)
 }

pkg/operator/ceph/pool/controller.go

@@ -76,6 +76,7 @@ type ReconcileCephBlockPool struct {
 	blockPoolContexts map[string]*blockPoolHealth
 	opManagerContext  context.Context
 	recorder          record.EventRecorder
+	opConfig          opcontroller.OperatorConfig
 }
 
 type blockPoolHealth struct {
@@ -87,11 +88,11 @@ type blockPoolHealth struct {
 // Add creates a new CephBlockPool Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
 func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
-	return add(opManagerContext, mgr, newReconciler(mgr, context, opManagerContext))
+	return add(opManagerContext, mgr, newReconciler(mgr, context, opManagerContext, opConfig))
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
 	return &ReconcileCephBlockPool{
 		client:            mgr.GetClient(),
 		scheme:            mgr.GetScheme(),
@@ -99,6 +100,7 @@ func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerCont
 		blockPoolContexts: make(map[string]*blockPoolHealth),
 		opManagerContext:  opManagerContext,
 		recorder:          mgr.GetEventRecorderFor("rook-" + controllerName),
+		opConfig:          opConfig,
 	}
 }
 
@@ -245,6 +247,12 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
 		logger.Infof("deleting pool %q", poolSpec.Name)
 		err = deletePool(r.context, clusterInfo, &poolSpec)
 		if err != nil {
+			if opcontroller.ForceDeleteRequested(cephBlockPool.GetAnnotations()) {
+				cleanupErr := r.cleanup(cephBlockPool, &cephCluster)
+				if cleanupErr != nil {
+					return reconcile.Result{}, *cephBlockPool, errors.Wrapf(cleanupErr, "failed to create clean up job for ceph blockpool %q", cephBlockPool.Name)
+				}
+			}
 			return opcontroller.ImmediateRetryResult, *cephBlockPool, errors.Wrapf(err, "failed to delete pool %q. ", cephBlockPool.Name)
 		}
 
@@ -373,6 +381,20 @@ func (r *ReconcileCephBlockPool) reconcileCreatePool(clusterInfo *cephclient.Clu
 	return reconcile.Result{}, nil
 }
 
+func (r *ReconcileCephBlockPool) cleanup(cephblockpool *cephv1.CephBlockPool, cephCluster *cephv1.CephCluster) error {
+	logger.Infof("starting cleanup of the ceph resources for CephBlockPool %q in k8s namespace %q", cephblockpool.Name, cephblockpool.Namespace)
+	cleanupConfig := map[string]string{
+		opcontroller.CephBlockPoolNameEnv: cephblockpool.Name,
+	}
+	cleanup := opcontroller.NewResourceCleanup(cephblockpool, cephCluster, r.opConfig.Image, cleanupConfig)
+	jobName := k8sutil.TruncateNodeNameForJob("cleanup-cephblockpool-%s", cephblockpool.Name)
+	err := cleanup.StartJob(r.clusterInfo.Context, r.context.Clientset, jobName)
+	if err != nil {
+		return errors.Wrapf(err, "failed to run clean up job to clean the cephblockpool %q", cephblockpool.Name)
+	}
+	return nil
+}
+
 // Create the pool
 func createPool(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, clusterSpec *cephv1.ClusterSpec, p *cephv1.NamedPoolSpec) error {
 	// Set the application name to rbd by default, but override later for special pools