From 7cabfd056fe6e6ba5a17b85f7285d777a731ea96 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 23 Oct 2024 12:10:53 +0200 Subject: [PATCH] core: cleanup blockpool with annotation This is similar to #14052 we did for radosnamespace and this is an extension to support cleanup at the blockpool level to cleanup the images and the snapshots in a pool. Signed-off-by: Madhu Rajanna (cherry picked from commit 4e29717317d6e62c57b404282b6c9f155d7b1531) --- .../Storage-Configuration/ceph-teardown.md | 1 + cmd/rook/ceph/cleanup.go | 31 ++++++++++- pkg/daemon/ceph/cleanup/radosnamespace.go | 43 +++++++++++---- .../ceph/cleanup/radosnamespace_test.go | 53 +++++++++++++++++++ pkg/daemon/ceph/client/image.go | 19 +++++-- pkg/operator/ceph/pool/controller.go | 26 ++++++++- 6 files changed, 157 insertions(+), 16 deletions(-) diff --git a/Documentation/Storage-Configuration/ceph-teardown.md b/Documentation/Storage-Configuration/ceph-teardown.md index b207efc47..6196854ed 100644 --- a/Documentation/Storage-Configuration/ceph-teardown.md +++ b/Documentation/Storage-Configuration/ceph-teardown.md @@ -205,3 +205,4 @@ This cleanup is supported only for the following custom resources: | -------- | ------- | | CephFilesystemSubVolumeGroup | CSI stored RADOS OMAP details for pvc/volumesnapshots, subvolume snapshots, subvolume clones, subvolumes | | CephBlockPoolRadosNamespace | Images and snapshots in the RADOS namespace| +| CephBlockPool | Images and snapshots in the BlockPool| diff --git a/cmd/rook/ceph/cleanup.go b/cmd/rook/ceph/cleanup.go index f158619d9..0a2dea1fd 100644 --- a/cmd/rook/ceph/cleanup.go +++ b/cmd/rook/ceph/cleanup.go @@ -62,6 +62,12 @@ var cleanUpRadosNamespaceCmd = &cobra.Command{ Short: "Starts the cleanup process for a CephBlockPoolRadosNamespace", } +var cleanUpBlockPoolCmd = &cobra.Command{ + // the subcommand matches CRD kind of the custom resource to be cleaned up + Use: "CephBlockPool", + Short: "Starts the cleanup process for a CephBlockPool", +} + func init() { cleanUpHostCmd.Flags().StringVar(&dataDirHostPath, "data-dir-host-path", "", "dataDirHostPath on the node") cleanUpHostCmd.Flags().StringVar(&namespaceDir, "namespace-dir", "", "dataDirHostPath on the node") @@ -74,11 +80,12 @@ func init() { flags.SetFlagsFromEnv(cleanUpHostCmd.Flags(), rook.RookEnvVarPrefix) flags.SetFlagsFromEnv(cleanUpSubVolumeGroupCmd.Flags(), rook.RookEnvVarPrefix) - cleanUpCmd.AddCommand(cleanUpHostCmd, cleanUpSubVolumeGroupCmd, cleanUpRadosNamespaceCmd) + cleanUpCmd.AddCommand(cleanUpHostCmd, cleanUpSubVolumeGroupCmd, cleanUpRadosNamespaceCmd, cleanUpBlockPoolCmd) cleanUpHostCmd.RunE = startHostCleanUp cleanUpSubVolumeGroupCmd.RunE = startSubVolumeGroupCleanUp cleanUpRadosNamespaceCmd.RunE = startRadosNamespaceCleanup + cleanUpBlockPoolCmd.RunE = startBlockPoolCleanup } func startHostCleanUp(cmd *cobra.Command, args []string) error { @@ -174,3 +181,25 @@ func startRadosNamespaceCleanup(cmd *cobra.Command, args []string) error { return nil } + +func startBlockPoolCleanup(cmd *cobra.Command, args []string) error { + rook.SetLogLevel() + rook.LogStartupInfo(cleanUpRadosNamespaceCmd.Flags()) + + ctx := cmd.Context() + context := createContext() + namespace := os.Getenv(k8sutil.PodNamespaceEnvVar) + clusterInfo := client.AdminClusterInfo(ctx, namespace, "") + + poolName := os.Getenv(opcontroller.CephBlockPoolNameEnv) + if poolName == "" { + rook.TerminateFatal(fmt.Errorf("cephblockpool name is not available in the pod environment variables")) + } + + err := cleanup.BlockPoolCleanup(context, clusterInfo, poolName) + if err != nil { + rook.TerminateFatal(fmt.Errorf("failed to cleanup cephBlockPool %q resource %v", poolName, err)) + } + + return nil +} diff --git a/pkg/daemon/ceph/cleanup/radosnamespace.go b/pkg/daemon/ceph/cleanup/radosnamespace.go index 1ebb6f09a..a7381b0d1 100644 --- a/pkg/daemon/ceph/cleanup/radosnamespace.go +++ b/pkg/daemon/ceph/cleanup/radosnamespace.go @@ -17,6 +17,8 @@ limitations under the License. package cleanup import ( + "fmt" + "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" @@ -26,46 +28,67 @@ import ( func RadosNamespaceCleanup(context *clusterd.Context, clusterInfo *client.ClusterInfo, poolName, radosNamespace string) error { logger.Infof("starting clean up of CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName) + err := cleanupImages(context, clusterInfo, poolName, radosNamespace) + if err != nil { + logger.Errorf("failed to clean up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName) + return err + } + + logger.Infof("successfully cleaned up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName) + return nil +} + +func cleanupImages(context *clusterd.Context, clusterInfo *client.ClusterInfo, poolName, radosNamespace string) error { + msg := fmt.Sprintf("cephblockpool %q", poolName) + if radosNamespace != "" { + msg = fmt.Sprintf("%s in rados namespace %q", msg, radosNamespace) + } images, err := cephclient.ListImagesInRadosNamespace(context, clusterInfo, poolName, radosNamespace) if err != nil { - return errors.Wrapf(err, "failed to list images in cephblockpool %q in rados namespace %q", poolName, radosNamespace) + return errors.Wrapf(err, "failed to list images in %s", msg) } var retErr error for _, image := range images { snaps, err := cephclient.ListSnapshotsInRadosNamespace(context, clusterInfo, poolName, image.Name, radosNamespace) if err != nil { - retErr = errors.Wrapf(err, "failed to list snapshots for the image %q in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace) + retErr = errors.Wrapf(err, "failed to list snapshots for the image %q in %s", image.Name, msg) logger.Error(retErr) } for _, snap := range snaps { err := cephclient.DeleteSnapshotInRadosNamespace(context, clusterInfo, poolName, image.Name, snap.Name, radosNamespace) if err != nil { - retErr = errors.Wrapf(err, "failed to delete snapshot %q of the image %q in cephblockpool %q in rados namespace %q.", snap.Name, image.Name, poolName, radosNamespace) + retErr = errors.Wrapf(err, "failed to delete snapshot %q of the image %q in %s", snap.Name, image.Name, msg) logger.Error(retErr) } else { - logger.Infof("successfully deleted snapshot %q of image %q in cephblockpool %q in rados namespace %q", snap.Name, image.Name, poolName, radosNamespace) + logger.Infof("successfully deleted snapshot %q of image %q in %s", snap.Name, image.Name, msg) } } err = cephclient.MoveImageToTrashInRadosNamespace(context, clusterInfo, poolName, image.Name, radosNamespace) if err != nil { - retErr = errors.Wrapf(err, "failed to move image %q to trash in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace) + retErr = errors.Wrapf(err, "failed to move image %q to trash in %s", image.Name, msg) logger.Error(retErr) } err = cephclient.DeleteImageFromTrashInRadosNamespace(context, clusterInfo, poolName, image.ID, radosNamespace) if err != nil { - retErr = errors.Wrapf(err, "failed to add task to remove image %q from trash in cephblockpool %q in rados namespace %q.", image.Name, poolName, radosNamespace) + retErr = errors.Wrapf(err, "failed to add task to remove image %q from trash in %s", image.Name, msg) logger.Error(retErr) } } + return retErr +} - if retErr != nil { - logger.Errorf("failed to clean up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName) - return retErr +func BlockPoolCleanup(context *clusterd.Context, clusterInfo *client.ClusterInfo, poolName string) error { + logger.Infof("starting clean up of CephBlockPool %q resource", poolName) + + err := cleanupImages(context, clusterInfo, poolName, "") + if err != nil { + logger.Errorf("failed to clean up CephBlockPool %q resource", poolName) + return err } - logger.Infof("successfully cleaned up CephBlockPoolRadosNamespace %q resources in cephblockpool %q", radosNamespace, poolName) + logger.Infof("successfully cleaned up CephBlockPool %q resource", poolName) return nil } diff --git a/pkg/daemon/ceph/cleanup/radosnamespace_test.go b/pkg/daemon/ceph/cleanup/radosnamespace_test.go index eb6d7aca2..6010982e4 100644 --- a/pkg/daemon/ceph/cleanup/radosnamespace_test.go +++ b/pkg/daemon/ceph/cleanup/radosnamespace_test.go @@ -90,3 +90,56 @@ func TestRadosNamespace(t *testing.T) { assert.NoError(t, err) }) } + +func TestBlockPoolCleanup(t *testing.T) { + clusterInfo := cephclient.AdminTestClusterInfo("mycluster") + poolName := "test-pool" + + t.Run("no images in blockpool", func(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + if args[0] == "ls" && args[1] == "-l" { + assert.Equal(t, poolName, args[2]) + return "", nil + } + return "", errors.New("unknown command") + } + context := &clusterd.Context{Executor: executor} + err := BlockPoolCleanup(context, clusterInfo, poolName) + assert.NoError(t, err) + }) + + t.Run("images with snapshots available in blockpool", func(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + // list all subvolumes in subvolumegroup + if args[0] == "ls" && args[1] == "-l" { + assert.Equal(t, poolName, args[2]) + return mockImageLSResponse, nil + } + if args[0] == "snap" && args[1] == "ls" { + assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d", args[2]) + return mockSnapshotsResponse, nil + } + if args[0] == "snap" && args[1] == "rm" { + assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d@snap1", args[2]) + return "", nil + } + if args[0] == "trash" && args[1] == "mv" { + assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d", args[2]) + return "", nil + } + if args[0] == "rbd" && args[1] == "task" && args[2] == "add" && args[3] == "trash" { + // pool-name/image-id + assert.Equal(t, "test-pool/16e35cfa56a7", args[5]) + return "", nil + } + return "", errors.New("unknown command") + } + context := &clusterd.Context{Executor: executor} + err := BlockPoolCleanup(context, clusterInfo, poolName) + assert.NoError(t, err) + }) +} diff --git a/pkg/daemon/ceph/client/image.go b/pkg/daemon/ceph/client/image.go index f16fa2334..8ea707d9c 100644 --- a/pkg/daemon/ceph/client/image.go +++ b/pkg/daemon/ceph/client/image.go @@ -85,7 +85,10 @@ func ListImagesInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterI // ListSnapshotsInRadosNamespace lists all the snapshots created for an image in a cephblockpool in a given rados namespace func ListSnapshotsInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, namespace string) ([]CephBlockImageSnapshot, error) { snapshots := []CephBlockImageSnapshot{} - args := []string{"snap", "ls", getImageSpec(imageName, poolName), "--namespace", namespace} + args := []string{"snap", "ls", getImageSpec(imageName, poolName)} + if namespace != "" { + args = append(args, "--namespace", namespace) + } cmd := NewRBDCommand(context, clusterInfo, args) cmd.JsonOutput = true buf, err := cmd.Run() @@ -101,7 +104,10 @@ func ListSnapshotsInRadosNamespace(context *clusterd.Context, clusterInfo *Clust // DeleteSnapshotInRadosNamespace deletes a image snapshot created in block pool in a given rados namespace func DeleteSnapshotInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, snapshot, namespace string) error { - args := []string{"snap", "rm", getImageSnapshotSpec(poolName, imageName, snapshot), "--namespace", namespace} + args := []string{"snap", "rm", getImageSnapshotSpec(poolName, imageName, snapshot)} + if namespace != "" { + args = append(args, "--namespace", namespace) + } cmd := NewRBDCommand(context, clusterInfo, args) _, err := cmd.Run() if err != nil { @@ -112,7 +118,10 @@ func DeleteSnapshotInRadosNamespace(context *clusterd.Context, clusterInfo *Clus // MoveImageToTrashInRadosNamespace moves the cephblockpool image to trash in the rados namespace func MoveImageToTrashInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, namespace string) error { - args := []string{"trash", "mv", getImageSpec(imageName, poolName), "--namespace", namespace} + args := []string{"trash", "mv", getImageSpec(imageName, poolName)} + if namespace != "" { + args = append(args, "--namespace", namespace) + } cmd := NewRBDCommand(context, clusterInfo, args) _, err := cmd.Run() if err != nil { @@ -268,8 +277,12 @@ func getImageSpec(name, poolName string) string { } func getImageSpecInRadosNamespace(poolName, namespace, imageID string) string { + if namespace == "" { + return fmt.Sprintf("%s/%s", poolName, imageID) + } return fmt.Sprintf("%s/%s/%s", poolName, namespace, imageID) } + func getImageSnapshotSpec(poolName, imageName, snapshot string) string { return fmt.Sprintf("%s/%s@%s", poolName, imageName, snapshot) } diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index 14f57c341..4ca91b481 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -76,6 +76,7 @@ type ReconcileCephBlockPool struct { blockPoolContexts map[string]*blockPoolHealth opManagerContext context.Context recorder record.EventRecorder + opConfig opcontroller.OperatorConfig } type blockPoolHealth struct { @@ -87,11 +88,11 @@ type blockPoolHealth struct { // Add creates a new CephBlockPool Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { - return add(opManagerContext, mgr, newReconciler(mgr, context, opManagerContext)) + return add(opManagerContext, mgr, newReconciler(mgr, context, opManagerContext, opConfig)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { return &ReconcileCephBlockPool{ client: mgr.GetClient(), scheme: mgr.GetScheme(), @@ -99,6 +100,7 @@ func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerCont blockPoolContexts: make(map[string]*blockPoolHealth), opManagerContext: opManagerContext, recorder: mgr.GetEventRecorderFor("rook-" + controllerName), + opConfig: opConfig, } } @@ -245,6 +247,12 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile logger.Infof("deleting pool %q", poolSpec.Name) err = deletePool(r.context, clusterInfo, &poolSpec) if err != nil { + if opcontroller.ForceDeleteRequested(cephBlockPool.GetAnnotations()) { + cleanupErr := r.cleanup(cephBlockPool, &cephCluster) + if cleanupErr != nil { + return reconcile.Result{}, *cephBlockPool, errors.Wrapf(cleanupErr, "failed to create clean up job for ceph blockpool %q", cephBlockPool.Name) + } + } return opcontroller.ImmediateRetryResult, *cephBlockPool, errors.Wrapf(err, "failed to delete pool %q. ", cephBlockPool.Name) } @@ -373,6 +381,20 @@ func (r *ReconcileCephBlockPool) reconcileCreatePool(clusterInfo *cephclient.Clu return reconcile.Result{}, nil } +func (r *ReconcileCephBlockPool) cleanup(cephblockpool *cephv1.CephBlockPool, cephCluster *cephv1.CephCluster) error { + logger.Infof("starting cleanup of the ceph resources for CephBlockPool %q in k8s namespace %q", cephblockpool.Name, cephblockpool.Namespace) + cleanupConfig := map[string]string{ + opcontroller.CephBlockPoolNameEnv: cephblockpool.Name, + } + cleanup := opcontroller.NewResourceCleanup(cephblockpool, cephCluster, r.opConfig.Image, cleanupConfig) + jobName := k8sutil.TruncateNodeNameForJob("cleanup-cephblockpool-%s", cephblockpool.Name) + err := cleanup.StartJob(r.clusterInfo.Context, r.context.Clientset, jobName) + if err != nil { + return errors.Wrapf(err, "failed to run clean up job to clean the cephblockpool %q", cephblockpool.Name) + } + return nil +} + // Create the pool func createPool(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, clusterSpec *cephv1.ClusterSpec, p *cephv1.NamedPoolSpec) error { // Set the application name to rbd by default, but override later for special pools