osd: zap disks for forceful OSD installation

When cephCluster is resinstalled and WipeDevicesFromOtherClusters is set
to true, then Rook cleans up the OSD disks using ceph-volum lvm zap, so
that new OSD can created on this disk. This PR removes the dependencies
of the "ceph-volume lvm zap" and provides implements the entry disk
"ceph-volume lvm zap" logic within rook

Signed-off-by: Santosh <sapillai@redhat.com>
(cherry picked from commit 92324588ba)

pkg/daemon/ceph/osd/volume.go

@@ -853,6 +853,69 @@ func (a *OsdAgent) appendDeviceClassArg(device *DeviceOsdIDEntry, args []string)
 	return args
 }
 
+// ZapDevice wipes a device using two methods for maximum reliability:
+//  1. First, run ceph-volume lvm zap to let Ceph handle cleanup
+//  2. Then, perform direct cleanup as a follow-up to ensure all metadata is removed:
+//     a. Unmount the device if mounted
+//     b. Remove filesystem signatures with wipefs
+//     c. Remove BlueStore signatures with ceph-bluestore-tool
+//     d. Zero the first 10MB of the device to clear any previous filesystem traces
+//
+// Running both methods ensures the device is fully cleaned regardless of
+// changes in ceph-volume behavior across Ceph versions.
+func ZapDevice(context *clusterd.Context, devicePath string) error {
+	// First, attempt ceph-volume lvm zap
+	output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", devicePath)
+	if err != nil {
+		// log and continue with direct cleanup
+		logger.Warningf("ceph-volume lvm zap failed for %q (will continue with direct cleanup): %s. %v", devicePath, output, err)
+	} else {
+		logger.Infof("ceph-volume lvm zap output for %q: %s", devicePath, output)
+	}
+
+	// Follow up with Rook's internal cleanup to ensure all metadata is removed
+	logger.Infof("running rook's internal cleanup for device %q", devicePath)
+
+	// Unmount the device if it's mounted
+	if err := unmountDevice(context, devicePath); err != nil {
+		logger.Warningf("failed to unmount %q (may not be mounted): %v", devicePath, err)
+	}
+
+	// Remove all filesystem signatures with wipefs
+	output, err = context.Executor.ExecuteCommandWithCombinedOutput("wipefs", "--all", devicePath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to wipefs on %q: %s", devicePath, output)
+	}
+	logger.Infof("wipefs output for %q: %s", devicePath, output)
+
+	// Remove any BlueStore signatures
+	output, err = context.Executor.ExecuteCommandWithCombinedOutput("ceph-bluestore-tool", "zap-device", "--dev", devicePath, "--yes-i-really-really-mean-it")
+	if err != nil {
+		// Non-fatal: device may not have BlueStore signatures
+		logger.Warningf("ceph-bluestore-tool zap-device failed for %q (may not have BlueStore data): %s. %v", devicePath, output, err)
+	}
+
+	// Zero out the first 10MB to clear any remaining metadata
+	output, err = context.Executor.ExecuteCommandWithCombinedOutput("dd", "if=/dev/zero", fmt.Sprintf("of=%s", devicePath), "bs=1M", "count=10", "conv=fsync")
+	if err != nil {
+		return errors.Wrapf(err, "failed to zero device %q: %s", devicePath, output)
+	}
+
+	logger.Infof("successfully zapped device %q", devicePath)
+	return nil
+}
+
+func unmountDevice(context *clusterd.Context, devicePath string) error {
+	output, err := context.Executor.ExecuteCommandWithCombinedOutput("umount", devicePath)
+	if err != nil {
+		if strings.Contains(output, "not mounted") || strings.Contains(err.Error(), "not mounted") {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to unmount %q: %s", devicePath, output)
+	}
+	return nil
+}
+
 // WipeDevicesFromOtherClusters wipes the OSD backed disks if they have metadata from a different ceph cluster.
 // The wiped disks can then be used to prepare OSDs for the current ceph cluster.
 func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error {
@@ -900,11 +963,9 @@ func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error
 				}
 				logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", osdID, deviceToWipe, existingOSD.CephFsid)
 				logger.Infof("zap OSD.%d on device path %q", osdID, deviceToWipe)
-				output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", deviceToWipe)
-				if err != nil {
-					return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", osdID, deviceToWipe, output)
+				if err := ZapDevice(context, deviceToWipe); err != nil {
+					return errors.Wrapf(err, "failed to zap osd.%d path %q", osdID, deviceToWipe)
 				}
-				logger.Infof("ceph-volume output: %s", output)
 				logger.Infof("successfully zapped osd.%d path %q", osdID, deviceToWipe)
 				logger.Infof("completed wiping OSD %d device %q belonging to a different ceph cluster", osdID, deviceToWipe)
 				// Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check.
@@ -937,9 +998,8 @@ func wipeEncryptedDevicesFromOtherClusters(context *clusterd.Context, currentClu
 			currentDiskCephFSID := strings.SplitAfter(ceph_fsid, "=")[1]
 			if currentDiskCephFSID != currentClusterFSID {
 				logger.Infof("cleaning encrypted disk %q (%q) that is part of a different ceph cluster %q", disk.Name, disk.RealPath, currentDiskCephFSID)
-				output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", disk.RealPath)
-				if err != nil {
-					return errors.Wrapf(err, "failed to zap encrypted disk with different ceph cluster %q. %s.", disk.RealPath, output)
+				if err := ZapDevice(context, disk.RealPath); err != nil {
+					return errors.Wrapf(err, "failed to zap encrypted disk with different ceph cluster %q", disk.RealPath)
 				}
 				logger.Infof("completed wiping device %q belonging to a different ceph cluster", disk.RealPath)
 				// Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check.

pkg/daemon/ceph/osd/volume_test.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"slices"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -300,6 +301,19 @@ var cephVolumeRawPartitionTestResult = `{
     }
 }`
 
+// ceph-volume raw list reports the device via an LVM symlink (/dev/rhel/ceph-data)
+// while the inventory RealPath is /dev/mapper/rhel-ceph--data.
+var cephVolumeRAWLVMSymlinkTestResult = `{
+    "0": {
+        "ceph_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89",
+        "device": "/dev/rhel/ceph-data",
+        "osd_id": 0,
+        "osd_uuid": "c03d7353-96e5-4a41-98de-830dfff97d06",
+        "type": "bluestore"
+    }
+}
+`
+
 func createPVCAvailableDevices() *DeviceOsdMapping {
 	devices := &DeviceOsdMapping{
 		Entries: map[string]*DeviceOsdIDEntry{
@@ -2179,12 +2193,28 @@ func TestWipeDevicesFromOtherClusters(t *testing.T) {
 			return luksDump, nil
 		}
 
-		if contains(args, "zap") {
-			if !contains(args, devicePath) {
-				return "", errors.Errorf("device %s should not be zapped", devicePath)
+		// ZapDevice calls: stdbuf (ceph-volume lvm zap), umount, wipefs, ceph-bluestore-tool, dd
+		if command == "stdbuf" {
+			if !slices.Contains(args, "zap") || !slices.Contains(args, devicePath) {
+				return "", errors.Errorf("device %s should not be zapped, got %v", devicePath, args)
 			}
 			return "", nil
 		}
+		if command == "umount" {
+			return "not mounted", errors.New("not mounted")
+		}
+		if command == "wipefs" {
+			if args[1] != devicePath {
+				return "", errors.Errorf("device %s should not be zapped, got %s", devicePath, args[1])
+			}
+			return "", nil
+		}
+		if command == "ceph-bluestore-tool" {
+			return "", nil
+		}
+		if command == "dd" {
+			return "", nil
+		}
 		return "", errors.Errorf("unknown command %s %s", command, args)
 	}
 	context := &clusterd.Context{
@@ -2234,4 +2264,52 @@ func TestWipeDevicesFromOtherClusters(t *testing.T) {
 	context.Executor = executor
 	err = agent.WipeDevicesFromOtherClusters(context)
 	assert.NoError(t, err)
+
+	// `ceph-volume raw list` returns an LVM symlink path (/dev/rhel/ceph-data) while
+	// the device RealPath is /dev/mapper/rhel-ceph--data. The device should still be
+	// matched via DevLinks and zapped.
+	executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+		logger.Infof("%s %v", command, args)
+		if slices.Contains(args, "raw") && slices.Contains(args, "list") {
+			return cephVolumeRAWLVMSymlinkTestResult, nil
+		}
+		return "", errors.Errorf("unknown command %s %s", command, args)
+	}
+	executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
+		logger.Infof("%s %v", command, args)
+		devicePath := "/dev/mapper/rhel-ceph--data"
+
+		// ZapDevice calls: stdbuf (ceph-volume lvm zap), umount, wipefs, ceph-bluestore-tool, dd
+		if command == "stdbuf" {
+			if !slices.Contains(args, "zap") || !slices.Contains(args, devicePath) {
+				return "", errors.Errorf("expected device %s to be zapped but got %v", devicePath, args)
+			}
+			return "", nil
+		}
+		if command == "umount" {
+			return "not mounted", errors.New("not mounted")
+		}
+		if command == "wipefs" {
+			if args[1] != devicePath {
+				return "", errors.Errorf("expected device %s to be zapped but got %v", devicePath, args)
+			}
+			return "", nil
+		}
+		if command == "ceph-bluestore-tool" {
+			return "", nil
+		}
+		if command == "dd" {
+			return "", nil
+		}
+		return "", errors.Errorf("unknown command %s %s", command, args)
+	}
+	context = &clusterd.Context{
+		Devices: []*sys.LocalDisk{{
+			RealPath: "/dev/mapper/rhel-ceph--data",
+			DevLinks: "/dev/rhel/ceph-data /dev/disk/by-id/dm-name-rhel-ceph--data /dev/mapper/rhel-ceph--data",
+		}},
+	}
+	context.Executor = executor
+	err = agent.WipeDevicesFromOtherClusters(context)
+	assert.NoError(t, err)
 }