osd: zap disks for forceful OSD installation
When cephCluster is resinstalled and WipeDevicesFromOtherClusters is set
to true, then Rook cleans up the OSD disks using ceph-volum lvm zap, so
that new OSD can created on this disk. This PR removes the dependencies
of the "ceph-volume lvm zap" and provides implements the entry disk
"ceph-volume lvm zap" logic within rook
Signed-off-by: Santosh <sapillai@redhat.com>
(cherry picked from commit 92324588ba)
This commit is contained in:
1 parent
4dbc2ea3c5
commit
00b127ceb4
2 files changed
+105
-12
No files matched your search
@@ -861,6 +861,69 @@ func (a *OsdAgent) appendDeviceClassArg(device *DeviceOsdIDEntry, args []string)
|
||||
return args
|
||||
}
|
||||
|
||||
// ZapDevice wipes a device using two methods for maximum reliability:
|
||||
// 1. First, run ceph-volume lvm zap to let Ceph handle cleanup
|
||||
// 2. Then, perform direct cleanup as a follow-up to ensure all metadata is removed:
|
||||
// a. Unmount the device if mounted
|
||||
// b. Remove filesystem signatures with wipefs
|
||||
// c. Remove BlueStore signatures with ceph-bluestore-tool
|
||||
// d. Zero the first 10MB of the device to clear any previous filesystem traces
|
||||
//
|
||||
// Running both methods ensures the device is fully cleaned regardless of
|
||||
// changes in ceph-volume behavior across Ceph versions.
|
||||
func ZapDevice(context *clusterd.Context, devicePath string) error {
|
||||
// First, attempt ceph-volume lvm zap
|
||||
output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", devicePath)
|
||||
if err != nil {
|
||||
// log and continue with direct cleanup
|
||||
logger.Warningf("ceph-volume lvm zap failed for %q (will continue with direct cleanup): %s. %v", devicePath, output, err)
|
||||
} else {
|
||||
logger.Infof("ceph-volume lvm zap output for %q: %s", devicePath, output)
|
||||
}
|
||||
|
||||
// Follow up with Rook's internal cleanup to ensure all metadata is removed
|
||||
logger.Infof("running rook's internal cleanup for device %q", devicePath)
|
||||
|
||||
// Unmount the device if it's mounted
|
||||
if err := unmountDevice(context, devicePath); err != nil {
|
||||
logger.Warningf("failed to unmount %q (may not be mounted): %v", devicePath, err)
|
||||
}
|
||||
|
||||
// Remove all filesystem signatures with wipefs
|
||||
output, err = context.Executor.ExecuteCommandWithCombinedOutput("wipefs", "--all", devicePath)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to wipefs on %q: %s", devicePath, output)
|
||||
}
|
||||
logger.Infof("wipefs output for %q: %s", devicePath, output)
|
||||
|
||||
// Remove any BlueStore signatures
|
||||
output, err = context.Executor.ExecuteCommandWithCombinedOutput("ceph-bluestore-tool", "zap-device", "--dev", devicePath, "--yes-i-really-really-mean-it")
|
||||
if err != nil {
|
||||
// Non-fatal: device may not have BlueStore signatures
|
||||
logger.Warningf("ceph-bluestore-tool zap-device failed for %q (may not have BlueStore data): %s. %v", devicePath, output, err)
|
||||
}
|
||||
|
||||
// Zero out the first 10MB to clear any remaining metadata
|
||||
output, err = context.Executor.ExecuteCommandWithCombinedOutput("dd", "if=/dev/zero", fmt.Sprintf("of=%s", devicePath), "bs=1M", "count=10", "conv=fsync")
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to zero device %q: %s", devicePath, output)
|
||||
}
|
||||
|
||||
logger.Infof("successfully zapped device %q", devicePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func unmountDevice(context *clusterd.Context, devicePath string) error {
|
||||
output, err := context.Executor.ExecuteCommandWithCombinedOutput("umount", devicePath)
|
||||
if err != nil {
|
||||
if strings.Contains(output, "not mounted") || strings.Contains(err.Error(), "not mounted") {
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "failed to unmount %q: %s", devicePath, output)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WipeDevicesFromOtherClusters wipes the OSD backed disks if they have metadata from a different ceph cluster.
|
||||
// The wiped disks can then be used to prepare OSDs for the current ceph cluster.
|
||||
func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error {
|
||||
@@ -908,11 +971,9 @@ func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error
|
||||
}
|
||||
logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", osdID, deviceToWipe, existingOSD.CephFsid)
|
||||
logger.Infof("zap OSD.%d on device path %q", osdID, deviceToWipe)
|
||||
output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", deviceToWipe)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", osdID, deviceToWipe, output)
|
||||
if err := ZapDevice(context, deviceToWipe); err != nil {
|
||||
return errors.Wrapf(err, "failed to zap osd.%d path %q", osdID, deviceToWipe)
|
||||
}
|
||||
logger.Infof("ceph-volume output: %s", output)
|
||||
logger.Infof("successfully zapped osd.%d path %q", osdID, deviceToWipe)
|
||||
logger.Infof("completed wiping OSD %d device %q belonging to a different ceph cluster", osdID, deviceToWipe)
|
||||
// Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check.
|
||||
@@ -945,9 +1006,8 @@ func wipeEncryptedDevicesFromOtherClusters(context *clusterd.Context, currentClu
|
||||
currentDiskCephFSID := strings.SplitAfter(ceph_fsid, "=")[1]
|
||||
if currentDiskCephFSID != currentClusterFSID {
|
||||
logger.Infof("cleaning encrypted disk %q (%q) that is part of a different ceph cluster %q", disk.Name, disk.RealPath, currentDiskCephFSID)
|
||||
output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", disk.RealPath)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to zap encrypted disk with different ceph cluster %q. %s.", disk.RealPath, output)
|
||||
if err := ZapDevice(context, disk.RealPath); err != nil {
|
||||
return errors.Wrapf(err, "failed to zap encrypted disk with different ceph cluster %q", disk.RealPath)
|
||||
}
|
||||
logger.Infof("completed wiping device %q belonging to a different ceph cluster", disk.RealPath)
|
||||
// Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check.
|
||||
|
||||
@@ -2240,12 +2240,28 @@ func TestWipeDevicesFromOtherClusters(t *testing.T) {
|
||||
return luksDump, nil
|
||||
}
|
||||
|
||||
if slices.Contains(args, "zap") {
|
||||
if !slices.Contains(args, devicePath) {
|
||||
return "", errors.Errorf("device %s should not be zapped", devicePath)
|
||||
// ZapDevice calls: stdbuf (ceph-volume lvm zap), umount, wipefs, ceph-bluestore-tool, dd
|
||||
if command == "stdbuf" {
|
||||
if !slices.Contains(args, "zap") || !slices.Contains(args, devicePath) {
|
||||
return "", errors.Errorf("device %s should not be zapped, got %v", devicePath, args)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if command == "umount" {
|
||||
return "not mounted", errors.New("not mounted")
|
||||
}
|
||||
if command == "wipefs" {
|
||||
if args[1] != devicePath {
|
||||
return "", errors.Errorf("device %s should not be zapped, got %s", devicePath, args[1])
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if command == "ceph-bluestore-tool" {
|
||||
return "", nil
|
||||
}
|
||||
if command == "dd" {
|
||||
return "", nil
|
||||
}
|
||||
return "", errors.Errorf("unknown command %s %s", command, args)
|
||||
}
|
||||
context := &clusterd.Context{
|
||||
@@ -2309,12 +2325,29 @@ func TestWipeDevicesFromOtherClusters(t *testing.T) {
|
||||
executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
|
||||
logger.Infof("%s %v", command, args)
|
||||
devicePath := "/dev/mapper/rhel-ceph--data"
|
||||
if slices.Contains(args, "zap") {
|
||||
if !slices.Contains(args, devicePath) {
|
||||
|
||||
// ZapDevice calls: stdbuf (ceph-volume lvm zap), umount, wipefs, ceph-bluestore-tool, dd
|
||||
if command == "stdbuf" {
|
||||
if !slices.Contains(args, "zap") || !slices.Contains(args, devicePath) {
|
||||
return "", errors.Errorf("expected device %s to be zapped but got %v", devicePath, args)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if command == "umount" {
|
||||
return "not mounted", errors.New("not mounted")
|
||||
}
|
||||
if command == "wipefs" {
|
||||
if args[1] != devicePath {
|
||||
return "", errors.Errorf("expected device %s to be zapped but got %v", devicePath, args)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if command == "ceph-bluestore-tool" {
|
||||
return "", nil
|
||||
}
|
||||
if command == "dd" {
|
||||
return "", nil
|
||||
}
|
||||
return "", errors.Errorf("unknown command %s %s", command, args)
|
||||
}
|
||||
context = &clusterd.Context{
|
||||
|
||||
Reference in new issue
Block a user