diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index f747e5533..b48dd4ec6 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -877,21 +877,34 @@ func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error for _, existingOSD := range existingOSDs { // Wipe the devices that will be used for preparing OSD but already have OSD metadata from another ceph cluster if existingOSD.CephFsid != a.clusterInfo.FSID { - device := desiredDeviceFromOtherCluster(context.Devices, existingOSD.Device) - if device != nil { - logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", existingOSD.OsdID, existingOSD.Device, existingOSD.CephFsid) - logger.Infof("zap OSD.%d on device path %q", existingOSD.OsdID, existingOSD.Device) - output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", existingOSD.Device) + osdID := existingOSD.OsdID + deviceToWipe := existingOSD.Device + osdDisk, encryptedBlock, err := getOSDDiskToBeWiped(context, existingOSD.Device) + if err != nil { + return errors.Wrapf(err, "failed to get the actual device path to be wiped for existing OSD %d with path %q", osdID, osdDisk.RealPath) + } + if osdDisk != nil { + deviceToWipe := osdDisk.RealPath + if encryptedBlock != "" { + err = removeEncryptedDevice(context, encryptedBlock) + if err != nil { + logger.Warningf("failed to remove stale dm device %q: %q", encryptedBlock, err) + continue + } + } + logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", osdID, deviceToWipe, existingOSD.CephFsid) + logger.Infof("zap OSD.%d on device path %q", osdID, deviceToWipe) + output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", deviceToWipe) if err != nil { - return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", existingOSD.OsdID, existingOSD.Device, output) + return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", osdID, deviceToWipe, output) } logger.Infof("ceph-volume output: %s", output) - logger.Infof("successfully zapped osd.%d path %q", existingOSD.OsdID, existingOSD.Device) - logger.Infof("completed wiping OSD %d device belonging to a different ceph cluster", existingOSD.OsdID) + logger.Infof("successfully zapped osd.%d path %q", osdID, deviceToWipe) + logger.Infof("completed wiping OSD %d device %q belonging to a different ceph cluster", osdID, deviceToWipe) // Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check. - device.Filesystem = "" + osdDisk.Filesystem = "" } else { - logger.Infof("skip wiping OSD %d device belonging to a different ceph cluster %q since not a desired device", existingOSD.OsdID, existingOSD.CephFsid) + logger.Infof("skip wiping OSD %d device %q belonging to a different ceph cluster %q since not a desired device", osdID, deviceToWipe, existingOSD.CephFsid) } } } @@ -899,14 +912,28 @@ func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error return nil } -// desiredDeviceFromOtherCluster returns that desired device that needs to be wiped as it belongs to another cluster -func desiredDeviceFromOtherCluster(desiredDevices []*sys.LocalDisk, deviceFromOtherCluster string) *sys.LocalDisk { - for _, desiredDevice := range desiredDevices { - if desiredDevice.RealPath == deviceFromOtherCluster { - return desiredDevice +// getOSDDiskToBeWiped returns OSD disk path and the dmcrypt block in case of encrypted OSDs +func getOSDDiskToBeWiped(context *clusterd.Context, existingOSDDevice string) (*sys.LocalDisk, string, error) { + var err error + var encryptedBlock string + // encrypted OSDs have /dev/mapper/* entries. Find the real device path in case of encrypted OSDs + if strings.Contains(existingOSDDevice, "mapper") { + encryptedBlock = existingOSDDevice + existingOSDDevice, err = GetBackingDeviceForEncryptedBlock(context, existingOSDDevice) + if err != nil { + logger.Warningf("failed to get actual device used for the dmcrypt block %q: %q", encryptedBlock, err) + return nil, "", nil + } + logger.Infof("%q is the actual disk behind the %q encrypted block", existingOSDDevice, encryptedBlock) + } + + var osdDisk *sys.LocalDisk + for _, desiredDevice := range context.Devices { + if desiredDevice.RealPath == existingOSDDevice { + osdDisk = desiredDevice } } - return nil + return osdDisk, encryptedBlock, nil } func lvmPreReq(context *clusterd.Context) error { @@ -1336,3 +1363,25 @@ func isInOSDInfoList(uuid string, osds []oposd.OSDInfo) bool { return false } + +func GetBackingDeviceForEncryptedBlock(context *clusterd.Context, disk string) (string, error) { + output, err := context.Executor.ExecuteCommandWithOutput(cryptsetupBinary, "status", disk, "-v") + if err != nil { + return "", errors.Wrapf(err, "failed to run cryptsetup status on %s", disk) + } + + // Example output line: "device: /dev/sdb1" + lines := strings.Split(output, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "device:") { + parts := strings.Fields(line) + logger.Infof("parts: %s", parts) + if len(parts) == 2 { + return parts[1], nil + } + } + } + + return "", errors.Errorf("failed to find backing device for encrypted block %q", disk) +} diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index c6a56820c..9b1559a8c 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -272,6 +272,24 @@ var cephVolumeRAWTestResult = `{ } ` +var cephVolumeRAWEncryptedTestResult = `{ + "0": { + "ceph_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", + "device": "/dev/mapper/set2-data-0jkntr-block-dmcrypt", + "osd_id": 0, + "osd_uuid": "c03d7353-96e5-4a41-98de-830dfff97d06", + "type": "bluestore" + }, + "1": { + "ceph_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", + "device": "/dev/mapper/set3-data-0sc77p-block-dmcrypt", + "osd_id": 1, + "osd_uuid": "62132914-e779-48cf-8f55-fbc9692c8ce5", + "type": "bluestore" + } +} +` + var cephVolumeRawPartitionTestResult = `{ "0": { "ceph_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", @@ -2135,13 +2153,13 @@ func TestWipeDevicesFromOtherClusters(t *testing.T) { return "", errors.Errorf("unknown command %s %s", command, args) } - // "/dev/vdc" should not be zapped + // `ceph-volume raw list` returns OSDs on "vdb" and "vdc" but only "vdb" should be zapped executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { - devicePath := "/dev/vdc" + devicePath := "/dev/vdb" logger.Infof("%s %v", command, args) if contains(args, "zap") { - if contains(args, devicePath) { + if !contains(args, devicePath) { return "", errors.Errorf("device %s should not be zapped", devicePath) } return "", nil @@ -2154,4 +2172,30 @@ func TestWipeDevicesFromOtherClusters(t *testing.T) { context.Executor = executor err := agent.WipeDevicesFromOtherClusters(context) assert.NoError(t, err) + + // `ceph-volume raw list` returns dmcrypt devices on "vdb" and "vdc" but only "vdb" should be zapped + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if contains(args, "raw") && contains(args, "list") { + return cephVolumeRAWEncryptedTestResult, nil + } + if command == cryptsetupBinary && args[0] == "status" && args[1] == "/dev/mapper/set2-data-0jkntr-block-dmcrypt" { + return `type: LUKS1 + device: /dev/vdb`, + nil + } + if command == cryptsetupBinary && args[0] == "status" && args[1] == "/dev/mapper/set3-data-0sc77p-block-dmcrypt" { + return `type: LUKS1 + device: /dev/vdc`, + nil + } + return "", errors.Errorf("unknown command %s %s", command, args) + } + + context = &clusterd.Context{ + Devices: []*sys.LocalDisk{{RealPath: "/dev/vdb"}}, + } + context.Executor = executor + err = agent.WipeDevicesFromOtherClusters(context) + assert.NoError(t, err) }