feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

osd: wipe encryted osd disk from other cluster

Browse Source 
During the OSD preparation step, if the encrypted OSD disk belongs to a
different cluster, then clean the disk if `wipeDeviceFromOtherClusters`
is set to true.

Signed-off-by: Santosh Pillai <sapillai@redhat.com>
This commit is contained in:
Santosh Pillai
2025-06-11 15:22:28 +05:30
parent 2c1bb7443a
commit 71bcdb533e
2 changed files with 112 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/daemon/ceph/osd/volume.go
+65 -16
View File
@@ -877,21 +877,34 @@ func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error
for _, existingOSD := range existingOSDs {
// Wipe the devices that will be used for preparing OSD but already have OSD metadata from another ceph cluster
if existingOSD.CephFsid != a.clusterInfo.FSID {
device := desiredDeviceFromOtherCluster(context.Devices, existingOSD.Device)
if device != nil {
logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", existingOSD.OsdID, existingOSD.Device, existingOSD.CephFsid)
logger.Infof("zap OSD.%d on device path %q", existingOSD.OsdID, existingOSD.Device)
output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", existingOSD.Device)
osdID := existingOSD.OsdID
deviceToWipe := existingOSD.Device
osdDisk, encryptedBlock, err := getOSDDiskToBeWiped(context, existingOSD.Device)
if err != nil {
return errors.Wrapf(err, "failed to get the actual device path to be wiped for existing OSD %d with path %q", osdID, osdDisk.RealPath)
}
if osdDisk != nil {
deviceToWipe := osdDisk.RealPath
if encryptedBlock != "" {
err = removeEncryptedDevice(context, encryptedBlock)
if err != nil {
logger.Warningf("failed to remove stale dm device %q: %q", encryptedBlock, err)
continue
}
}
logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", osdID, deviceToWipe, existingOSD.CephFsid)
logger.Infof("zap OSD.%d on device path %q", osdID, deviceToWipe)
output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", deviceToWipe)
if err != nil {
return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", existingOSD.OsdID, existingOSD.Device, output)
return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", osdID, deviceToWipe, output)
}
logger.Infof("ceph-volume output: %s", output)
logger.Infof("successfully zapped osd.%d path %q", existingOSD.OsdID, existingOSD.Device)
logger.Infof("completed wiping OSD %d device belonging to a different ceph cluster", existingOSD.OsdID)
logger.Infof("successfully zapped osd.%d path %q", osdID, deviceToWipe)
logger.Infof("completed wiping OSD %d device %q belonging to a different ceph cluster", osdID, deviceToWipe)
// Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check.
device.Filesystem = ""
osdDisk.Filesystem = ""
} else {
logger.Infof("skip wiping OSD %d device belonging to a different ceph cluster %q since not a desired device", existingOSD.OsdID, existingOSD.CephFsid)
logger.Infof("skip wiping OSD %d device %q belonging to a different ceph cluster %q since not a desired device", osdID, deviceToWipe, existingOSD.CephFsid)
}
}
}
@@ -899,14 +912,28 @@ func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error
return nil
}
// desiredDeviceFromOtherCluster returns that desired device that needs to be wiped as it belongs to another cluster
func desiredDeviceFromOtherCluster(desiredDevices []*sys.LocalDisk, deviceFromOtherCluster string) *sys.LocalDisk {
for _, desiredDevice := range desiredDevices {
if desiredDevice.RealPath == deviceFromOtherCluster {
return desiredDevice
// getOSDDiskToBeWiped returns OSD disk path and the dmcrypt block in case of encrypted OSDs
func getOSDDiskToBeWiped(context *clusterd.Context, existingOSDDevice string) (*sys.LocalDisk, string, error) {
var err error
var encryptedBlock string
// encrypted OSDs have /dev/mapper/* entries. Find the real device path in case of encrypted OSDs
if strings.Contains(existingOSDDevice, "mapper") {
encryptedBlock = existingOSDDevice
existingOSDDevice, err = GetBackingDeviceForEncryptedBlock(context, existingOSDDevice)
if err != nil {
logger.Warningf("failed to get actual device used for the dmcrypt block %q: %q", encryptedBlock, err)
return nil, "", nil
}
logger.Infof("%q is the actual disk behind the %q encrypted block", existingOSDDevice, encryptedBlock)
}
var osdDisk *sys.LocalDisk
for _, desiredDevice := range context.Devices {
if desiredDevice.RealPath == existingOSDDevice {
osdDisk = desiredDevice
}
}
return nil
return osdDisk, encryptedBlock, nil
}
func lvmPreReq(context *clusterd.Context) error {
@@ -1336,3 +1363,25 @@ func isInOSDInfoList(uuid string, osds []oposd.OSDInfo) bool {
return false
}
func GetBackingDeviceForEncryptedBlock(context *clusterd.Context, disk string) (string, error) {
output, err := context.Executor.ExecuteCommandWithOutput(cryptsetupBinary, "status", disk, "-v")
if err != nil {
return "", errors.Wrapf(err, "failed to run cryptsetup status on %s", disk)
}
// Example output line: "device: /dev/sdb1"
lines := strings.Split(output, "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "device:") {
parts := strings.Fields(line)
logger.Infof("parts: %s", parts)
if len(parts) == 2 {
return parts[1], nil
}
}
}
return "", errors.Errorf("failed to find backing device for encrypted block %q", disk)
}
pkg/daemon/ceph/osd/volume_test.go
+47 -3
View File
@@ -272,6 +272,24 @@ var cephVolumeRAWTestResult = `{
}
`
var cephVolumeRAWEncryptedTestResult = `{
"0": {
"ceph_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89",
"device": "/dev/mapper/set2-data-0jkntr-block-dmcrypt",
"osd_id": 0,
"osd_uuid": "c03d7353-96e5-4a41-98de-830dfff97d06",
"type": "bluestore"
},
"1": {
"ceph_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89",
"device": "/dev/mapper/set3-data-0sc77p-block-dmcrypt",
"osd_id": 1,
"osd_uuid": "62132914-e779-48cf-8f55-fbc9692c8ce5",
"type": "bluestore"
}
}
`
var cephVolumeRawPartitionTestResult = `{
"0": {
"ceph_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89",
@@ -2135,13 +2153,13 @@ func TestWipeDevicesFromOtherClusters(t *testing.T) {
return "", errors.Errorf("unknown command %s %s", command, args)
}
// "/dev/vdc" should not be zapped
// `ceph-volume raw list` returns OSDs on "vdb" and "vdc" but only "vdb" should be zapped
executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
devicePath := "/dev/vdc"
devicePath := "/dev/vdb"
logger.Infof("%s %v", command, args)
if contains(args, "zap") {
if contains(args, devicePath) {
if !contains(args, devicePath) {
return "", errors.Errorf("device %s should not be zapped", devicePath)
}
return "", nil
@@ -2154,4 +2172,30 @@ func TestWipeDevicesFromOtherClusters(t *testing.T) {
context.Executor = executor
err := agent.WipeDevicesFromOtherClusters(context)
assert.NoError(t, err)
// `ceph-volume raw list` returns dmcrypt devices on "vdb" and "vdc" but only "vdb" should be zapped
executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
logger.Infof("%s %v", command, args)
if contains(args, "raw") && contains(args, "list") {
return cephVolumeRAWEncryptedTestResult, nil
}
if command == cryptsetupBinary && args[0] == "status" && args[1] == "/dev/mapper/set2-data-0jkntr-block-dmcrypt" {
return `type: LUKS1
device: /dev/vdb`,
nil
}
if command == cryptsetupBinary && args[0] == "status" && args[1] == "/dev/mapper/set3-data-0sc77p-block-dmcrypt" {
return `type: LUKS1
device: /dev/vdc`,
nil
}
return "", errors.Errorf("unknown command %s %s", command, args)
}
context = &clusterd.Context{
Devices: []*sys.LocalDisk{{RealPath: "/dev/vdb"}},
}
context.Executor = executor
err = agent.WipeDevicesFromOtherClusters(context)
assert.NoError(t, err)
}