From 24fa9ed935b3287acaf2867c58d015f5addd1b43 Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Tue, 6 Dec 2022 13:12:54 +0530 Subject: [PATCH] osd: remove stale dm device during osd-prepare-job During osd-prepare-job, The encrypted device is found to be closed in some cases when the OSD deployment has been removed manually accompanied by any of following cases: - node reboot - csi managed PVC being unmounted etc while re-opening the block to re-hydrate the OSDInfo, the dm device `-block-dmcrypt` clashes with the one used by OSD pod which is stale by now. This commit adds cmd to remove this stale dm device. Error with output "No such device" is ignored. Signed-off-by: Rakshith R --- pkg/daemon/ceph/osd/encryption.go | 19 ++++++++++++++++--- pkg/daemon/ceph/osd/volume.go | 5 +++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/pkg/daemon/ceph/osd/encryption.go b/pkg/daemon/ceph/osd/encryption.go index e9dfbf094..3adb6529d 100644 --- a/pkg/daemon/ceph/osd/encryption.go +++ b/pkg/daemon/ceph/osd/encryption.go @@ -33,9 +33,10 @@ import ( ) const ( - cryptsetupBinary = "cryptsetup" - dmsetupBinary = "dmsetup" - luksOpenCmdTimeOut = 90 * time.Second + cryptsetupBinary = "cryptsetup" + dmsetupBinary = "dmsetup" + luksOpenCmdTimeOut = 90 * time.Second + removeEncryptedDeviceCmdTimeOut = 30 * time.Second ) var ( @@ -172,6 +173,18 @@ func openEncryptedDevice(context *clusterd.Context, disk, target, passphrase str return nil } +func removeEncryptedDevice(context *clusterd.Context, target string) error { + args := []string{"remove", "--force", target} + output, err := context.Executor.ExecuteCommandWithTimeout(removeEncryptedDeviceCmdTimeOut, "dmsetup", args...) + // ignore error if no device was found. + if err != nil { + return errors.Wrapf(err, "failed to remove dm device %q: %q", target, output) + } + logger.Debugf("successfully removed stale dm device %q", target) + + return nil +} + func isCephEncryptedBlock(context *clusterd.Context, currentClusterFSID string, disk string) bool { metadata, err := dumpLUKS(context, disk) if err != nil { diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 3f20fb782..dcf9e927b 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -974,6 +974,11 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster } target := oposd.EncryptionDMName(pvcName, oposd.DmcryptBlockType) + // remove stale dm device left by previous OSD. + err = removeEncryptedDevice(context, target) + if err != nil { + logger.Warningf("failed to remove stale dm device %q: %q", target, err) + } err = openEncryptedDevice(context, block, target, passphrase) if err != nil { return nil, errors.Wrapf(err, "failed to open encrypted block device %q on %q", block, target)