diff --git a/pkg/daemon/ceph/osd/encryption.go b/pkg/daemon/ceph/osd/encryption.go
index e9dfbf094..3adb6529d 100644
--- a/pkg/daemon/ceph/osd/encryption.go
+++ b/pkg/daemon/ceph/osd/encryption.go
@@ -33,9 +33,10 @@ import (
 )
 
 const (
-	cryptsetupBinary   = "cryptsetup"
-	dmsetupBinary      = "dmsetup"
-	luksOpenCmdTimeOut = 90 * time.Second
+	cryptsetupBinary                = "cryptsetup"
+	dmsetupBinary                   = "dmsetup"
+	luksOpenCmdTimeOut              = 90 * time.Second
+	removeEncryptedDeviceCmdTimeOut = 30 * time.Second
 )
 
 var (
@@ -172,6 +173,18 @@ func openEncryptedDevice(context *clusterd.Context, disk, target, passphrase str
 	return nil
 }
 
+func removeEncryptedDevice(context *clusterd.Context, target string) error {
+	args := []string{"remove", "--force", target}
+	output, err := context.Executor.ExecuteCommandWithTimeout(removeEncryptedDeviceCmdTimeOut, "dmsetup", args...)
+	// ignore error if no device was found.
+	if err != nil {
+		return errors.Wrapf(err, "failed to remove dm device %q: %q", target, output)
+	}
+	logger.Debugf("successfully removed stale dm device %q", target)
+
+	return nil
+}
+
 func isCephEncryptedBlock(context *clusterd.Context, currentClusterFSID string, disk string) bool {
 	metadata, err := dumpLUKS(context, disk)
 	if err != nil {
diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go
index 3f20fb782..dcf9e927b 100644
--- a/pkg/daemon/ceph/osd/volume.go
+++ b/pkg/daemon/ceph/osd/volume.go
@@ -974,6 +974,11 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
 				}
 
 				target := oposd.EncryptionDMName(pvcName, oposd.DmcryptBlockType)
+				// remove stale dm device left by previous OSD.
+				err = removeEncryptedDevice(context, target)
+				if err != nil {
+					logger.Warningf("failed to remove stale dm device %q: %q", target, err)
+				}
 				err = openEncryptedDevice(context, block, target, passphrase)
 				if err != nil {
 					return nil, errors.Wrapf(err, "failed to open encrypted block device %q on %q", block, target)