diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index 0597ca9b1..bb9077c73 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -4500,6 +4500,19 @@ bool
AllowUninstallWithVolumes defines whether we can proceed with the uninstall if they are RBD images still present
+
+
+wipeDevicesFromOtherClusters
+
+bool
+
+ |
+
+(Optional)
+ WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster
+was reinstalled but OSD disk still contains the metadata from previous ceph cluster.
+ |
+
ClientSpec
diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go
index 7505a94a8..82da36271 100644
--- a/cmd/rook/ceph/osd.go
+++ b/cmd/rook/ceph/osd.go
@@ -66,22 +66,23 @@ var osdRemoveCmd = &cobra.Command{
}
var (
- osdDataDeviceFilter string
- osdDataDevicePathFilter string
- ownerRefID string
- clusterName string
- osdID int
- replaceOSDID int
- osdStoreType string
- osdStringID string
- osdUUID string
- osdIsDevice bool
- pvcBackedOSD bool
- blockPath string
- lvBackedPV bool
- osdIDsToRemove string
- preservePVC string
- forceOSDRemoval string
+ osdDataDeviceFilter string
+ osdDataDevicePathFilter string
+ ownerRefID string
+ clusterName string
+ osdID int
+ replaceOSDID int
+ osdStoreType string
+ osdStringID string
+ osdUUID string
+ osdIsDevice bool
+ pvcBackedOSD bool
+ blockPath string
+ lvBackedPV bool
+ osdIDsToRemove string
+ preservePVC string
+ forceOSDRemoval string
+ wipeDevicesFromOtherClusters bool
)
const (
@@ -102,6 +103,7 @@ func addOSDFlags(command *cobra.Command) {
provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
"true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!")
provisionCmd.Flags().BoolVar(&cfg.pvcBacked, "pvc-backed-osd", false, "true to specify a block mode pvc is backing the OSD")
+ provisionCmd.Flags().BoolVar(&wipeDevicesFromOtherClusters, "wipe-devices-from-other-clusters", false, "wipe the OSD devices that are configured for a different ceph cluster")
// flags for generating the osd config
osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config")
osdConfigCmd.Flags().BoolVar(&osdIsDevice, "is-device", false, "whether the osd is a device")
@@ -274,7 +276,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
}
agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat,
- cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, replaceOSD, cfg.pvcBacked)
+ cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, replaceOSD, cfg.pvcBacked, wipeDevicesFromOtherClusters)
if cfg.metadataDevice != "" {
metaDevice = cfg.metadataDevice
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index b1cc157c7..2bbcaa6b0 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -1498,6 +1498,11 @@ spec:
- quick
type: string
type: object
+ wipeDevicesFromOtherClusters:
+ description: |-
+ WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster
+ was reinstalled but OSD disk still contains the metadata from previous ceph cluster.
+ type: boolean
type: object
continueUpgradeAfterChecksEvenIfNotHealthy:
description: ContinueUpgradeAfterChecksEvenIfNotHealthy defines if an upgrade should continue even if PGs are not clean
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 2b4956bbf..7ce5132f7 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -1496,6 +1496,11 @@ spec:
- quick
type: string
type: object
+ wipeDevicesFromOtherClusters:
+ description: |-
+ WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster
+ was reinstalled but OSD disk still contains the metadata from previous ceph cluster.
+ type: boolean
type: object
continueUpgradeAfterChecksEvenIfNotHealthy:
description: ContinueUpgradeAfterChecksEvenIfNotHealthy defines if an upgrade should continue even if PGs are not clean
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 115f9c65b..0b2b31f5d 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -3030,6 +3030,11 @@ type CleanupPolicySpec struct {
// AllowUninstallWithVolumes defines whether we can proceed with the uninstall if they are RBD images still present
// +optional
AllowUninstallWithVolumes bool `json:"allowUninstallWithVolumes,omitempty"`
+
+ // WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster
+ // was reinstalled but OSD disk still contains the metadata from previous ceph cluster.
+ // +optional
+ WipeDevicesFromOtherClusters bool `json:"wipeDevicesFromOtherClusters"`
}
// CleanupConfirmationProperty represents the cleanup confirmation
diff --git a/pkg/daemon/ceph/osd/agent.go b/pkg/daemon/ceph/osd/agent.go
index 428c1fb95..ca6c7871c 100644
--- a/pkg/daemon/ceph/osd/agent.go
+++ b/pkg/daemon/ceph/osd/agent.go
@@ -30,32 +30,34 @@ const (
// OsdAgent represents the OSD struct of an agent
type OsdAgent struct {
- clusterInfo *cephclient.ClusterInfo
- nodeName string
- forceFormat bool
- devices []DesiredDevice
- metadataDevice string
- storeConfig config.StoreConfig
- kv *k8sutil.ConfigMapKVStore
- pvcBacked bool
- replaceOSD *oposd.OSDInfo
+ clusterInfo *cephclient.ClusterInfo
+ nodeName string
+ forceFormat bool
+ devices []DesiredDevice
+ metadataDevice string
+ storeConfig config.StoreConfig
+ kv *k8sutil.ConfigMapKVStore
+ pvcBacked bool
+ replaceOSD *oposd.OSDInfo
+ wipeDevicesFromOtherClusters bool
}
// NewAgent is the instantiation of the OSD agent
func NewAgent(context *clusterd.Context, devices []DesiredDevice, metadataDevice string, forceFormat bool,
storeConfig config.StoreConfig, clusterInfo *cephclient.ClusterInfo, nodeName string, kv *k8sutil.ConfigMapKVStore,
- replaceOSD *oposd.OSDInfo, pvcBacked bool,
+ replaceOSD *oposd.OSDInfo, pvcBacked, wipDevicesFromOtherClusters bool,
) *OsdAgent {
return &OsdAgent{
- devices: devices,
- metadataDevice: metadataDevice,
- forceFormat: forceFormat,
- storeConfig: storeConfig,
- clusterInfo: clusterInfo,
- nodeName: nodeName,
- kv: kv,
- pvcBacked: pvcBacked,
- replaceOSD: replaceOSD,
+ devices: devices,
+ metadataDevice: metadataDevice,
+ forceFormat: forceFormat,
+ storeConfig: storeConfig,
+ clusterInfo: clusterInfo,
+ nodeName: nodeName,
+ kv: kv,
+ pvcBacked: pvcBacked,
+ replaceOSD: replaceOSD,
+ wipeDevicesFromOtherClusters: wipDevicesFromOtherClusters,
}
}
diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go
index 98b507775..6ee9e92e9 100644
--- a/pkg/daemon/ceph/osd/daemon.go
+++ b/pkg/daemon/ceph/osd/daemon.go
@@ -201,6 +201,15 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo
context.Devices = rawDevices
+ // Wipe the desired OSD disks in case they belong to a different ceph cluster.
+ if agent.wipeDevicesFromOtherClusters {
+ logger.Info("checking for OSD disks from a different cluster")
+ err := agent.WipeDevicesFromOtherClusters(context)
+ if err != nil {
+ return errors.Wrapf(err, "failed to wipe devices from other clusters")
+ }
+ }
+
logger.Info("creating and starting the osds")
// determine the set of devices that can/should be used for OSDs.
diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go
index f102671c5..f747e5533 100644
--- a/pkg/daemon/ceph/osd/volume.go
+++ b/pkg/daemon/ceph/osd/volume.go
@@ -853,6 +853,62 @@ func (a *OsdAgent) appendDeviceClassArg(device *DeviceOsdIDEntry, args []string)
return args
}
+// WipeDevicesFromOtherClusters wipes the OSD backed disks if they have metadata from a different ceph cluster.
+// The wiped disks can then be used to prepare OSDs for the current ceph cluster.
+func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error {
+ args := []string{"raw", "list", "--format", "json"}
+
+ result, err := callCephVolume(context, args...)
+ if err != nil {
+ return errors.Wrapf(err, "failed to retrieve ceph-volume raw list results")
+ }
+
+ var existingOSDs map[string]osdInfoBlock
+ err = json.Unmarshal([]byte(result), &existingOSDs)
+ if err != nil {
+ return errors.Wrapf(err, "failed to unmarshal ceph-volume raw list results")
+ }
+
+ if len(existingOSDs) == 0 {
+ logger.Info("no existing OSDs were found. No disks to be wiped")
+ return nil
+ }
+
+ for _, existingOSD := range existingOSDs {
+ // Wipe the devices that will be used for preparing OSD but already have OSD metadata from another ceph cluster
+ if existingOSD.CephFsid != a.clusterInfo.FSID {
+ device := desiredDeviceFromOtherCluster(context.Devices, existingOSD.Device)
+ if device != nil {
+ logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", existingOSD.OsdID, existingOSD.Device, existingOSD.CephFsid)
+ logger.Infof("zap OSD.%d on device path %q", existingOSD.OsdID, existingOSD.Device)
+ output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", existingOSD.Device)
+ if err != nil {
+ return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", existingOSD.OsdID, existingOSD.Device, output)
+ }
+ logger.Infof("ceph-volume output: %s", output)
+ logger.Infof("successfully zapped osd.%d path %q", existingOSD.OsdID, existingOSD.Device)
+ logger.Infof("completed wiping OSD %d device belonging to a different ceph cluster", existingOSD.OsdID)
+ // Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check.
+ device.Filesystem = ""
+ } else {
+ logger.Infof("skip wiping OSD %d device belonging to a different ceph cluster %q since not a desired device", existingOSD.OsdID, existingOSD.CephFsid)
+ }
+ }
+ }
+
+ return nil
+}
+
+// desiredDeviceFromOtherCluster returns that desired device that needs to be wiped as it belongs to another cluster
+func desiredDeviceFromOtherCluster(desiredDevices []*sys.LocalDisk, deviceFromOtherCluster string) *sys.LocalDisk {
+ for _, desiredDevice := range desiredDevices {
+ if desiredDevice.RealPath == deviceFromOtherCluster {
+ return desiredDevice
+ }
+ }
+ return nil
+}
+
func lvmPreReq(context *clusterd.Context) error {
// Check for the presence of LVM on the host when NOT running on PVC
// since this scenario is still using LVM
diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go
index 47edda9e7..c6a56820c 100644
--- a/pkg/daemon/ceph/osd/volume_test.go
+++ b/pkg/daemon/ceph/osd/volume_test.go
@@ -2118,3 +2118,40 @@ func TestLVMModeAllowed(t *testing.T) {
storeConfig.EncryptedDevice = true
assert.False(t, lvmModeAllowed(device, storeConfig))
}
+
+func TestWipeDevicesFromOtherClusters(t *testing.T) {
+ agent := &OsdAgent{
+ clusterInfo: &cephclient.ClusterInfo{
+ FSID: "c03d7353-96e5-4a41-98de-830dfff97d06",
+ },
+ }
+
+ executor := &exectest.MockExecutor{}
+ executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+ logger.Infof("%s %v", command, args)
+ if contains(args, "raw") && contains(args, "list") {
+ return cephVolumeRAWTestResult, nil
+ }
+ return "", errors.Errorf("unknown command %s %s", command, args)
+ }
+
+ // "/dev/vdc" should not be zapped
+ executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
+ devicePath := "/dev/vdc"
+ logger.Infof("%s %v", command, args)
+
+ if contains(args, "zap") {
+ if contains(args, devicePath) {
+ return "", errors.Errorf("device %s should not be zapped", devicePath)
+ }
+ return "", nil
+ }
+ return "", errors.Errorf("unknown command %s %s", command, args)
+ }
+ context := &clusterd.Context{
+ Devices: []*sys.LocalDisk{{RealPath: "/dev/vdb"}},
+ }
+ context.Executor = executor
+ err := agent.WipeDevicesFromOtherClusters(context)
+ assert.NoError(t, err)
+}
diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go
index c82edb060..74fe9bab6 100644
--- a/pkg/operator/ceph/cluster/osd/envs.go
+++ b/pkg/operator/ceph/cluster/osd/envs.go
@@ -51,6 +51,7 @@ const (
ReplaceOSDIDVarName = "ROOK_REPLACE_OSD"
CrushRootVarName = "ROOK_CRUSHMAP_ROOT"
tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES"
+ wipeDevicesFromOtherClustersVarName = "ROOK_WIPE_DEVICES_FROM_OTHER_CLUSTERS"
)
var cephEnvConfigFile = "/etc/sysconfig/ceph"
@@ -145,6 +146,10 @@ func pvcBackedOSDEnvVar(pvcBacked string) v1.EnvVar {
return v1.EnvVar{Name: PVCBackedOSDVarName, Value: pvcBacked}
}
+func wipeDevicesFromOtherClustersEnvVar() v1.EnvVar {
+ return v1.EnvVar{Name: wipeDevicesFromOtherClustersVarName, Value: "true"}
+}
+
func setDebugLogLevelEnvVar(debug bool) v1.EnvVar {
level := "INFO"
if debug {
diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go
index 4b15bc8c1..9de4b474d 100644
--- a/pkg/operator/ceph/cluster/osd/provision_spec.go
+++ b/pkg/operator/ceph/cluster/osd/provision_spec.go
@@ -205,6 +205,10 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
// enable debug logging in the prepare job
envVars = append(envVars, setDebugLogLevelEnvVar(true))
+ if c.spec.CleanupPolicy.WipeDevicesFromOtherClusters {
+ envVars = append(envVars, wipeDevicesFromOtherClustersEnvVar())
+ }
+
// only 1 of device list, device filter, device path filter and use all devices can be specified. We prioritize in that order.
if len(osdProps.devices) > 0 {
configuredDevices := []config.ConfiguredDevice{}