Merge pull request #15968 from rook/mergify/bp/release-1.17/pr-15796

osd: clean osd disks before reinstalling cluster (backport #15796)

Documentation/CRDs/specification.md

@@ -4500,6 +4500,19 @@ bool
 <p>AllowUninstallWithVolumes defines whether we can proceed with the uninstall if they are RBD images still present</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>wipeDevicesFromOtherClusters</code><br/>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster
+was reinstalled but OSD disk still contains the metadata from previous ceph cluster.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.ClientSpec">ClientSpec

cmd/rook/ceph/osd.go

@@ -66,22 +66,23 @@ var osdRemoveCmd = &cobra.Command{
 }
 
 var (
-	osdDataDeviceFilter     string
-	osdDataDevicePathFilter string
-	ownerRefID              string
-	clusterName             string
-	osdID                   int
-	replaceOSDID            int
-	osdStoreType            string
-	osdStringID             string
-	osdUUID                 string
-	osdIsDevice             bool
-	pvcBackedOSD            bool
-	blockPath               string
-	lvBackedPV              bool
-	osdIDsToRemove          string
-	preservePVC             string
-	forceOSDRemoval         string
+	osdDataDeviceFilter          string
+	osdDataDevicePathFilter      string
+	ownerRefID                   string
+	clusterName                  string
+	osdID                        int
+	replaceOSDID                 int
+	osdStoreType                 string
+	osdStringID                  string
+	osdUUID                      string
+	osdIsDevice                  bool
+	pvcBackedOSD                 bool
+	blockPath                    string
+	lvBackedPV                   bool
+	osdIDsToRemove               string
+	preservePVC                  string
+	forceOSDRemoval              string
+	wipeDevicesFromOtherClusters bool
 )
 
 const (
@@ -102,6 +103,7 @@ func addOSDFlags(command *cobra.Command) {
 	provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
 		"true to force the format of any specified devices, even if they already have a filesystem.  BE CAREFUL!")
 	provisionCmd.Flags().BoolVar(&cfg.pvcBacked, "pvc-backed-osd", false, "true to specify a block mode pvc is backing the OSD")
+	provisionCmd.Flags().BoolVar(&wipeDevicesFromOtherClusters, "wipe-devices-from-other-clusters", false, "wipe the OSD devices that are configured for a different ceph cluster")
 	// flags for generating the osd config
 	osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config")
 	osdConfigCmd.Flags().BoolVar(&osdIsDevice, "is-device", false, "whether the osd is a device")
@@ -274,7 +276,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
 	}
 
 	agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat,
-		cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, replaceOSD, cfg.pvcBacked)
+		cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, replaceOSD, cfg.pvcBacked, wipeDevicesFromOtherClusters)
 
 	if cfg.metadataDevice != "" {
 		metaDevice = cfg.metadataDevice

deploy/charts/rook-ceph/templates/resources.yaml

@@ -1498,6 +1498,11 @@ spec:
                             - quick
                           type: string
                       type: object
+                    wipeDevicesFromOtherClusters:
+                      description: |-
+                        WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster
+                        was reinstalled but OSD disk still contains the metadata from previous ceph cluster.
+                      type: boolean
                   type: object
                 continueUpgradeAfterChecksEvenIfNotHealthy:
                   description: ContinueUpgradeAfterChecksEvenIfNotHealthy defines if an upgrade should continue even if PGs are not clean

deploy/examples/crds.yaml

@@ -1496,6 +1496,11 @@ spec:
                             - quick
                           type: string
                       type: object
+                    wipeDevicesFromOtherClusters:
+                      description: |-
+                        WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster
+                        was reinstalled but OSD disk still contains the metadata from previous ceph cluster.
+                      type: boolean
                   type: object
                 continueUpgradeAfterChecksEvenIfNotHealthy:
                   description: ContinueUpgradeAfterChecksEvenIfNotHealthy defines if an upgrade should continue even if PGs are not clean

pkg/apis/ceph.rook.io/v1/types.go

@@ -3030,6 +3030,11 @@ type CleanupPolicySpec struct {
 	// AllowUninstallWithVolumes defines whether we can proceed with the uninstall if they are RBD images still present
 	// +optional
 	AllowUninstallWithVolumes bool `json:"allowUninstallWithVolumes,omitempty"`
+
+	// WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster
+	// was reinstalled but OSD disk still contains the metadata from previous ceph cluster.
+	// +optional
+	WipeDevicesFromOtherClusters bool `json:"wipeDevicesFromOtherClusters"`
 }
 
 // CleanupConfirmationProperty represents the cleanup confirmation

pkg/daemon/ceph/osd/agent.go

@@ -30,32 +30,34 @@ const (
 
 // OsdAgent represents the OSD struct of an agent
 type OsdAgent struct {
-	clusterInfo    *cephclient.ClusterInfo
-	nodeName       string
-	forceFormat    bool
-	devices        []DesiredDevice
-	metadataDevice string
-	storeConfig    config.StoreConfig
-	kv             *k8sutil.ConfigMapKVStore
-	pvcBacked      bool
-	replaceOSD     *oposd.OSDInfo
+	clusterInfo                  *cephclient.ClusterInfo
+	nodeName                     string
+	forceFormat                  bool
+	devices                      []DesiredDevice
+	metadataDevice               string
+	storeConfig                  config.StoreConfig
+	kv                           *k8sutil.ConfigMapKVStore
+	pvcBacked                    bool
+	replaceOSD                   *oposd.OSDInfo
+	wipeDevicesFromOtherClusters bool
 }
 
 // NewAgent is the instantiation of the OSD agent
 func NewAgent(context *clusterd.Context, devices []DesiredDevice, metadataDevice string, forceFormat bool,
 	storeConfig config.StoreConfig, clusterInfo *cephclient.ClusterInfo, nodeName string, kv *k8sutil.ConfigMapKVStore,
-	replaceOSD *oposd.OSDInfo, pvcBacked bool,
+	replaceOSD *oposd.OSDInfo, pvcBacked, wipDevicesFromOtherClusters bool,
 ) *OsdAgent {
 	return &OsdAgent{
-		devices:        devices,
-		metadataDevice: metadataDevice,
-		forceFormat:    forceFormat,
-		storeConfig:    storeConfig,
-		clusterInfo:    clusterInfo,
-		nodeName:       nodeName,
-		kv:             kv,
-		pvcBacked:      pvcBacked,
-		replaceOSD:     replaceOSD,
+		devices:                      devices,
+		metadataDevice:               metadataDevice,
+		forceFormat:                  forceFormat,
+		storeConfig:                  storeConfig,
+		clusterInfo:                  clusterInfo,
+		nodeName:                     nodeName,
+		kv:                           kv,
+		pvcBacked:                    pvcBacked,
+		replaceOSD:                   replaceOSD,
+		wipeDevicesFromOtherClusters: wipDevicesFromOtherClusters,
 	}
 }
 

pkg/daemon/ceph/osd/daemon.go

@@ -201,6 +201,15 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo
 
 	context.Devices = rawDevices
 
+	// Wipe the desired OSD disks in case they belong to a different ceph cluster.
+	if agent.wipeDevicesFromOtherClusters {
+		logger.Info("checking for OSD disks from a different cluster")
+		err := agent.WipeDevicesFromOtherClusters(context)
+		if err != nil {
+			return errors.Wrapf(err, "failed to wipe devices from other clusters")
+		}
+	}
+
 	logger.Info("creating and starting the osds")
 
 	// determine the set of devices that can/should be used for OSDs.

pkg/daemon/ceph/osd/volume.go

@@ -853,6 +853,62 @@ func (a *OsdAgent) appendDeviceClassArg(device *DeviceOsdIDEntry, args []string)
 	return args
 }
 
+// WipeDevicesFromOtherClusters wipes the OSD backed disks if they have metadata from a different ceph cluster.
+// The wiped disks can then be used to prepare OSDs for the current ceph cluster.
+func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error {
+	args := []string{"raw", "list", "--format", "json"}
+
+	result, err := callCephVolume(context, args...)
+	if err != nil {
+		return errors.Wrapf(err, "failed to retrieve ceph-volume raw list results")
+	}
+
+	var existingOSDs map[string]osdInfoBlock
+	err = json.Unmarshal([]byte(result), &existingOSDs)
+	if err != nil {
+		return errors.Wrapf(err, "failed to unmarshal ceph-volume raw list results")
+	}
+
+	if len(existingOSDs) == 0 {
+		logger.Info("no existing OSDs were found. No disks to be wiped")
+		return nil
+	}
+
+	for _, existingOSD := range existingOSDs {
+		// Wipe the devices that will be used for preparing OSD but already have OSD metadata from another ceph cluster
+		if existingOSD.CephFsid != a.clusterInfo.FSID {
+			device := desiredDeviceFromOtherCluster(context.Devices, existingOSD.Device)
+			if device != nil {
+				logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", existingOSD.OsdID, existingOSD.Device, existingOSD.CephFsid)
+				logger.Infof("zap OSD.%d on device path %q", existingOSD.OsdID, existingOSD.Device)
+				output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", existingOSD.Device)
+				if err != nil {
+					return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", existingOSD.OsdID, existingOSD.Device, output)
+				}
+				logger.Infof("ceph-volume output: %s", output)
+				logger.Infof("successfully zapped osd.%d path %q", existingOSD.OsdID, existingOSD.Device)
+				logger.Infof("completed wiping OSD %d device belonging to a different ceph cluster", existingOSD.OsdID)
+				// Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check.
+				device.Filesystem = ""
+			} else {
+				logger.Infof("skip wiping OSD %d device belonging to a different ceph cluster %q since not a desired device", existingOSD.OsdID, existingOSD.CephFsid)
+			}
+		}
+	}
+
+	return nil
+}
+
+// desiredDeviceFromOtherCluster returns that desired device that needs to be wiped as it belongs to another cluster
+func desiredDeviceFromOtherCluster(desiredDevices []*sys.LocalDisk, deviceFromOtherCluster string) *sys.LocalDisk {
+	for _, desiredDevice := range desiredDevices {
+		if desiredDevice.RealPath == deviceFromOtherCluster {
+			return desiredDevice
+		}
+	}
+	return nil
+}
+
 func lvmPreReq(context *clusterd.Context) error {
 	// Check for the presence of LVM on the host when NOT running on PVC
 	// since this scenario is still using LVM

pkg/daemon/ceph/osd/volume_test.go

@@ -2118,3 +2118,40 @@ func TestLVMModeAllowed(t *testing.T) {
 	storeConfig.EncryptedDevice = true
 	assert.False(t, lvmModeAllowed(device, storeConfig))
 }
+
+func TestWipeDevicesFromOtherClusters(t *testing.T) {
+	agent := &OsdAgent{
+		clusterInfo: &cephclient.ClusterInfo{
+			FSID: "c03d7353-96e5-4a41-98de-830dfff97d06",
+		},
+	}
+
+	executor := &exectest.MockExecutor{}
+	executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+		logger.Infof("%s %v", command, args)
+		if contains(args, "raw") && contains(args, "list") {
+			return cephVolumeRAWTestResult, nil
+		}
+		return "", errors.Errorf("unknown command %s %s", command, args)
+	}
+
+	// "/dev/vdc" should not be zapped
+	executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
+		devicePath := "/dev/vdc"
+		logger.Infof("%s %v", command, args)
+
+		if contains(args, "zap") {
+			if contains(args, devicePath) {
+				return "", errors.Errorf("device %s should not be zapped", devicePath)
+			}
+			return "", nil
+		}
+		return "", errors.Errorf("unknown command %s %s", command, args)
+	}
+	context := &clusterd.Context{
+		Devices: []*sys.LocalDisk{{RealPath: "/dev/vdb"}},
+	}
+	context.Executor = executor
+	err := agent.WipeDevicesFromOtherClusters(context)
+	assert.NoError(t, err)
+}

pkg/operator/ceph/cluster/osd/envs.go

@@ -51,6 +51,7 @@ const (
 	ReplaceOSDIDVarName                 = "ROOK_REPLACE_OSD"
 	CrushRootVarName                    = "ROOK_CRUSHMAP_ROOT"
 	tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES"
+	wipeDevicesFromOtherClustersVarName = "ROOK_WIPE_DEVICES_FROM_OTHER_CLUSTERS"
 )
 
 var cephEnvConfigFile = "/etc/sysconfig/ceph"
@@ -145,6 +146,10 @@ func pvcBackedOSDEnvVar(pvcBacked string) v1.EnvVar {
 	return v1.EnvVar{Name: PVCBackedOSDVarName, Value: pvcBacked}
 }
 
+func wipeDevicesFromOtherClustersEnvVar() v1.EnvVar {
+	return v1.EnvVar{Name: wipeDevicesFromOtherClustersVarName, Value: "true"}
+}
+
 func setDebugLogLevelEnvVar(debug bool) v1.EnvVar {
 	level := "INFO"
 	if debug {

pkg/operator/ceph/cluster/osd/provision_spec.go

@@ -205,6 +205,10 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
 	// enable debug logging in the prepare job
 	envVars = append(envVars, setDebugLogLevelEnvVar(true))
 
+	if c.spec.CleanupPolicy.WipeDevicesFromOtherClusters {
+		envVars = append(envVars, wipeDevicesFromOtherClustersEnvVar())
+	}
+
 	// only 1 of device list, device filter, device path filter and use all devices can be specified.  We prioritize in that order.
 	if len(osdProps.devices) > 0 {
 		configuredDevices := []config.ConfiguredDevice{}