From 1880856e8e601fcc221719ea43a54cc8cb71e4c2 Mon Sep 17 00:00:00 2001 From: Santosh Pillai Date: Tue, 29 Apr 2025 10:17:18 +0530 Subject: [PATCH] osd: clean osd disks before reinstalling cluster Allow force creation of OSD on a device previously configured in a different ceph cluster. The OSD prepare pod job will zap the OSD disk using ceph-volume lvm zap if that disk has metadata from a different ceph cluster. Signed-off-by: Santosh Pillai (cherry picked from commit 2bcf196b153698b455a0586eb83585e458ba6039) --- Documentation/CRDs/specification.md | 13 +++++ cmd/rook/ceph/osd.go | 36 ++++++------ .../charts/rook-ceph/templates/resources.yaml | 5 ++ deploy/examples/crds.yaml | 5 ++ pkg/apis/ceph.rook.io/v1/types.go | 5 ++ pkg/daemon/ceph/osd/agent.go | 40 ++++++------- pkg/daemon/ceph/osd/daemon.go | 9 +++ pkg/daemon/ceph/osd/volume.go | 56 +++++++++++++++++++ pkg/daemon/ceph/osd/volume_test.go | 37 ++++++++++++ pkg/operator/ceph/cluster/osd/envs.go | 5 ++ .../ceph/cluster/osd/provision_spec.go | 4 ++ 11 files changed, 179 insertions(+), 36 deletions(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 0597ca9b1..bb9077c73 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -4500,6 +4500,19 @@ bool

AllowUninstallWithVolumes defines whether we can proceed with the uninstall if they are RBD images still present

+ + +wipeDevicesFromOtherClusters
+ +bool + + + +(Optional) +

WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster +was reinstalled but OSD disk still contains the metadata from previous ceph cluster.

+ +

ClientSpec diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 7505a94a8..82da36271 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -66,22 +66,23 @@ var osdRemoveCmd = &cobra.Command{ } var ( - osdDataDeviceFilter string - osdDataDevicePathFilter string - ownerRefID string - clusterName string - osdID int - replaceOSDID int - osdStoreType string - osdStringID string - osdUUID string - osdIsDevice bool - pvcBackedOSD bool - blockPath string - lvBackedPV bool - osdIDsToRemove string - preservePVC string - forceOSDRemoval string + osdDataDeviceFilter string + osdDataDevicePathFilter string + ownerRefID string + clusterName string + osdID int + replaceOSDID int + osdStoreType string + osdStringID string + osdUUID string + osdIsDevice bool + pvcBackedOSD bool + blockPath string + lvBackedPV bool + osdIDsToRemove string + preservePVC string + forceOSDRemoval string + wipeDevicesFromOtherClusters bool ) const ( @@ -102,6 +103,7 @@ func addOSDFlags(command *cobra.Command) { provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false, "true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!") provisionCmd.Flags().BoolVar(&cfg.pvcBacked, "pvc-backed-osd", false, "true to specify a block mode pvc is backing the OSD") + provisionCmd.Flags().BoolVar(&wipeDevicesFromOtherClusters, "wipe-devices-from-other-clusters", false, "wipe the OSD devices that are configured for a different ceph cluster") // flags for generating the osd config osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config") osdConfigCmd.Flags().BoolVar(&osdIsDevice, "is-device", false, "whether the osd is a device") @@ -274,7 +276,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error { } agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat, - cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, replaceOSD, cfg.pvcBacked) + cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, replaceOSD, cfg.pvcBacked, wipeDevicesFromOtherClusters) if cfg.metadataDevice != "" { metaDevice = cfg.metadataDevice diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index b1cc157c7..2bbcaa6b0 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -1498,6 +1498,11 @@ spec: - quick type: string type: object + wipeDevicesFromOtherClusters: + description: |- + WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster + was reinstalled but OSD disk still contains the metadata from previous ceph cluster. + type: boolean type: object continueUpgradeAfterChecksEvenIfNotHealthy: description: ContinueUpgradeAfterChecksEvenIfNotHealthy defines if an upgrade should continue even if PGs are not clean diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 2b4956bbf..7ce5132f7 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -1496,6 +1496,11 @@ spec: - quick type: string type: object + wipeDevicesFromOtherClusters: + description: |- + WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster + was reinstalled but OSD disk still contains the metadata from previous ceph cluster. + type: boolean type: object continueUpgradeAfterChecksEvenIfNotHealthy: description: ContinueUpgradeAfterChecksEvenIfNotHealthy defines if an upgrade should continue even if PGs are not clean diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 115f9c65b..0b2b31f5d 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -3030,6 +3030,11 @@ type CleanupPolicySpec struct { // AllowUninstallWithVolumes defines whether we can proceed with the uninstall if they are RBD images still present // +optional AllowUninstallWithVolumes bool `json:"allowUninstallWithVolumes,omitempty"` + + // WipeDevicesFromOtherClusters wipes the OSD disks belonging to other clusters. This is useful in scenarios where ceph cluster + // was reinstalled but OSD disk still contains the metadata from previous ceph cluster. + // +optional + WipeDevicesFromOtherClusters bool `json:"wipeDevicesFromOtherClusters"` } // CleanupConfirmationProperty represents the cleanup confirmation diff --git a/pkg/daemon/ceph/osd/agent.go b/pkg/daemon/ceph/osd/agent.go index 428c1fb95..ca6c7871c 100644 --- a/pkg/daemon/ceph/osd/agent.go +++ b/pkg/daemon/ceph/osd/agent.go @@ -30,32 +30,34 @@ const ( // OsdAgent represents the OSD struct of an agent type OsdAgent struct { - clusterInfo *cephclient.ClusterInfo - nodeName string - forceFormat bool - devices []DesiredDevice - metadataDevice string - storeConfig config.StoreConfig - kv *k8sutil.ConfigMapKVStore - pvcBacked bool - replaceOSD *oposd.OSDInfo + clusterInfo *cephclient.ClusterInfo + nodeName string + forceFormat bool + devices []DesiredDevice + metadataDevice string + storeConfig config.StoreConfig + kv *k8sutil.ConfigMapKVStore + pvcBacked bool + replaceOSD *oposd.OSDInfo + wipeDevicesFromOtherClusters bool } // NewAgent is the instantiation of the OSD agent func NewAgent(context *clusterd.Context, devices []DesiredDevice, metadataDevice string, forceFormat bool, storeConfig config.StoreConfig, clusterInfo *cephclient.ClusterInfo, nodeName string, kv *k8sutil.ConfigMapKVStore, - replaceOSD *oposd.OSDInfo, pvcBacked bool, + replaceOSD *oposd.OSDInfo, pvcBacked, wipDevicesFromOtherClusters bool, ) *OsdAgent { return &OsdAgent{ - devices: devices, - metadataDevice: metadataDevice, - forceFormat: forceFormat, - storeConfig: storeConfig, - clusterInfo: clusterInfo, - nodeName: nodeName, - kv: kv, - pvcBacked: pvcBacked, - replaceOSD: replaceOSD, + devices: devices, + metadataDevice: metadataDevice, + forceFormat: forceFormat, + storeConfig: storeConfig, + clusterInfo: clusterInfo, + nodeName: nodeName, + kv: kv, + pvcBacked: pvcBacked, + replaceOSD: replaceOSD, + wipeDevicesFromOtherClusters: wipDevicesFromOtherClusters, } } diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 98b507775..6ee9e92e9 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -201,6 +201,15 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo context.Devices = rawDevices + // Wipe the desired OSD disks in case they belong to a different ceph cluster. + if agent.wipeDevicesFromOtherClusters { + logger.Info("checking for OSD disks from a different cluster") + err := agent.WipeDevicesFromOtherClusters(context) + if err != nil { + return errors.Wrapf(err, "failed to wipe devices from other clusters") + } + } + logger.Info("creating and starting the osds") // determine the set of devices that can/should be used for OSDs. diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index f102671c5..f747e5533 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -853,6 +853,62 @@ func (a *OsdAgent) appendDeviceClassArg(device *DeviceOsdIDEntry, args []string) return args } +// WipeDevicesFromOtherClusters wipes the OSD backed disks if they have metadata from a different ceph cluster. +// The wiped disks can then be used to prepare OSDs for the current ceph cluster. +func (a *OsdAgent) WipeDevicesFromOtherClusters(context *clusterd.Context) error { + args := []string{"raw", "list", "--format", "json"} + + result, err := callCephVolume(context, args...) + if err != nil { + return errors.Wrapf(err, "failed to retrieve ceph-volume raw list results") + } + + var existingOSDs map[string]osdInfoBlock + err = json.Unmarshal([]byte(result), &existingOSDs) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal ceph-volume raw list results") + } + + if len(existingOSDs) == 0 { + logger.Info("no existing OSDs were found. No disks to be wiped") + return nil + } + + for _, existingOSD := range existingOSDs { + // Wipe the devices that will be used for preparing OSD but already have OSD metadata from another ceph cluster + if existingOSD.CephFsid != a.clusterInfo.FSID { + device := desiredDeviceFromOtherCluster(context.Devices, existingOSD.Device) + if device != nil { + logger.Infof("begin wiping OSD %d device %q belonging to a different ceph cluster %q ", existingOSD.OsdID, existingOSD.Device, existingOSD.CephFsid) + logger.Infof("zap OSD.%d on device path %q", existingOSD.OsdID, existingOSD.Device) + output, err := context.Executor.ExecuteCommandWithCombinedOutput("stdbuf", "-oL", cephVolumeCmd, "lvm", "zap", existingOSD.Device) + if err != nil { + return errors.Wrapf(err, "failed to zap osd.%d path %q. %s.", existingOSD.OsdID, existingOSD.Device, output) + } + logger.Infof("ceph-volume output: %s", output) + logger.Infof("successfully zapped osd.%d path %q", existingOSD.OsdID, existingOSD.Device) + logger.Infof("completed wiping OSD %d device belonging to a different ceph cluster", existingOSD.OsdID) + // Since the device is wiped clean, clear the stale filesystem reference on the disk so that the wiped disk is not filtered out for filesystem check. + device.Filesystem = "" + } else { + logger.Infof("skip wiping OSD %d device belonging to a different ceph cluster %q since not a desired device", existingOSD.OsdID, existingOSD.CephFsid) + } + } + } + + return nil +} + +// desiredDeviceFromOtherCluster returns that desired device that needs to be wiped as it belongs to another cluster +func desiredDeviceFromOtherCluster(desiredDevices []*sys.LocalDisk, deviceFromOtherCluster string) *sys.LocalDisk { + for _, desiredDevice := range desiredDevices { + if desiredDevice.RealPath == deviceFromOtherCluster { + return desiredDevice + } + } + return nil +} + func lvmPreReq(context *clusterd.Context) error { // Check for the presence of LVM on the host when NOT running on PVC // since this scenario is still using LVM diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 47edda9e7..c6a56820c 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -2118,3 +2118,40 @@ func TestLVMModeAllowed(t *testing.T) { storeConfig.EncryptedDevice = true assert.False(t, lvmModeAllowed(device, storeConfig)) } + +func TestWipeDevicesFromOtherClusters(t *testing.T) { + agent := &OsdAgent{ + clusterInfo: &cephclient.ClusterInfo{ + FSID: "c03d7353-96e5-4a41-98de-830dfff97d06", + }, + } + + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if contains(args, "raw") && contains(args, "list") { + return cephVolumeRAWTestResult, nil + } + return "", errors.Errorf("unknown command %s %s", command, args) + } + + // "/dev/vdc" should not be zapped + executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { + devicePath := "/dev/vdc" + logger.Infof("%s %v", command, args) + + if contains(args, "zap") { + if contains(args, devicePath) { + return "", errors.Errorf("device %s should not be zapped", devicePath) + } + return "", nil + } + return "", errors.Errorf("unknown command %s %s", command, args) + } + context := &clusterd.Context{ + Devices: []*sys.LocalDisk{{RealPath: "/dev/vdb"}}, + } + context.Executor = executor + err := agent.WipeDevicesFromOtherClusters(context) + assert.NoError(t, err) +} diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index c82edb060..74fe9bab6 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -51,6 +51,7 @@ const ( ReplaceOSDIDVarName = "ROOK_REPLACE_OSD" CrushRootVarName = "ROOK_CRUSHMAP_ROOT" tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES" + wipeDevicesFromOtherClustersVarName = "ROOK_WIPE_DEVICES_FROM_OTHER_CLUSTERS" ) var cephEnvConfigFile = "/etc/sysconfig/ceph" @@ -145,6 +146,10 @@ func pvcBackedOSDEnvVar(pvcBacked string) v1.EnvVar { return v1.EnvVar{Name: PVCBackedOSDVarName, Value: pvcBacked} } +func wipeDevicesFromOtherClustersEnvVar() v1.EnvVar { + return v1.EnvVar{Name: wipeDevicesFromOtherClustersVarName, Value: "true"} +} + func setDebugLogLevelEnvVar(debug bool) v1.EnvVar { level := "INFO" if debug { diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 4b15bc8c1..9de4b474d 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -205,6 +205,10 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun // enable debug logging in the prepare job envVars = append(envVars, setDebugLogLevelEnvVar(true)) + if c.spec.CleanupPolicy.WipeDevicesFromOtherClusters { + envVars = append(envVars, wipeDevicesFromOtherClustersEnvVar()) + } + // only 1 of device list, device filter, device path filter and use all devices can be specified. We prioritize in that order. if len(osdProps.devices) > 0 { configuredDevices := []config.ConfiguredDevice{}