Merge pull request #12135 from rook/mergify/bp/release-1.11/pr-11976

osd: allow scanning devices with filter (backport #11976)

.github/workflows/rgw-multisite-test/action.yml

@@ -86,8 +86,8 @@ runs:
         ns_name_secondary_object_zone='"rook-ceph-secondary/zone-b"' # double quotes intended
         tests/scripts/github-action-helper.sh restart_operator
         not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore"
-        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary_object_zone}" 120
-        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary_object_zone}" 120
+        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary_object_zone}" 600
+        tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary_object_zone}" 600
 
     - name: collect common logs
       if: always()

cmd/rook/ceph/osd.go

@@ -202,7 +202,12 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
 		rook.TerminateFatal(err)
 	}
 
-	var dataDevices []osddaemon.DesiredDevice
+	var (
+		dataDevices  []osddaemon.DesiredDevice
+		deviceFilter string
+		metaDevice   string
+	)
+
 	if osdDataDeviceFilter != "" {
 		if cfg.devices != "" || osdDataDevicePathFilter != "" {
 			return errors.New("only one of --data-devices, --data-device-filter and --data-device-path-filter can be specified")
@@ -211,6 +216,8 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
 		dataDevices = []osddaemon.DesiredDevice{
 			{Name: osdDataDeviceFilter, IsFilter: true, OSDsPerDevice: cfg.storeConfig.OSDsPerDevice},
 		}
+
+		deviceFilter = osdDataDeviceFilter
 	} else if osdDataDevicePathFilter != "" {
 		if cfg.devices != "" {
 			return errors.New("only one of --data-devices, --data-device-filter and --data-device-path-filter can be specified")
@@ -245,7 +252,11 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
 	agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat,
 		cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, cfg.pvcBacked)
 
-	err = osddaemon.Provision(context, agent, crushLocation, topologyAffinity)
+	if cfg.metadataDevice != "" {
+		metaDevice = cfg.metadataDevice
+	}
+
+	err = osddaemon.Provision(context, agent, crushLocation, topologyAffinity, deviceFilter, metaDevice)
 	if err != nil {
 		// something failed in the OSD orchestration, update the status map with failure details
 		status := oposd.OrchestrationStatus{

pkg/clusterd/disk.go

@@ -31,8 +31,12 @@ import (
 )
 
 var (
-	logger = capnslog.NewPackageLogger("github.com/rook/rook", "inventory")
-	isRBD  = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$")
+	logger         = capnslog.NewPackageLogger("github.com/rook/rook", "inventory")
+	isRBD          = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$")
+	listAllDevices = "all"
+
+	// We need to allow dm- devices (Kubernetes mountpoints) because dm- devices name are used as names for meta device
+	allowDeviceNamePattern = regexp.MustCompile("dm-")
 )
 
 func supportedDeviceType(device string) bool {
@@ -55,8 +59,7 @@ func ignoreDevice(d string) bool {
 	return isRBD.MatchString(d)
 }
 
-// DiscoverDevices returns all the details of devices available on the local node
-func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) {
+func DiscoverDevicesWithFilter(executor exec.Executor, deviceFilter, metaDevice string) ([]*sys.LocalDisk, error) {
 	var disks []*sys.LocalDisk
 	devices, err := sys.ListDevices(executor)
 	if err != nil {
@@ -71,6 +74,11 @@ func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) {
 			continue
 		}
 
+		if deviceFilter != "" && !deviceMatchWithFilter(d, deviceFilter, metaDevice) {
+			logger.Warningf("device %q skipped due to filter %q", d, deviceFilter)
+			continue
+		}
+
 		// Populate device information coming from lsblk
 		disk, err := PopulateDeviceInfo(d, executor)
 		if err != nil {
@@ -113,6 +121,35 @@ func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) {
 	return disks, nil
 }
 
+func deviceMatchWithFilter(device string, filter, metaDevice string) bool {
+	if filter == listAllDevices || device == metaDevice {
+		return true
+	}
+
+	if allowDeviceNamePattern.MatchString(device) {
+		return true
+	}
+
+	pattern, err := regexp.Compile(filter)
+	if err != nil {
+		return false
+	}
+
+	if !pattern.MatchString(device) {
+		return false
+	}
+	return true
+}
+
+// DiscoverDevices returns all the details of devices available on the local node
+func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) {
+	disks, err := DiscoverDevicesWithFilter(executor, "", "")
+	if err != nil {
+		return nil, err
+	}
+	return disks, nil
+}
+
 // PopulateDeviceInfo returns the information of the specified block device
 func PopulateDeviceInfo(d string, executor exec.Executor) (*sys.LocalDisk, error) {
 	diskProps, err := sys.GetDeviceProperties(d, executor)

pkg/clusterd/disk_test.go

@@ -38,6 +38,21 @@ func TestDiscoverDevices(t *testing.T) {
 	assert.Equal(t, 0, len(devices))
 }
 
+func TestDeviceMatchWithFilter(t *testing.T) {
+	result := deviceMatchWithFilter("nvme0n1p1", "l[o]+p", "")
+	assert.False(t, result)
+	result = deviceMatchWithFilter("nvme0n1p1", "nvme[0-1np]+", "")
+	assert.True(t, result)
+	result = deviceMatchWithFilter("nvme0n1p1", "all", "")
+	assert.True(t, result)
+	result = deviceMatchWithFilter("/dev/test-rook-vg/test-rook-lv", "nvme[0-1np]+", "/dev/test-rook-vg/test-rook-lv")
+	assert.True(t, result)
+	result = deviceMatchWithFilter("dm-0", "nvme[0-1np]+", "/dev/test-rook-vg/test-rook-lv")
+	assert.True(t, result)
+	result = deviceMatchWithFilter("dm-1", "nvme[0-1np]+", "/dev/test-rook-vg/test-rook-lv")
+	assert.True(t, result)
+}
+
 func TestIgnoreDevice(t *testing.T) {
 	cases := map[string]bool{
 		"rbd0":    true,

pkg/daemon/ceph/osd/daemon.go

@@ -161,7 +161,7 @@ func configRawDevice(name string, context *clusterd.Context) (*sys.LocalDisk, er
 }
 
 // Provision provisions an OSD
-func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity string) error {
+func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity, deviceFilter, metaDevice string) error {
 	if agent.pvcBacked && os.Getenv(oposd.EncryptedDeviceEnvVarName) == "true" {
 		logger.Debug("encryption configuration detecting, populating kek to an env variable")
 		// Init KMS store, retrieve the KEK and store it as an env var for ceph-volume
@@ -202,7 +202,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo
 		// Ideally, we would use the "ceph-volume inventory" command instead
 		// However, it suffers from some limitation such as exposing available partitions and LVs
 		// See: https://tracker.ceph.com/issues/43579
-		rawDevices, err = clusterd.DiscoverDevices(context.Executor)
+		rawDevices, err = clusterd.DiscoverDevicesWithFilter(context.Executor, deviceFilter, metaDevice)
 		if err != nil {
 			return errors.Wrap(err, "failed initial hardware discovery")
 		}