diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 6b8f61450..dbfa8bb69 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1271,7 +1271,7 @@ jobs: EOF clusterfsid=$($exec_fs_mirror ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror status myfs@1 |jq -r '.peers|keys[]') echo $clusterfsid - kubectl -n rook-ceph-secondary wait pod -l app=rook-direct-mount --for condition=Ready --timeout=300s + kubectl -n rook-ceph-secondary wait pod -l app=rook-direct-mount --for condition=Ready --timeout=400s kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- mkdir /tmp/registry mon_endpoints=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- grep mon_host /etc/ceph/ceph.conf | awk '{print $3}') my_secret=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- grep key /etc/ceph/keyring | awk '{print $3}') diff --git a/.github/workflows/rgw-multisite-test/action.yml b/.github/workflows/rgw-multisite-test/action.yml index 3ace1cd7b..2de4db5f7 100644 --- a/.github/workflows/rgw-multisite-test/action.yml +++ b/.github/workflows/rgw-multisite-test/action.yml @@ -86,8 +86,8 @@ runs: ns_name_secondary_object_zone='"rook-ceph-secondary/zone-b"' # double quotes intended tests/scripts/github-action-helper.sh restart_operator not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore" - tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary_object_zone}" 120 - tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary_object_zone}" 120 + tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary_object_zone}" 600 + tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary_object_zone}" 600 - name: collect common logs if: always() diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 140f7a0cf..840b6f68c 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -202,7 +202,12 @@ func prepareOSD(cmd *cobra.Command, args []string) error { rook.TerminateFatal(err) } - var dataDevices []osddaemon.DesiredDevice + var ( + dataDevices []osddaemon.DesiredDevice + deviceFilter string + metaDevice string + ) + if osdDataDeviceFilter != "" { if cfg.devices != "" || osdDataDevicePathFilter != "" { return errors.New("only one of --data-devices, --data-device-filter and --data-device-path-filter can be specified") @@ -211,6 +216,8 @@ func prepareOSD(cmd *cobra.Command, args []string) error { dataDevices = []osddaemon.DesiredDevice{ {Name: osdDataDeviceFilter, IsFilter: true, OSDsPerDevice: cfg.storeConfig.OSDsPerDevice}, } + + deviceFilter = osdDataDeviceFilter } else if osdDataDevicePathFilter != "" { if cfg.devices != "" { return errors.New("only one of --data-devices, --data-device-filter and --data-device-path-filter can be specified") @@ -245,7 +252,11 @@ func prepareOSD(cmd *cobra.Command, args []string) error { agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat, cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, cfg.pvcBacked) - err = osddaemon.Provision(context, agent, crushLocation, topologyAffinity) + if cfg.metadataDevice != "" { + metaDevice = cfg.metadataDevice + } + + err = osddaemon.Provision(context, agent, crushLocation, topologyAffinity, deviceFilter, metaDevice) if err != nil { // something failed in the OSD orchestration, update the status map with failure details status := oposd.OrchestrationStatus{ diff --git a/pkg/clusterd/disk.go b/pkg/clusterd/disk.go index d45d25a4d..2fb504457 100644 --- a/pkg/clusterd/disk.go +++ b/pkg/clusterd/disk.go @@ -31,8 +31,12 @@ import ( ) var ( - logger = capnslog.NewPackageLogger("github.com/rook/rook", "inventory") - isRBD = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$") + logger = capnslog.NewPackageLogger("github.com/rook/rook", "inventory") + isRBD = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$") + listAllDevices = "all" + + // We need to allow dm- devices (Kubernetes mountpoints) because dm- devices name are used as names for meta device + allowDeviceNamePattern = regexp.MustCompile("dm-") ) func supportedDeviceType(device string) bool { @@ -55,8 +59,7 @@ func ignoreDevice(d string) bool { return isRBD.MatchString(d) } -// DiscoverDevices returns all the details of devices available on the local node -func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) { +func DiscoverDevicesWithFilter(executor exec.Executor, deviceFilter, metaDevice string) ([]*sys.LocalDisk, error) { var disks []*sys.LocalDisk devices, err := sys.ListDevices(executor) if err != nil { @@ -71,6 +74,11 @@ func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) { continue } + if deviceFilter != "" && !deviceMatchWithFilter(d, deviceFilter, metaDevice) { + logger.Warningf("device %q skipped due to filter %q", d, deviceFilter) + continue + } + // Populate device information coming from lsblk disk, err := PopulateDeviceInfo(d, executor) if err != nil { @@ -113,6 +121,35 @@ func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) { return disks, nil } +func deviceMatchWithFilter(device string, filter, metaDevice string) bool { + if filter == listAllDevices || device == metaDevice { + return true + } + + if allowDeviceNamePattern.MatchString(device) { + return true + } + + pattern, err := regexp.Compile(filter) + if err != nil { + return false + } + + if !pattern.MatchString(device) { + return false + } + return true +} + +// DiscoverDevices returns all the details of devices available on the local node +func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) { + disks, err := DiscoverDevicesWithFilter(executor, "", "") + if err != nil { + return nil, err + } + return disks, nil +} + // PopulateDeviceInfo returns the information of the specified block device func PopulateDeviceInfo(d string, executor exec.Executor) (*sys.LocalDisk, error) { diskProps, err := sys.GetDeviceProperties(d, executor) diff --git a/pkg/clusterd/disk_test.go b/pkg/clusterd/disk_test.go index c1ec2e404..c2ce6a0b8 100644 --- a/pkg/clusterd/disk_test.go +++ b/pkg/clusterd/disk_test.go @@ -38,6 +38,21 @@ func TestDiscoverDevices(t *testing.T) { assert.Equal(t, 0, len(devices)) } +func TestDeviceMatchWithFilter(t *testing.T) { + result := deviceMatchWithFilter("nvme0n1p1", "l[o]+p", "") + assert.False(t, result) + result = deviceMatchWithFilter("nvme0n1p1", "nvme[0-1np]+", "") + assert.True(t, result) + result = deviceMatchWithFilter("nvme0n1p1", "all", "") + assert.True(t, result) + result = deviceMatchWithFilter("/dev/test-rook-vg/test-rook-lv", "nvme[0-1np]+", "/dev/test-rook-vg/test-rook-lv") + assert.True(t, result) + result = deviceMatchWithFilter("dm-0", "nvme[0-1np]+", "/dev/test-rook-vg/test-rook-lv") + assert.True(t, result) + result = deviceMatchWithFilter("dm-1", "nvme[0-1np]+", "/dev/test-rook-vg/test-rook-lv") + assert.True(t, result) +} + func TestIgnoreDevice(t *testing.T) { cases := map[string]bool{ "rbd0": true, diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 99a85b744..b98c80819 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -161,7 +161,7 @@ func configRawDevice(name string, context *clusterd.Context) (*sys.LocalDisk, er } // Provision provisions an OSD -func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity string) error { +func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity, deviceFilter, metaDevice string) error { if agent.pvcBacked && os.Getenv(oposd.EncryptedDeviceEnvVarName) == "true" { logger.Debug("encryption configuration detecting, populating kek to an env variable") // Init KMS store, retrieve the KEK and store it as an env var for ceph-volume @@ -202,7 +202,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo // Ideally, we would use the "ceph-volume inventory" command instead // However, it suffers from some limitation such as exposing available partitions and LVs // See: https://tracker.ceph.com/issues/43579 - rawDevices, err = clusterd.DiscoverDevices(context.Executor) + rawDevices, err = clusterd.DiscoverDevicesWithFilter(context.Executor, deviceFilter, metaDevice) if err != nil { return errors.Wrap(err, "failed initial hardware discovery") }