From 28517376f8ce2940e1e252d3c27b1d016d4db655 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Mon, 22 Jun 2026 14:53:01 -0700 Subject: [PATCH] ci: replace loop devices with iscsi in canary osd tests The raw-disk-with-object, osd-with-metadata-device, and pvc canary jobs used loopback devices to supply an extra block device beyond the single spare disk a GitHub runner provides. Loop devices behave differently from real block devices -- Rook gates them behind ROOK_CEPH_ALLOW_LOOP_DEVICES and special-cases them in the OSD daemon -- so exercising core scenarios on them is less realistic than using real SCSI disks. Provision the extra device(s) over iSCSI (LIO) instead, reusing the existing create_extra_disk helper: it now creates N LUNs, the new find_extra_block_devs and find_second_block_dev helpers enumerate them, and deploy_cluster gains a two_raw_disks mode. The three jobs now run entirely on real disks, and the canary CI no longer creates loop devices or relies on the loop-device test helpers. Signed-off-by: Joshua Hoblitt (cherry picked from commit 4b7d7691022dbd572c428088ff8cf521b61c48b3) --- .github/workflows/canary-integration-test.yml | 48 ++++---- .../{loopDevicePV.sh => blockDevicePV.sh} | 21 ++-- tests/scripts/github-action-helper.sh | 111 ++++++++++++------ 3 files changed, 109 insertions(+), 71 deletions(-) rename tests/scripts/{loopDevicePV.sh => blockDevicePV.sh} (73%) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 4e8c5b92d..36fe4e020 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -488,21 +488,23 @@ jobs: - name: validate-yaml run: tests/scripts/github-action-helper.sh validate_yaml - - name: use local disk as OSD + - name: set up the first and second disks as OSDs run: | + # find_second_block_dev provisions a second iSCSI disk if the runner + # has only one tests/scripts/github-action-helper.sh use_local_disk BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" export BLOCK tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - - - name: prepare loop devices for osds - run: | - tests/scripts/github-action-helper.sh prepare_loop_devices 1 + EXTRA_DEV="$(tests/scripts/github-action-helper.sh find_second_block_dev)" + sudo sgdisk --zap-all "/dev/$EXTRA_DEV" || true - name: deploy cluster run: | - export ALLOW_LOOP_DEVICES=true - tests/scripts/github-action-helper.sh deploy_cluster loop + ROOK_DATA_DEV="$(tests/scripts/github-action-helper.sh find_extra_block_dev)" + ROOK_EXTRA_DEV="$(tests/scripts/github-action-helper.sh find_second_block_dev)" + export ROOK_DATA_DEV ROOK_EXTRA_DEV + tests/scripts/github-action-helper.sh deploy_cluster two_raw_disks tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster tests/scripts/github-action-helper.sh create_operator_toolbox @@ -696,18 +698,18 @@ jobs: - name: validate-yaml run: tests/scripts/github-action-helper.sh validate_yaml - - name: use local disk as OSD + - name: set up the first disk run: | BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" export BLOCK tests/scripts/github-action-helper.sh use_local_disk tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - - name: create LV on disk + - name: create the metadata LV on the second disk run: | - dd if=/dev/zero of=test-rook.img bs=1 count=0 seek=10G - # If we use metadata device, both data devices and metadata devices should be logical volumes or raw devices - tests/scripts/github-action-helper.sh create_LV_on_disk "$(sudo losetup --find --show test-rook.img)" + # both data and metadata devices must be logical volumes or raw devices + EXTRA_DEV="$(tests/scripts/github-action-helper.sh find_second_block_dev)" + tests/scripts/github-action-helper.sh create_LV_on_disk "/dev/$EXTRA_DEV" - name: deploy cluster run: | @@ -888,26 +890,25 @@ jobs: - name: set Ceph version in CephCluster manifest run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}" - - name: use local disk and create partitions for osds + - name: partition the first disk for OSDs run: | + # the first disk is split into two partitions (one OSD each); the + # second disk (provisioned by find_second_block_dev) is used whole as + # the 3rd OSD tests/scripts/github-action-helper.sh use_local_disk tests/scripts/github-action-helper.sh create_partitions_for_osds - - name: prepare loop devices for osds - run: | - tests/scripts/github-action-helper.sh prepare_loop_devices 1 - - name: create cluster prerequisites run: | BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" export BLOCK + EXTRA_DEV="$(tests/scripts/github-action-helper.sh find_second_block_dev)" tests/scripts/localPathPV.sh "$BLOCK" - tests/scripts/loopDevicePV.sh 1 + tests/scripts/blockDevicePV.sh "/dev/$EXTRA_DEV" tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy cluster run: | - yq write -i deploy/examples/operator.yaml "data.ROOK_CEPH_ALLOW_LOOP_DEVICES" --style=double "true" tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 3 @@ -930,16 +931,18 @@ jobs: lsblk BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" export BLOCK + EXTRA_DEV="$(tests/scripts/github-action-helper.sh find_second_block_dev)" sudo head --bytes=60 "${BLOCK}1" sudo head --bytes=60 "${BLOCK}2" - sudo head --bytes=60 /dev/loop1 + sudo head --bytes=60 "/dev/$EXTRA_DEV" - name: recreate cluster prerequisites run: | BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" export BLOCK + EXTRA_DEV="$(tests/scripts/github-action-helper.sh find_second_block_dev)" tests/scripts/localPathPV.sh "$BLOCK" - tests/scripts/loopDevicePV.sh 1 + tests/scripts/blockDevicePV.sh "/dev/$EXTRA_DEV" tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: redeploy cluster @@ -963,9 +966,10 @@ jobs: lsblk BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" export BLOCK + EXTRA_DEV="$(tests/scripts/github-action-helper.sh find_second_block_dev)" sudo head --bytes=60 "${BLOCK}1" sudo head --bytes=60 "${BLOCK}2" - sudo head --bytes=60 /dev/loop1 + sudo head --bytes=60 "/dev/$EXTRA_DEV" - name: collect common logs if: always() diff --git a/tests/scripts/loopDevicePV.sh b/tests/scripts/blockDevicePV.sh similarity index 73% rename from tests/scripts/loopDevicePV.sh rename to tests/scripts/blockDevicePV.sh index 41d8861b4..149b8b1a4 100755 --- a/tests/scripts/loopDevicePV.sh +++ b/tests/scripts/blockDevicePV.sh @@ -19,26 +19,28 @@ set -ex ############# # VARIABLES # ############# -osd_count=$1 +# The argument is a device path (/dev/...); create a single volumeMode: Block PV +# backed by that whole device, for use as an extra OSD by a PVC-based cluster. +device=$1 sudo lsblk ############# # FUNCTIONS # ############# -function add_loop_dev_pvc() { - local osd_count=$1 +# add_block_dev_pvc +# Create one volumeMode: Block PersistentVolume backed by the given device. +function add_block_dev_pvc() { + local path=$1 + local name=$2 local storage=6Gi - for osd in $(seq 1 "$osd_count"); do - path=/dev/loop${osd} - - cat </dev/null - sudo targetcli /iscsi/${target_iqn}/tpg1/acls create ${init_iqn} - sudo targetcli /iscsi/${target_iqn}/tpg1/acls/${init_iqn} create tpg_lun_or_backstore=lun0 mapped_lun=0 + # The target and its initiator ACL are shared by all disks; create them once. + # "|| true" keeps re-invocation (to append more disks) idempotent. + sudo targetcli /iscsi create ${target_iqn} 2>/dev/null || true + sudo targetcli /iscsi/${target_iqn}/tpg1/acls create ${init_iqn} 2>/dev/null || true + # Count disks that already exist so additional disks append rather than + # clobber lun0 (LIO auto-assigns LUN numbers sequentially in creation order). + local existing=0 f + for f in ~/iscsi-disk*.img; do + if [ -e "$f" ]; then + existing=$((existing + 1)) + fi + done + local j i lun + for j in $(seq 1 "$count"); do + i=$((existing + j)) + lun=$((i - 1)) + truncate -s 75G ~/iscsi-disk${i}.img + sudo targetcli /backstores/fileio create disk${i} ~/iscsi-disk${i}.img 75G + # "luns create" auto-maps the new LUN into the existing ACL as + # lun${lun} -> mapped_lun${lun}, so the explicit mapping below is a fallback + # for targetcli versions that do not auto-map; "|| true" tolerates the + # "already exists" error once it has been auto-mapped. + sudo targetcli /iscsi/${target_iqn}/tpg1/luns create /backstores/fileio/disk${i} + sudo targetcli /iscsi/${target_iqn}/tpg1/acls/${init_iqn} create tpg_lun_or_backstore=lun${lun} mapped_lun=${lun} 2>/dev/null || true + done sudo iscsiadm -m discovery -t sendtargets -p 127.0.0.1 - sudo iscsiadm -m node --login + sudo iscsiadm -m node --login 2>/dev/null || true + sudo iscsiadm -m node -R 2>/dev/null || true # rescan to pick up newly-added LUNs + sudo udevadm settle 2>/dev/null || true # wait for the new /dev/sd* to appear } +# Print the basename of the first non-boot/loop/nbd whole disk, provisioning an +# iSCSI disk if none exist. Thin wrapper over find_extra_block_devs(); kept as a +# convenience for its many callers that expect a single basename. function find_extra_block_dev() { + find_extra_block_devs 1 | head -1 +} + +# find_extra_block_devs [min_count] +# Print ALL non-boot/loop/nbd whole-disk basenames, one per line, in stable +# lsblk order, provisioning additional iSCSI disks if fewer than min_count exist. +# This is the single source of truth for disk discovery; find_extra_block_dev() +# returns the first and find_second_block_dev() the second. Debug goes to stderr +# so stdout stays clean. +function find_extra_block_devs() { + local min_count="${1:-1}" # shellcheck disable=SC2005 # redirect doesn't work with sudo, so use echo - echo "$(sudo lsblk)" >/dev/stderr # print lsblk output to stderr for debugging in case of future errors + echo "$(sudo lsblk)" >/dev/stderr # print lsblk output to stderr for debugging # relevant lsblk --pairs example: (MOUNTPOINT identifies boot partition)(PKNAME is Parent dev ID) # NAME="sda15" SIZE="106M" TYPE="part" MOUNTPOINT="/boot/efi" PKNAME="sda" # NAME="sdb" SIZE="75G" TYPE="disk" MOUNTPOINT="" PKNAME="" # NAME="sdb1" SIZE="75G" TYPE="part" MOUNTPOINT="/mnt" PKNAME="sdb" boot_dev="$(sudo lsblk --noheading --list --output MOUNTPOINT,PKNAME | grep boot | awk '{print $2}')" - echo " == find_extra_block_dev(): boot_dev='$boot_dev'" >/dev/stderr # debug in case of future errors + local devs count # --nodeps ignores partitions - extra_dev="$(sudo lsblk --noheading --list --nodeps --output KNAME | egrep -v "($boot_dev|loop|nbd)" | head -1)" - if [ -z "$extra_dev" ]; then - create_extra_disk >/dev/stderr - extra_dev="$(sudo lsblk --noheading --list --nodeps --output KNAME | egrep -v "($boot_dev|loop|nbd)" | head -1)" + devs="$(sudo lsblk --noheading --list --nodeps --output KNAME | egrep -v "($boot_dev|loop|nbd)")" + count="$(echo "$devs" | grep -c . || true)" + if [ "$count" -lt "$min_count" ]; then + create_extra_disk "$((min_count - count))" >/dev/stderr + devs="$(sudo lsblk --noheading --list --nodeps --output KNAME | egrep -v "($boot_dev|loop|nbd)")" fi - echo " == find_extra_block_dev(): extra_dev='$extra_dev'" >/dev/stderr # debug in case of future errors - echo "$extra_dev" # output of function + echo " == find_extra_block_devs(): wanted>=${min_count} devs='$(echo "$devs" | tr '\n' ' ')'" >/dev/stderr + echo "$devs" # output of function +} + +# Print the basename of a second whole disk, distinct from the primary disk +# returned by find_extra_block_dev(). Ensures at least two disks exist first +# (provisioning an iSCSI disk if needed). Debug goes to stderr; stdout is clean. +function find_second_block_dev() { + local devs + mapfile -t devs < <(find_extra_block_devs 2) + echo "${devs[1]}" # the second disk; find_extra_block_dev() returns the first } function block_dev() { @@ -108,23 +158,6 @@ function print_k8s_cluster_status() { kubectl get pods -n kube-system } -function prepare_loop_devices() { - if [ $# -ne 1 ]; then - echo "usage: $0 loop_deivce_count" - exit 1 - fi - OSD_COUNT=$1 - if [ $OSD_COUNT -le 0 ]; then - echo "Invalid OSD_COUNT $OSD_COUNT. OSD_COUNT must be larger than 0." - exit 1 - fi - for i in $(seq 1 $OSD_COUNT); do - sudo dd if=/dev/zero of=~/data${i}.img bs=1M seek=6144 count=0 - sudo losetup /dev/loop${i} ~/data${i}.img - done - sudo lsblk -} - function use_local_disk() { BLOCK_DATA_PART="$(block_dev)1" sudo apt purge snapd -y @@ -320,9 +353,6 @@ function deploy_manifest_with_local_build() { if [[ "$USE_LOCAL_BUILD" != "false" ]]; then sed -i "s|image: docker.io/rook/ceph:.*|image: docker.io/rook/ceph:local-build|g" $1 fi - if [[ "$ALLOW_LOOP_DEVICES" = "true" ]]; then - sed -i "s|ROOK_CEPH_ALLOW_LOOP_DEVICES: \"false\"|ROOK_CEPH_ALLOW_LOOP_DEVICES: \"true\"|g" $1 - fi sed -i "s|ROOK_LOG_LEVEL:.*|ROOK_LOG_LEVEL: DEBUG|g" "$1" kubectl create -f $1 } @@ -374,9 +404,12 @@ function deploy_cluster() { sed -i "s|#deviceFilter:|deviceFilter: $(block_dev_basename)\n config:\n encryptedDevice: \"true\"|g" cluster-test.yaml elif [ "$1" = "lvm" ]; then sed -i "s|#deviceFilter:|devices:\n - name: \"/dev/test-rook-vg/test-rook-lv\"|g" cluster-test.yaml - elif [ "$1" = "loop" ]; then - # add both /dev/sdX1 and loop device to test them at the same time - sed -i "s|#deviceFilter:|devices:\n - name: \"$(block_dev_basename)\"\n - name: \"/dev/loop1\"|g" cluster-test.yaml + elif [ "$1" = "two_raw_disks" ]; then + # use two real disks as raw OSDs; ROOK_DATA_DEV and ROOK_EXTRA_DEV are + # basenames (e.g. sdb, sdc) exported by the job. An explicit devices list + # plus useAllDevices=false pins the OSD count regardless of any stray disk. + sed -i "s|#deviceFilter:|devices:\n - name: \"${ROOK_DATA_DEV}\"\n - name: \"${ROOK_EXTRA_DEV}\"|g" cluster-test.yaml + yq w -i -d0 cluster-test.yaml spec.storage.useAllDevices false else echo "invalid argument: $*" >&2 exit 1