test: add object-with-cephblockpool canary test

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
(cherry picked from commit ed8156017e)

.github/workflows/canary-integration-test.yml

@@ -1698,3 +1698,53 @@ jobs:
         uses: ./.github/workflows/collect-logs
         with:
           name: ${{ github.job }}-${{ matrix.ceph-image }}
+
+  object-with-cephblockpools:
+    runs-on: ubuntu-22.04
+    if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
+    strategy:
+      matrix:
+        ceph-image: ${{ fromJson(inputs.ceph_images) }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          fetch-depth: 0
+
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
+      - name: setup cluster resources
+        uses: ./.github/workflows/canary-test-config
+
+      - name: set Ceph version in CephCluster manifest
+        run: tests/scripts/github-action-helper.sh replace_ceph_image  "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}"
+
+      - name: validate-yaml
+        run: tests/scripts/github-action-helper.sh validate_yaml
+
+      - name: use local disk and create partitions for osds
+        run: |
+          tests/scripts/github-action-helper.sh use_local_disk
+          tests/scripts/github-action-helper.sh create_partitions_for_osds
+
+      - name: deploy cluster
+        run: tests/scripts/github-action-helper.sh deploy_cluster
+
+      - name: create CephBlockPool(s) and CephObjectStore
+        shell: bash --noprofile --norc -eo pipefail -x {0}
+        run: kubectl create -f deploy/examples/object-with-cephblockpools-test.yaml
+
+      - name: wait for CephObjectStore to be ready
+        run: tests/scripts/validate_cluster.sh rgw object-with-cephblockpools
+
+      - name: check for pools created by RGW that are unexpected
+        run: tests/scripts/github-action-helper.sh test_object_with_cephblockpools_extra_pools
+
+      - name: collect common logs
+        if: always()
+        uses: ./.github/workflows/collect-logs
+        with:
+          name: ${{ github.job }}-${{ matrix.ceph-image }}

deploy/examples/object-with-cephblockpools-test.yaml

@@ -0,0 +1,135 @@
+##########################################################
+# Create an object store using pre-created pools
+#  kubectl create -f object-with-cephblockpools-test.yaml
+##########################################################
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: object-with-cephblockpools
+  namespace: rook-ceph # namespace:cluster
+spec:
+  gateway:
+    port: 80
+    instances: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: rgw.root
+  namespace: rook-ceph # namespace:cluster
+spec:
+  application: rgw
+  failureDomain: osd
+  name: .rgw.root
+  parameters:
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: object-with-cephblockpools.rgw.control
+  namespace: rook-ceph # namespace:cluster
+spec:
+  application: rgw
+  failureDomain: osd
+  parameters:
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: object-with-cephblockpools.rgw.meta
+  namespace: rook-ceph # namespace:cluster
+spec:
+  application: rgw
+  failureDomain: osd
+  parameters:
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: object-with-cephblockpools.rgw.log
+  namespace: rook-ceph # namespace:cluster
+spec:
+  application: rgw
+  failureDomain: osd
+  parameters:
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: object-with-cephblockpools.rgw.buckets.index
+  namespace: rook-ceph # namespace:cluster
+spec:
+  application: rgw
+  failureDomain: osd
+  parameters:
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: object-with-cephblockpools.rgw.buckets.non-ec
+  namespace: rook-ceph # namespace:cluster
+spec:
+  application: rgw
+  failureDomain: osd
+  parameters:
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: object-with-cephblockpools.rgw.otp
+  namespace: rook-ceph # namespace:cluster
+spec:
+  application: rgw
+  failureDomain: osd
+  parameters:
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: object-with-cephblockpools.rgw.buckets.data
+  namespace: rook-ceph # namespace:cluster
+spec:
+  application: rgw
+  failureDomain: osd
+  parameters:
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1

tests/scripts/github-action-helper.sh

@@ -754,6 +754,43 @@ function radosgw-admin() {
   toolbox radosgw-admin "$@"
 }
 
+function test_object_with_cephblockpools_extra_pools() {
+  expected_pools=(
+    .mgr
+    .rgw.root
+    object-with-cephblockpools.rgw.control
+    object-with-cephblockpools.rgw.meta
+    object-with-cephblockpools.rgw.log
+    object-with-cephblockpools.rgw.buckets.index
+    object-with-cephblockpools.rgw.buckets.non-ec
+    object-with-cephblockpools.rgw.otp
+    object-with-cephblockpools.rgw.buckets.data
+  )
+
+  output=$(ceph osd pool ls)
+  readarray -t live_pools < <(printf '%s' "$output")
+
+  errors=0
+  for l in "${live_pools[@]}"; do
+    found=false
+    for e in "${expected_pools[@]}"; do
+      if [[ "$l" == "$e" ]]; then
+        found=true
+        break
+      fi
+    done
+    if [[ "$found" == false ]]; then
+      echo "Live pool $l is not an expected pool"
+      errors=$((errors+1))
+    fi
+  done
+
+  if [[ $errors -gt 0 ]]; then
+    echo "Found $errors errors"
+    exit $errors
+  fi
+}
+
 FUNCTION="$1"
 shift # remove function arg now that we've recorded it
 # call the function with the remainder of the user-provided args