From e80368674d14f185a4d06ea524eb8c1d0cc5a4b7 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Wed, 6 Oct 2021 13:38:02 -0600 Subject: [PATCH] test: run RGW multisite test in nightly job In the nightly job, run the test with the latest Ceph version so we can detect if there are RGW changes in Ceph that might break multisite. Use a reusable GitHub action workflow to duplicate as little code as possible. Signed-off-by: Blaine Gardner --- .github/workflows/canary-integration-test.yml | 69 +------------- .github/workflows/daily-nightly-jobs.yml | 24 +++++ .../workflows/rgw-multisite-test/action.yml | 93 +++++++++++++++++++ tests/scripts/github-action-helper.sh | 13 ++- 4 files changed, 132 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/rgw-multisite-test/action.yml diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index b3a9929fe..23d96e8d4 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -762,72 +762,11 @@ jobs: with: fetch-depth: 0 - - name: setup golang - uses: actions/setup-go@v2 + - name: run RGW multisite test + uses: ./.github/workflows/rgw-multisite-test with: - go-version: 1.16 - - - name: install deps - run: | - tests/scripts/github-action-helper.sh install_deps - sudo apt-get install -y s3cmd - - name: setup minikube - uses: manusa/actions-setup-minikube@v2.3.1 - with: - minikube version: 'v1.23.2' - kubernetes version: 'v1.22.2' - start args: --memory 6g --cpus=2 - github token: ${{ secrets.GITHUB_TOKEN }} - - - name: use local disk into two partitions - run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 - sudo lsblk - - name: build rook - run: tests/scripts/github-action-helper.sh build_rook - - - name: deploy first cluster rook - run: | - tests/scripts/github-action-helper.sh deploy_first_rook_cluster - kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml - # wait for multisite store to be created - tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph - - name: prep second cluster pull realm config - run: | - cd cluster/examples/kubernetes/ceph/ - IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") - yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80 - BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") - BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") - sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml - sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml - - name: deploy second cluster rook - run: | - tests/scripts/github-action-helper.sh deploy_second_rook_cluster - kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml - # wait for realms to be pulled and zone-b-multisite-store to be created - tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph-secondary - - name: write an object to one cluster, read from the other - run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2 - - # if this test fails, it could mean the RGW `period get` or `period update` output has changed - - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile - run: | - ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended - ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended - committed_msg="committing changes to RGW configuration period for CephObjectStore" - tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}" - tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}" - tests/scripts/github-action-helper.sh restart_operator - not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore" - tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120 - tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90 - - name: collect common logs - if: always() - run: | - tests/scripts/collect-logs.sh + github-token: ${{ secrets.GITHUB_TOKEN }} + # ceph-image: # use default - name: upload test result uses: actions/upload-artifact@v2 diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml index afc437e38..fee8a7442 100644 --- a/.github/workflows/daily-nightly-jobs.yml +++ b/.github/workflows/daily-nightly-jobs.yml @@ -292,3 +292,27 @@ jobs: with: name: ceph-upgrade-suite-octopus-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + + rgw-multisite-test-with-ceph-devel: + runs-on: ubuntu-18.04 + strategy: + matrix: + ceph-image-tag: ['latest-master-devel', 'latest-octopus-devel', 'latest-pacific-devel'] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: run RGW multisite test + uses: ./.github/workflows/rgw-multisite-test + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + ceph-image: quay.io/ceph/daemon-base:${{ matrix.ceph-image-tag }} + + - name: upload test result + uses: actions/upload-artifact@v2 + if: always() + with: + name: rgw-multisite-testing-ceph-${{ matrix.ceph-image-tag }} + path: test diff --git a/.github/workflows/rgw-multisite-test/action.yml b/.github/workflows/rgw-multisite-test/action.yml new file mode 100644 index 000000000..ff3125d7f --- /dev/null +++ b/.github/workflows/rgw-multisite-test/action.yml @@ -0,0 +1,93 @@ +name: RGW Multisite Test +description: Reusable workflow to test RGW multisite integration +inputs: + github-token: + description: GITHUB_TOKEN from the calling workflow + required: true + ceph-image: + description: Ceph image to use for the workflow (e.g., quay.io/ceph/ceph:v16.2.5) + required: false + +runs: + using: "composite" + steps: + - name: setup cluster resources + uses: ./.github/workflows/setup-cluster-resources + with: + github-token: ${{ inputs.github-token }} + + - name: install additional deps for object testing + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + sudo apt-get install -y s3cmd + + - name: use local disk into two partitions + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/github-action-helper.sh use_local_disk + tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 + sudo lsblk + + - name: set Ceph version in CephCluster manifest + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh replace_ceph_image \ + "cluster/examples/kubernetes/ceph/cluster-test.yaml" "${{ inputs.ceph-image }}" + + - name: deploy first cluster rook + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh deploy_first_rook_cluster + kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml + # wait for multisite store to be created + tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph + + - name: prep second cluster pull realm config + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + cd cluster/examples/kubernetes/ceph/ + IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") + yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80 + BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") + BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") + sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml + sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml + + - name: deploy second cluster rook + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh deploy_second_rook_cluster + kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml + # wait for realms to be pulled and zone-b-multisite-store to be created + tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph-secondary + + - name: wait for ceph cluster 1 to be ready + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/validate_cluster.sh osd 1 + kubectl -n rook-ceph get pods + + - name: write an object to one cluster, read from the other + shell: bash --noprofile --norc -eo pipefail -x {0} + run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2 + + # if this test fails, it could mean the RGW `period get` or `period update` output has changed + - name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + ns_name_primary='"rook-ceph/multisite-store"' # double quotes intended + ns_name_secondary='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended + committed_msg="committing changes to RGW configuration period for CephObjectStore" + tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary}" + tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary}" + tests/scripts/github-action-helper.sh restart_operator + not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore" + tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary}" 120 + tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary}" 90 + + - name: collect common logs + if: always() + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/collect-logs.sh diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 1a4fd88ae..15c395ae8 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -192,6 +192,16 @@ function deploy_manifest_with_local_build() { kubectl create -f $1 } +function replace_ceph_image() { + local file="$1" # parameter 1: the file in which to replace the ceph image + local ceph_image="${2:-}" # parameter 2: the new ceph image to use + if [[ -z ${ceph_image} ]]; then + echo "No Ceph image given. Not adjusting manifests." + return 0 + fi + sed -i "s|image: .*ceph/ceph:.*|image: ${ceph_image}|g" "${file}" +} + function deploy_cluster() { cd cluster/examples/kubernetes/ceph deploy_manifest_with_local_build operator.yaml @@ -265,7 +275,6 @@ function deploy_first_rook_cluster() { BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1) create_cluster_prerequisites cd cluster/examples/kubernetes/ceph/ - deploy_manifest_with_local_build operator.yaml yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false @@ -283,7 +292,7 @@ function deploy_second_rook_cluster() { yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external" kubectl create -f cluster-test.yaml yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary - deploy_manifest_with_local_build toolbox.yaml toolbox.yaml + deploy_manifest_with_local_build toolbox.yaml } function wait_for_rgw() {