diff --git a/.github/workflows/ceph-suite-integration-test.yml b/.github/workflows/ceph-suite-integration-test.yml new file mode 100644 index 000000000..29b451da6 --- /dev/null +++ b/.github/workflows/ceph-suite-integration-test.yml @@ -0,0 +1,160 @@ +name: Reusable Ceph suite integration tests +on: + # ONLY on.workflow_call ; call this from other files if needed + workflow_call: + inputs: + ceph_suite_versions: + description: "JSON list of Ceph suite versions such as squid-devel, tentacle-devel, umbrella-devel, main" + required: true + type: string + upgrade_suite_tests: + description: "JSON list of upgrade suite subtests such as TestUpgradeCephToSquidDevel, TestUpgradeCephToTentacleDevel" + default: '["TestUpgradeCephToSquidDevel", "TestUpgradeCephToTentacleDevel"]' + type: string + object_suite_tests: + description: "JSON list of object suite subtests such as TestWithoutTLS, TestWithTLS" + default: '["TestWithoutTLS", "TestWithTLS"]' + type: string + +defaults: + run: + # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell + shell: bash --noprofile --norc -eo pipefail -x {0} + +permissions: + contents: read + +env: + # Required for go-ceph rgw/admin account APIs. + GOFLAGS: "-tags=ceph_preview" + +jobs: + smoke-suite: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + ceph-suite-version: ${{ fromJson(inputs.ceph_suite_versions) }} + steps: + - name: checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: consider tmate debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + + - name: setup cluster resources + uses: ./.github/workflows/integration-test-setup-cluster-resources + with: + kubernetes-version: "v1.36.2" + + - name: TestCephSmokeSuite + run: | + DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) + export DEVICE_FILTER + SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="${{ matrix.ceph-suite-version }}" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration + + - name: collect common logs + if: always() + run: | + export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" + export CLUSTER_NAMESPACE="smoke-ns" + export OPERATOR_NAMESPACE="smoke-ns-system" + tests/scripts/collect-logs.sh + + - name: Artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: ceph-smoke-suite-${{ matrix.ceph-suite-version }}-artifact + path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + + upgrade-suite: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + upgrade-test: ${{ fromJson(inputs.upgrade_suite_tests) }} + steps: + - name: checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: consider tmate debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + + - name: setup cluster resources + uses: ./.github/workflows/integration-test-setup-cluster-resources + with: + kubernetes-version: "v1.36.2" + + - name: TestCephUpgradeSuite + run: | + DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) + export DEVICE_FILTER + go test -v -timeout 1800s -failfast -run "TestCephUpgradeSuite/${{ matrix.upgrade-test }}" github.com/rook/rook/tests/integration + + - name: collect common logs + if: always() + run: | + export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" + export CLUSTER_NAMESPACE="upgrade" + export OPERATOR_NAMESPACE="upgrade-system" + tests/scripts/collect-logs.sh + + - name: Artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: ceph-upgrade-suite-${{ matrix.upgrade-test }}-artifact + path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + + object-suite: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + ceph-suite-version: ${{ fromJson(inputs.ceph_suite_versions) }} + object-test: ${{ fromJson(inputs.object_suite_tests) }} + steps: + - name: checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: consider tmate debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + + - name: setup cluster resources + uses: ./.github/workflows/integration-test-setup-cluster-resources + with: + kubernetes-version: "v1.36.2" + + - name: TestCephObjectSuite + run: | + DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) + export DEVICE_FILTER + SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="${{ matrix.ceph-suite-version }}" go test -v -timeout 1800s -failfast -run "CephObjectSuite/${{ matrix.object-test }}" github.com/rook/rook/tests/integration + + - name: collect common logs + if: always() + run: | + export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" + export CLUSTER_NAMESPACE="object-ns" + export OPERATOR_NAMESPACE="object-ns-system" + tests/scripts/collect-logs.sh + + - name: Artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: ceph-object-suite-${{ matrix.ceph-suite-version }}-${{ matrix.object-test }}-artifact + path: /home/runner/work/rook/rook/tests/integration/_output/tests/ diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml index 2e0ab97f7..964358e2b 100644 --- a/.github/workflows/daily-nightly-jobs.yml +++ b/.github/workflows/daily-nightly-jobs.yml @@ -37,7 +37,7 @@ jobs: - name: setup cluster resources uses: ./.github/workflows/integration-test-setup-cluster-resources with: - kubernetes-version: "v1.36.1" + kubernetes-version: "v1.36.2" - name: validate-yaml run: tests/scripts/github-action-helper.sh validate_yaml @@ -70,291 +70,19 @@ jobs: with: name: canary-arm64 - smoke-suite-squid-devel: + ceph-suite-tests: if: github.repository == 'rook/rook' - runs-on: ubuntu-22.04 - steps: - - name: checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: consider tmate debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/integration-test-setup-cluster-resources - with: - kubernetes-version: "v1.36.1" - - - name: TestCephSmokeSuite - run: | - DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) - export DEVICE_FILTER - SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="squid-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration - - - name: collect common logs - if: always() - run: | - export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" - export CLUSTER_NAMESPACE="smoke-ns" - export OPERATOR_NAMESPACE="smoke-ns-system" - tests/scripts/collect-logs.sh - - - name: Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: ceph-smoke-suite-squid-artifact - path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - - smoke-suite-tentacle-devel: - if: github.repository == 'rook/rook' - runs-on: ubuntu-22.04 - steps: - - name: checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: consider tmate debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/integration-test-setup-cluster-resources - with: - kubernetes-version: "v1.36.1" - - - name: TestCephSmokeSuite - run: | - DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) - export DEVICE_FILTER - SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="tentacle-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration - - - name: collect common logs - if: always() - run: | - export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" - export CLUSTER_NAMESPACE="smoke-ns" - export OPERATOR_NAMESPACE="smoke-ns-system" - tests/scripts/collect-logs.sh - - - name: Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: ceph-smoke-suite-tentacle-artifact - path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - - smoke-suite-ceph-main: - if: github.repository == 'rook/rook' - runs-on: ubuntu-22.04 - steps: - - name: checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: consider tmate debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/integration-test-setup-cluster-resources - with: - kubernetes-version: "v1.36.1" - - - name: TestCephSmokeSuite - run: | - DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) - export DEVICE_FILTER - SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=main go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration - - - name: collect common logs - if: always() - run: | - export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" - export CLUSTER_NAMESPACE="smoke-ns" - export OPERATOR_NAMESPACE="smoke-ns-system" - tests/scripts/collect-logs.sh - - - name: Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: ceph-smoke-suite-main-artifact - path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - - object-suite-ceph-main: - if: github.repository == 'rook/rook' - runs-on: ubuntu-22.04 - steps: - - name: checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: consider tmate debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/integration-test-setup-cluster-resources - with: - kubernetes-version: "v1.36.1" - - - name: TestCephObjectSuite (without TLS) - run: | - DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) - export DEVICE_FILTER - SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=main go test -v -timeout 1800s -failfast -run CephObjectSuite/TestWithoutTLS github.com/rook/rook/tests/integration - - - name: collect common logs - if: always() - run: | - export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" - export CLUSTER_NAMESPACE="object-ns" - export OPERATOR_NAMESPACE="object-ns-system" - tests/scripts/collect-logs.sh - - - name: Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: ceph-object-suite-main-artifact - path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - - object-suite-tls-ceph-main: - if: github.repository == 'rook/rook' - runs-on: ubuntu-22.04 - steps: - - name: checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: consider tmate debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/integration-test-setup-cluster-resources - with: - kubernetes-version: "v1.36.1" - - - name: TestCephObjectSuiteTLS - run: | - DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) - export DEVICE_FILTER - SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=main go test -v -timeout 1800s -failfast -run CephObjectSuite/TestWithTLS github.com/rook/rook/tests/integration - - - name: collect common logs - if: always() - run: | - export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" - export CLUSTER_NAMESPACE="object-ns" - export OPERATOR_NAMESPACE="object-ns-system" - tests/scripts/collect-logs.sh - - - name: Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: ceph-object-suite-tls-main-artifact - path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - - upgrade-from-squid-stable-to-squid-devel: - if: github.repository == 'rook/rook' - runs-on: ubuntu-22.04 - steps: - - name: checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: consider tmate debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/integration-test-setup-cluster-resources - with: - kubernetes-version: "v1.36.1" - - - name: TestCephUpgradeSuite - run: | - DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) - export DEVICE_FILTER - go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToSquidDevel github.com/rook/rook/tests/integration - - - name: collect common logs - if: always() - run: | - export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" - export CLUSTER_NAMESPACE="upgrade" - export OPERATOR_NAMESPACE="upgrade-system" - tests/scripts/collect-logs.sh - - - name: Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: ceph-upgrade-suite-squid-artifact - path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - - upgrade-from-tentacle-stable-to-tentacle-devel: - if: github.repository == 'rook/rook' - runs-on: ubuntu-22.04 - steps: - - name: checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: consider tmate debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/integration-test-setup-cluster-resources - with: - kubernetes-version: "v1.36.1" - - - name: TestCephUpgradeSuite - run: | - DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) - export DEVICE_FILTER - go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToTentacleDevel github.com/rook/rook/tests/integration - - - name: collect common logs - if: always() - run: | - export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" - export CLUSTER_NAMESPACE="upgrade" - export OPERATOR_NAMESPACE="upgrade-system" - tests/scripts/collect-logs.sh - - - name: Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: ceph-upgrade-suite-tentacle-artifact - path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + uses: ./.github/workflows/ceph-suite-integration-test.yml + with: + ceph_suite_versions: '["squid-devel", "tentacle-devel", "umbrella-devel", "main"]' + object_suite_tests: '["TestWithoutTLS", "TestWithTLS"]' + secrets: inherit canary-tests: if: github.repository == 'rook/rook' uses: ./.github/workflows/canary-integration-test.yml with: - ceph_images: '["quay.io/ceph/ceph:v19", "quay.io/ceph/ceph:v20", "quay.ceph.io/ceph-ci/ceph:main", "quay.ceph.io/ceph-ci/ceph:squid", "quay.ceph.io/ceph-ci/ceph:tentacle"]' + ceph_images: '["quay.io/ceph/ceph:v19", "quay.io/ceph/ceph:v20", "quay.ceph.io/ceph-ci/ceph:main", "quay.ceph.io/ceph-ci/ceph:squid", "quay.ceph.io/ceph-ci/ceph:tentacle", "quay.ceph.io/ceph-ci/ceph:umbrella"]' secrets: inherit check-markdown-links: diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 5b092b44a..0f079cd95 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -48,6 +48,7 @@ const ( // test with the current development versions squidDevelTestImage = "quay.ceph.io/ceph-ci/ceph:squid" tentacleDevelTestImage = "quay.ceph.io/ceph-ci/ceph:tentacle" + umbrellaDevelTestImage = "quay.ceph.io/ceph-ci/ceph:umbrella" // test with the latest Ceph main image mainTestImage = "quay.ceph.io/ceph-ci/ceph:main" cephOperatorLabel = "app=rook-ceph-operator" @@ -71,6 +72,7 @@ var ( SquidDevelVersion = cephv1.CephVersionSpec{Image: squidDevelTestImage} TentacleVersion = cephv1.CephVersionSpec{Image: tentacleTestImage} TentacleDevelVersion = cephv1.CephVersionSpec{Image: tentacleDevelTestImage, AllowUnsupported: true} + UmbrellaDevelVersion = cephv1.CephVersionSpec{Image: umbrellaDevelTestImage, AllowUnsupported: true} MainVersion = cephv1.CephVersionSpec{Image: mainTestImage, AllowUnsupported: true} volumeReplicationBaseURL = fmt.Sprintf("https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/%s/config/crd/bases/", volumeReplicationVersion) volumeReplicationCRDURL = volumeReplicationBaseURL + "replication.storage.openshift.io_volumereplications.yaml" @@ -97,6 +99,8 @@ func ReturnCephVersion() cephv1.CephVersionSpec { return SquidDevelVersion case "tentacle-devel": return TentacleDevelVersion + case "umbrella-devel": + return UmbrellaDevelVersion default: // Default to the latest stable version return SquidVersion