From 22d4139b74fce0951f3fea5c6a6c95aa82d96f15 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 11 Jul 2024 16:57:08 -0600 Subject: [PATCH] core: add support for ceph squid With the release of the first squid RC, we add squid to the supported versions and add tests to run Rook against the squid release. Signed-off-by: Travis Nielsen --- .github/workflows/daily-nightly-jobs.yml | 83 ++++++++++++++++++++- PendingReleaseNotes.md | 2 + deploy/examples/cluster-test.yaml | 2 +- pkg/operator/ceph/cluster/version_test.go | 10 ++- pkg/operator/ceph/version/version.go | 25 ++++++- pkg/operator/ceph/version/version_test.go | 2 +- tests/framework/installer/ceph_installer.go | 8 +- tests/integration/ceph_upgrade_test.go | 44 +++++++++++ 8 files changed, 167 insertions(+), 9 deletions(-) diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml index 2766a905e..4fa73a4ed 100644 --- a/.github/workflows/daily-nightly-jobs.yml +++ b/.github/workflows/daily-nightly-jobs.yml @@ -195,6 +195,46 @@ jobs: name: ceph-smoke-suite-reef-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + smoke-suite-squid-devel: + if: github.repository == 'rook/rook' + runs-on: ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: consider debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + + - name: setup cluster resources + uses: ./.github/workflows/integration-test-config-latest-k8s + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + kubernetes-version: "1.28.4" + + - name: TestCephSmokeSuite + run: | + export DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) + SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="squid-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration + + - name: collect common logs + if: always() + run: | + export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" + export CLUSTER_NAMESPACE="smoke-ns" + export OPERATOR_NAMESPACE="smoke-ns-system" + tests/scripts/collect-logs.sh + + - name: Artifact + uses: actions/upload-artifact@v4 + if: failure() + with: + name: ceph-smoke-suite-squid-artifact + path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + smoke-suite-ceph-main: if: github.repository == 'rook/rook' runs-on: ubuntu-22.04 @@ -315,6 +355,47 @@ jobs: name: ceph-object-suite-master-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + upgrade-from-squid-stable-to-squid-devel: + if: github.repository == 'rook/rook' + runs-on: ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: consider debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + + - name: setup cluster resources + uses: ./.github/workflows/integration-test-config-latest-k8s + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + kubernetes-version: "1.28.4" + + - name: TestCephUpgradeSuite + run: | + export DEVICE_FILTER=$(tests/scripts/github-action-helper.sh find_extra_block_dev) + go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToSquidDevel github.com/rook/rook/tests/integration + + - name: collect common logs + if: always() + run: | + export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" + export CLUSTER_NAMESPACE="upgrade" + export OPERATOR_NAMESPACE="upgrade-system" + tests/scripts/collect-logs.sh + + - name: Artifact + uses: actions/upload-artifact@v4 + if: failure() + with: + name: ceph-upgrade-suite-squid-artifact + path: /home/runner/work/rook/rook/tests/integration/_output/tests/ + + upgrade-from-reef-stable-to-reef-devel: if: github.repository == 'rook/rook' runs-on: ubuntu-22.04 @@ -399,5 +480,5 @@ jobs: if: github.repository == 'rook/rook' uses: ./.github/workflows/canary-integration-test.yml with: - ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/daemon-base:latest-main-devel", "quay.io/ceph/daemon-base:latest-quincy-devel", "quay.io/ceph/daemon-base:latest-reef-devel"]' + ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/daemon-base:latest-main-devel", "quay.io/ceph/daemon-base:latest-quincy-devel", "quay.io/ceph/daemon-base:latest-reef-devel", "quay.io/ceph/daemon-base:latest-squid-devel"]' secrets: inherit diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 0cf183144..556160b33 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -7,3 +7,5 @@ please update the `BucketClass` and `BucketAccessClass` for resolving refer [her - During CephBlockPool updates, return an error if an invalid device class is specified. Pools with invalid device classes may start failing reconcile until the correct device class is specified. See #14057. ## Features + +- Added support for Ceph Squid (v19) diff --git a/deploy/examples/cluster-test.yaml b/deploy/examples/cluster-test.yaml index 2f088c5d5..db4c6d75f 100644 --- a/deploy/examples/cluster-test.yaml +++ b/deploy/examples/cluster-test.yaml @@ -15,7 +15,7 @@ metadata: spec: dataDirHostPath: /var/lib/rook cephVersion: - image: quay.io/ceph/ceph:v18 + image: quay.io/ceph/ceph:v19 allowUnsupported: true mon: count: 1 diff --git a/pkg/operator/ceph/cluster/version_test.go b/pkg/operator/ceph/cluster/version_test.go index 5f7234083..8f4a2d657 100755 --- a/pkg/operator/ceph/cluster/version_test.go +++ b/pkg/operator/ceph/cluster/version_test.go @@ -165,6 +165,8 @@ func TestMinVersion(t *testing.T) { assert.NoError(t, c.validateCephVersion(v)) v = &cephver.CephVersion{Major: 18} assert.NoError(t, c.validateCephVersion(v)) + v = &cephver.CephVersion{Major: 19} + assert.NoError(t, c.validateCephVersion(v)) } func TestSupportedVersion(t *testing.T) { @@ -183,8 +185,12 @@ func TestSupportedVersion(t *testing.T) { v = &cephver.CephVersion{Major: 18, Minor: 2, Extra: 0} assert.NoError(t, c.validateCephVersion(v)) - // Squid is not supported - v = &cephver.CephVersion{Major: 19, Minor: 1, Extra: 0} + // Squid is supported + v = &cephver.CephVersion{Major: 19, Minor: 2, Extra: 0} + assert.NoError(t, c.validateCephVersion(v)) + + // Tentacle release is not supported + v = &cephver.CephVersion{Major: 20, Minor: 1, Extra: 0} assert.Error(t, c.validateCephVersion(v)) // Unsupported versions are now valid diff --git a/pkg/operator/ceph/version/version.go b/pkg/operator/ceph/version/version.go index 92d764d6e..705258102 100644 --- a/pkg/operator/ceph/version/version.go +++ b/pkg/operator/ceph/version/version.go @@ -48,9 +48,11 @@ var ( Reef = CephVersion{18, 0, 0, 0, ""} // Squid ceph version Squid = CephVersion{19, 0, 0, 0, ""} + // Tentacle ceph version + Tentacle = CephVersion{20, 0, 0, 0, ""} // supportedVersions are production-ready versions that rook supports - supportedVersions = []CephVersion{Quincy, Reef} + supportedVersions = []CephVersion{Quincy, Reef, Squid} // unsupportedVersions are possibly Ceph pin-point release that introduced breaking changes and not recommended unsupportedVersions []CephVersion @@ -90,6 +92,8 @@ func (v *CephVersion) ReleaseName() string { return "reef" case Squid.Major: return "squid" + case Tentacle.Major: + return "tentacle" default: return unknownVersionString } @@ -180,6 +184,11 @@ func (v *CephVersion) IsSquid() bool { return v.isRelease(Squid) } +// IsTentacle checks if the Ceph version is Tentacle +func (v *CephVersion) IsTentacle() bool { + return v.isRelease(Tentacle) +} + // IsAtLeast checks a given Ceph version is at least a given one func (v *CephVersion) IsAtLeast(other CephVersion) bool { if v.Major > other.Major { @@ -203,12 +212,22 @@ func (v *CephVersion) IsAtLeast(other CephVersion) bool { return true } -// IsAtLeastReef check that the Ceph version is at least Reef +// IsAtLeastTentacle checks that the Ceph version is at least Tentacle +func (v *CephVersion) IsAtLeastTentacle() bool { + return v.IsAtLeast(Tentacle) +} + +// IsAtLeastSquid checks that the Ceph version is at least Squid +func (v *CephVersion) IsAtLeastSquid() bool { + return v.IsAtLeast(Squid) +} + +// IsAtLeastReef checks that the Ceph version is at least Reef func (v *CephVersion) IsAtLeastReef() bool { return v.IsAtLeast(Reef) } -// IsAtLeastQuincy check that the Ceph version is at least Quincy +// IsAtLeastQuincy checks that the Ceph version is at least Quincy func (v *CephVersion) IsAtLeastQuincy() bool { return v.IsAtLeast(Quincy) } diff --git a/pkg/operator/ceph/version/version_test.go b/pkg/operator/ceph/version/version_test.go index 36e2e895c..1a95a3f1c 100644 --- a/pkg/operator/ceph/version/version_test.go +++ b/pkg/operator/ceph/version/version_test.go @@ -101,7 +101,7 @@ func TestSupported(t *testing.T) { for _, v := range supportedVersions { assert.True(t, v.Supported()) } - assert.False(t, Squid.Supported()) + assert.False(t, Tentacle.Supported()) } func TestIsRelease(t *testing.T) { diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 680925465..9d7a725b7 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -45,9 +45,11 @@ const ( // test with the latest releases quincyTestImage = "quay.io/ceph/ceph:v17" reefTestImage = "quay.io/ceph/ceph:v18" + squidTestImage = "quay.io/ceph/ceph:v19" // test with the current development versions quincyDevelTestImage = "quay.io/ceph/daemon-base:latest-quincy-devel" reefDevelTestImage = "quay.io/ceph/daemon-base:latest-reef-devel" + squidDevelTestImage = "quay.io/ceph/daemon-base:latest-squid-devel" // test with the latest Ceph main image mainTestImage = "quay.io/ceph/daemon-base:latest-main-devel" cephOperatorLabel = "app=rook-ceph-operator" @@ -71,6 +73,8 @@ var ( QuincyDevelVersion = cephv1.CephVersionSpec{Image: quincyDevelTestImage} ReefVersion = cephv1.CephVersionSpec{Image: reefTestImage} ReefDevelVersion = cephv1.CephVersionSpec{Image: reefDevelTestImage} + SquidVersion = cephv1.CephVersionSpec{Image: squidTestImage} + SquidDevelVersion = cephv1.CephVersionSpec{Image: squidDevelTestImage} MainVersion = cephv1.CephVersionSpec{Image: mainTestImage, AllowUnsupported: true} volumeReplicationBaseURL = fmt.Sprintf("https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/%s/config/crd/bases/", volumeReplicationVersion) volumeReplicationCRDURL = volumeReplicationBaseURL + "replication.storage.openshift.io_volumereplications.yaml" @@ -97,8 +101,10 @@ func ReturnCephVersion() cephv1.CephVersionSpec { return QuincyDevelVersion case "reef-devel": return ReefDevelVersion + case "squid-devel": + return SquidDevelVersion default: - return ReefDevelVersion + return SquidDevelVersion } } diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 98db0ee65..9c10a4cdb 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -168,6 +168,17 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) logger.Infof("Verified upgrade from quincy to reef") + // + // Upgrade from reef to squid + // + logger.Infof("*** UPGRADING CEPH FROM REEF TO SQUID ***") + s.gatherLogs(s.settings.OperatorNamespace, "_before_squid_upgrade") + s.upgradeCephVersion(installer.SquidVersion.Image, numOSDs) + // Verify reading and writing to the test clients + newFile = "post-squid-upgrade-file" + s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) + logger.Infof("Verified upgrade from reef to squid") + checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false) } @@ -237,6 +248,39 @@ func (s *UpgradeSuite) TestUpgradeCephToReefDevel() { checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false) } +func (s *UpgradeSuite) TestUpgradeCephToSquidDevel() { + s.baseSetup(false, installer.SquidVersion) + + objectUserID := "upgraded-user" + preFilename := "pre-upgrade-file" + s.settings.CephVersion = installer.SquidVersion + numOSDs, rbdFilesToRead, cephfsFilesToRead := s.deployClusterforUpgrade(objectUserID, preFilename) + clusterInfo := client.AdminTestClusterInfo(s.namespace) + requireBlockImagesRemoved := false + defer func() { + blockTestDataCleanUp(s.helper, s.k8sh, &s.Suite, clusterInfo, installer.BlockPoolName, installer.BlockPoolSCName, blockName, rbdPodName, requireBlockImagesRemoved) + cleanupFilesystemConsumer(s.helper, s.k8sh, &s.Suite, s.namespace, filePodName) + cleanupFilesystem(s.helper, s.k8sh, &s.Suite, s.namespace, installer.FilesystemName) + _ = s.helper.ObjectUserClient.Delete(s.namespace, objectUserID) + _ = s.helper.BucketClient.DeleteObc(obcName, installer.ObjectStoreSCName, bucketPrefix, maxObject, false) + _ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, installer.ObjectStoreName, installer.ObjectStoreSCName, "Delete") + objectStoreCleanUp(&s.Suite, s.helper, s.k8sh, s.settings.Namespace, installer.ObjectStoreName) + }() + + // + // Upgrade from squid to squid devel + // + logger.Infof("*** UPGRADING CEPH FROM SQUID STABLE TO SQUID DEVEL ***") + s.gatherLogs(s.settings.OperatorNamespace, "_before_squid_upgrade") + s.upgradeCephVersion(installer.SquidDevelVersion.Image, numOSDs) + // Verify reading and writing to the test clients + newFile := "post-squid-upgrade-file" + s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) + logger.Infof("verified upgrade from squid stable to squid devel") + + checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false) +} + func (s *UpgradeSuite) deployClusterforUpgrade(objectUserID, preFilename string) (int, []string, []string) { // // Create block, object, and file storage before the upgrade