From 114a949cfbd39437c91cdfe1dae76df98163fc1a Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 17 Dec 2020 15:04:24 -0700 Subject: [PATCH] ceph: test latest ceph on raw devices and previous ceph on partitions The latest ceph-volume does not support creating OSDs on partitions. The github actions only have a partition available, so we will run the github tests on v14.2.12 and v15.2.7 that still support partitions, while the Jenkins environment has a raw device available where we can run the latest versions of Ceph in the tests. Signed-off-by: Travis Nielsen --- .github/workflows/canary-integration-test.yml | 8 +++--- Jenkinsfile | 1 + tests/framework/installer/ceph_installer.go | 27 ++++++++++++++++--- tests/integration/ceph_flex_test.go | 2 +- tests/integration/ceph_helm_test.go | 2 +- tests/integration/ceph_multi_cluster_test.go | 4 +-- tests/integration/ceph_smoke_test.go | 2 +- tests/integration/ceph_upgrade_test.go | 4 +-- 8 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 17f5a4256..de3c644c5 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -49,11 +49,11 @@ jobs: docker images docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master - + - name: validate-yaml run: | cd cluster/examples/kubernetes/ceph/ - kubectl create -f crds.yaml -f common.yaml + kubectl create -f crds.yaml -f common.yaml # skipping folders and some yamls that are only for openshift. kubectl create $(ls -I scc.yaml -I operator-openshift.yaml -I "*.sh" -I "*.py" -p | grep -v / | awk ' { print " -f " $1 } ') --dry-run @@ -61,6 +61,8 @@ jobs: run: | kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml sed -i "s|#deviceFilter:|deviceFilter: $(lsblk|awk '/14G/ {print $1}'| head -1)|g" cluster/examples/kubernetes/ceph/cluster-test.yaml + # recent c-v changes don't allow OSDs on a partition, so we stick with v15.2.7 until that is resolved + sed -i "s|:v15|:v15.2.7|g" cluster/examples/kubernetes/ceph/cluster-test.yaml kubectl create -f cluster/examples/kubernetes/ceph/cluster-test.yaml kubectl create -f cluster/examples/kubernetes/ceph/object-test.yaml kubectl create -f cluster/examples/kubernetes/ceph/pool-test.yaml @@ -627,4 +629,4 @@ jobs: - name: validate vault run: | tests/scripts/deploy-validate-vault.sh validate - sudo lsblk \ No newline at end of file + sudo lsblk diff --git a/Jenkinsfile b/Jenkinsfile index 08e5030b5..254fc8edd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -244,6 +244,7 @@ def RunIntegrationTest(k, v) { STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+''' \ TEST_ARGUMENTS='''+"${env.testArgs}"+''' \ TEST_IS_OFFICIAL_BUILD='''+"${env.isOfficialBuild}"+''' \ + TEST_OSDS_ON_PARTITIONS="false" \ TEST_SCRATCH_DEVICE=/dev/nvme0n1 kubectl config view _output/tests/linux_amd64/integration -test.v -test.timeout 7200s 2>&1 | tee _output/tests/integrationTests.log''' diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 6f72813bd..5bc63beb0 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -42,19 +42,24 @@ import ( const ( // test with the latest nautilus build - nautilusTestImage = "ceph/ceph:v14.2.12" + nautilusTestImage = "ceph/ceph:v14" + // test with the latest nautilus build. ceph-volume is not allowing OSDs on partitions on v14.2.13 and newer. + nautilusTestImageOnPartitions = "ceph/ceph:v14.2.12" // test with the latest octopus build octopusTestImage = "ceph/ceph:v15" + // test with the latest octopus build. ceph-volume is not allowing OSDs on partitions on v15.2.8 and newer. + octopusTestImageOnPartitions = "ceph/ceph:v15.2.7" // test with the latest master image masterTestImage = "ceph/daemon-base:latest-master-devel" cephOperatorLabel = "app=rook-ceph-operator" defaultclusterName = "test-cluster" + // if false, expect to create OSDs on raw devices, + // otherwise use a version of ceph that is compatible with OSDs on partitions + usePartitionEnvVar = "TEST_OSDS_ON_PARTITIONS" ) var ( - NautilusVersion = cephv1.CephVersionSpec{Image: nautilusTestImage} - OctopusVersion = cephv1.CephVersionSpec{Image: octopusTestImage} - MasterVersion = cephv1.CephVersionSpec{Image: masterTestImage, AllowUnsupported: true} + MasterVersion = cephv1.CephVersionSpec{Image: masterTestImage, AllowUnsupported: true} ) // CephInstaller wraps installing and uninstalling rook on a platform @@ -72,6 +77,20 @@ type CephInstaller struct { cleanupHost bool } +func NautilusVersion() cephv1.CephVersionSpec { + if os.Getenv(usePartitionEnvVar) == "false" { + return cephv1.CephVersionSpec{Image: nautilusTestImage} + } + return cephv1.CephVersionSpec{Image: nautilusTestImageOnPartitions} +} + +func OctopusVersion() cephv1.CephVersionSpec { + if os.Getenv(usePartitionEnvVar) == "false" { + return cephv1.CephVersionSpec{Image: octopusTestImage} + } + return cephv1.CephVersionSpec{Image: octopusTestImageOnPartitions} +} + // CreateCephOperator creates rook-operator via kubectl func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { logger.Infof("Starting Rook Operator") diff --git a/tests/integration/ceph_flex_test.go b/tests/integration/ceph_flex_test.go index e00618e70..a90e6c4a3 100644 --- a/tests/integration/ceph_flex_test.go +++ b/tests/integration/ceph_flex_test.go @@ -96,7 +96,7 @@ func (s *CephFlexDriverSuite) SetupSuite() { skipOSDCreation: false, minimalMatrixK8sVersion: flexDriverMinimalTestVersion, rookVersion: installer.VersionMaster, - cephVersion: installer.OctopusVersion, + cephVersion: installer.OctopusVersion(), } s.clusterInfo = client.AdminClusterInfo(s.namespace) diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index a50420cdb..95305a5e8 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -74,7 +74,7 @@ func (hs *HelmSuite) SetupSuite() { skipOSDCreation: false, minimalMatrixK8sVersion: helmMinimalTestVersion, rookVersion: installer.VersionMaster, - cephVersion: installer.NautilusVersion, + cephVersion: installer.NautilusVersion(), } hs.op, hs.kh = StartTestCluster(hs.T, &helmTestCluster) diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index d1f6373c5..8a45cc66f 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -142,7 +142,7 @@ func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 stri checkIfShouldRunForMinimalTestMatrix(t, kh, multiClusterMinimalTestVersion) cleanupHost := false - i := installer.NewCephInstaller(t, kh.Clientset, false, "", installer.VersionMaster, installer.NautilusVersion, cleanupHost) + i := installer.NewCephInstaller(t, kh.Clientset, false, "", installer.VersionMaster, installer.NautilusVersion(), cleanupHost) op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1), "", false} if kh.VersionAtLeast("v1.13.0") { @@ -199,7 +199,7 @@ func (o MCTestOperations) Teardown() { func (o MCTestOperations) startCluster(namespace, store string) error { logger.Infof("starting cluster %s", namespace) err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName, - cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, false, false, installer.NautilusVersion) + cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, false, false, installer.NautilusVersion()) if err != nil { o.T().Fail() o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index fa6981343..58d4911e6 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -99,7 +99,7 @@ func (suite *SmokeSuite) SetupSuite() { skipOSDCreation: false, minimalMatrixK8sVersion: smokeSuiteMinimalTestVersion, rookVersion: installer.VersionMaster, - cephVersion: installer.OctopusVersion, + cephVersion: installer.OctopusVersion(), } suite.op, suite.k8sh = StartTestCluster(suite.T, &smokeTestCluster) diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index d96a393c4..b2ebe164f 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -86,7 +86,7 @@ func (s *UpgradeSuite) SetupSuite() { skipOSDCreation: false, minimalMatrixK8sVersion: upgradeMinimalTestVersion, rookVersion: installer.Version1_4, - cephVersion: installer.NautilusVersion, + cephVersion: installer.NautilusVersion(), } s.op, s.k8sh = StartTestCluster(s.T, &upgradeTestCluster) @@ -177,7 +177,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // logger.Infof("*** UPGRADING CEPH FROM Nautilus TO Octopus ***") s.gatherLogs(systemNamespace, "_before_octopus_upgrade") - s.upgradeCephVersion(installer.OctopusVersion.Image, numOSDs) + s.upgradeCephVersion(installer.OctopusVersion().Image, numOSDs) // Verify reading and writing to the test clients newFile = "post-octopus-upgrade-file" s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead)