From d40e4904440b47fe127da0d4f9c86985b106d685 Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 16 Jul 2019 22:16:15 -0600 Subject: [PATCH 1/3] allow executing tests specific to a storage provider Since most PRs are focused on a single storage provider, the tests for other storage providers do not need to be run. The CI will allow skipping tests for other storage providers by specifying keywords in the PR description. Similar to [skip ci], you can now specify [test storage-provider]. For example, [test ceph] will only trigger the test suites for ceph and none of the other storage providers. Signed-off-by: travisn --- INSTALL.md | 18 +++++++++++ Jenkinsfile | 30 +++++++++---------- PendingReleaseNotes.md | 1 + tests/framework/installer/installer.go | 22 ++++++++++++++ tests/integration/block_create_test.go | 4 +++ tests/integration/cassandra_test.go | 4 +++ tests/integration/cockroachdb_test.go | 4 +++ tests/integration/edgefs_test.go | 4 +++ tests/integration/helm_test.go | 4 +++ tests/integration/multi_cluster_test.go | 4 +++ tests/integration/nfs_test.go | 4 +++ tests/integration/smoke_test.go | 4 +++ tests/integration/upgrade_test.go | 4 +++ .../integration/zblock_mount_unmount_test.go | 4 +++ 14 files changed, 96 insertions(+), 15 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index dfda07b53..47ed66507 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -165,3 +165,21 @@ This option should **not** be used if there are any changes to `.go` files, CI s In case there are known CI failures that are blocking a merge, project maintainers can add this flag to unblock the CI and merge the PR. This should be used with extreme care so regressions are not introduced. + +### [test storage-provider] + +Jenkins will only run the tests for an individual storage provider if specified. There is a keyword for each +storage provider that has integration tests: +- `[test cassandra]` +- `[test ceph]` +- `[test cockroachdb]` +- `[test edgefs]` +- `[test nfs]` + +Since the majority of effort for a PR generally focuses on a single storage provider it will save time +in the CI if you only trigger the tests for your provider. If this key phrase is not found in the PR, +all tests will be executed. + +These options should **not** be used if there are any changes to code which is shared with other +storage providers. If there is any risk of affecting another storage provider, all tests should +be executed in the CI. diff --git a/Jenkinsfile b/Jenkinsfile index 3a05972d3..d1ea25e9e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,12 +27,22 @@ pipeline { if (body.contains("[skip tests]")) { env.shouldTest = "false" } - if (body.contains("[smoke only]")) { - env.smokeOnly = "true" - } if (body.contains("[all logs]")) { env.getLogs = "all" } + // extract which specific storage provider to test + if (body.contains("[test cassandra]")) { + env.testProvider = "cassandra" + } else if (body.contains("[test ceph]")) { + env.testProvider = "ceph" + } else if (body.contains("[test cockroachdb]")) { + env.testProvider = "cockroachdb" + } else if (body.contains("[test edgefs]")) { + env.testProvider = "edgefs" + } else if (body.contains("[test nfs]")) { + env.testProvider = "nfs" + } + echo ("integration test provider: ${env.testProvider}") } } } @@ -159,24 +169,14 @@ def RunIntegrationTest(k, v) { export KUBECONFIG=$HOME/admin.conf tests/scripts/helm.sh up''' try{ - if ("${env.smokeOnly}" == "true") { - echo "Running Smoke Tests" - sh '''#!/bin/bash - set -o pipefail - export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \ - KUBECONFIG=$HOME/admin.conf - kubectl config view - _output/tests/linux_amd64/integration -test.v -test.timeout 1800s -test.run SmokeSuite --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log''' - } - else { echo "Running full regression" sh '''#!/bin/bash set -o pipefail export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \ - KUBECONFIG=$HOME/admin.conf + KUBECONFIG=$HOME/admin.conf \ + STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+''' kubectl config view _output/tests/linux_amd64/integration -test.v -test.timeout 7200s --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log''' - } } finally{ sh "journalctl -u kubelet > _output/tests/kubelet_${v}.log" diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 5af951e3c..fd6838302 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -9,6 +9,7 @@ an example usage - Added K8s 1.15 to the test matrix and removed K8s 1.10 from the test matrix. - OwnerReferences are created with the fully qualified `apiVersion` such that the references will work properly on OpenShift. - Linear disk device can now be used for Ceph OSDs. +- The integration tests can be triggered for specific storage providers rather than always running all tests. See the [dev guide](INSTALL.md#test-storage-provider) for more details. ### Ceph diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index 6a3dfbb35..86de7d459 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "runtime" + "strings" "testing" "github.com/coreos/pkg/capnslog" @@ -33,6 +34,12 @@ const ( VersionMaster = "master" // Version tag for Rook v0.9 Version1_0 = "v1.0.1" + // test suite names + CassandraTestSuite = "cassandra" + CephTestSuite = "ceph" + CockroachDBTestSuite = "cockroachdb" + EdgeFSTestSuite = "edgefs" + NFSTestSuite = "nfs" ) var ( @@ -53,6 +60,21 @@ type TestSuite interface { Teardown() } +func SkipTestSuite(name string) bool { + testsToRun := os.Getenv("STORAGE_PROVIDER_TESTS") + if testsToRun == "" { + // run all test suites + return false + } + if strings.EqualFold(testsToRun, name) { + // this suite was requested + return false + } + + logger.Infof("skipping test suite since only %s should be tested rather than %s", testsToRun, name) + return true +} + func init() { // this default will only work if running kubernetes on the local machine baseTestDir, _ = os.Getwd() diff --git a/tests/integration/block_create_test.go b/tests/integration/block_create_test.go index 894b846a4..bc0418eab 100644 --- a/tests/integration/block_create_test.go +++ b/tests/integration/block_create_test.go @@ -33,6 +33,10 @@ import ( // Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes func TestBlockCreateSuite(t *testing.T) { + if installer.SkipTestSuite(installer.CephTestSuite) { + t.Skip() + } + s := new(BlockCreateSuite) defer func(s *BlockCreateSuite) { HandlePanics(recover(), s.op, s.T) diff --git a/tests/integration/cassandra_test.go b/tests/integration/cassandra_test.go index 76d519b59..67260f76d 100644 --- a/tests/integration/cassandra_test.go +++ b/tests/integration/cassandra_test.go @@ -52,6 +52,10 @@ type CassandraSuite struct { // TestCassandraSuite initiates the CassandraSuite func TestCassandraSuite(t *testing.T) { + if installer.SkipTestSuite(installer.CassandraTestSuite) { + t.Skip() + } + s := new(CassandraSuite) defer func(s *CassandraSuite) { r := recover() diff --git a/tests/integration/cockroachdb_test.go b/tests/integration/cockroachdb_test.go index 6fbceb704..3279ea2f7 100644 --- a/tests/integration/cockroachdb_test.go +++ b/tests/integration/cockroachdb_test.go @@ -40,6 +40,10 @@ import ( // - 25% cache, 25% maxSQLMemory // ************************************************ func TestCockroachDBSuite(t *testing.T) { + if installer.SkipTestSuite(installer.CockroachDBTestSuite) { + t.Skip() + } + s := new(CockroachDBSuite) defer func(s *CockroachDBSuite) { HandlePanics(recover(), s, s.T) diff --git a/tests/integration/edgefs_test.go b/tests/integration/edgefs_test.go index b887f6ef2..290a2dc9c 100644 --- a/tests/integration/edgefs_test.go +++ b/tests/integration/edgefs_test.go @@ -45,6 +45,10 @@ type EdgefsSuite struct { } func TestEdgefsSuite(t *testing.T) { + if installer.SkipTestSuite(installer.EdgeFSTestSuite) { + t.Skip() + } + s := new(EdgefsSuite) defer func(s *EdgefsSuite) { HandlePanics(recover(), s, s.T) diff --git a/tests/integration/helm_test.go b/tests/integration/helm_test.go index 55c7ffec8..576f1ee9e 100644 --- a/tests/integration/helm_test.go +++ b/tests/integration/helm_test.go @@ -46,6 +46,10 @@ var ( // - Create the object store via the CRD // *************************************************** func TestHelmSuite(t *testing.T) { + if installer.SkipTestSuite(installer.CephTestSuite) { + t.Skip() + } + s := new(HelmSuite) defer func(s *HelmSuite) { HandlePanics(recover(), s.op, s.T) diff --git a/tests/integration/multi_cluster_test.go b/tests/integration/multi_cluster_test.go index 5fe7717a4..7ef22baa6 100644 --- a/tests/integration/multi_cluster_test.go +++ b/tests/integration/multi_cluster_test.go @@ -45,6 +45,10 @@ import ( // - Create the object store via the CRD // ************************************************************* func TestMultiClusterDeploySuite(t *testing.T) { + if installer.SkipTestSuite(installer.CephTestSuite) { + t.Skip() + } + s := new(MultiClusterDeploySuite) defer func(s *MultiClusterDeploySuite) { HandlePanics(recover(), s.op, s.T) diff --git a/tests/integration/nfs_test.go b/tests/integration/nfs_test.go index d7b25978c..79f793d1d 100644 --- a/tests/integration/nfs_test.go +++ b/tests/integration/nfs_test.go @@ -38,6 +38,10 @@ import ( // - Mount a NFS export and write data to it and verify // ******************************************************* func TestNfsSuite(t *testing.T) { + if installer.SkipTestSuite(installer.NFSTestSuite) { + t.Skip() + } + s := new(NfsSuite) defer func(s *NfsSuite) { HandlePanics(recover(), s, s.T) diff --git a/tests/integration/smoke_test.go b/tests/integration/smoke_test.go index 1e7614167..0c8dd3635 100644 --- a/tests/integration/smoke_test.go +++ b/tests/integration/smoke_test.go @@ -50,6 +50,10 @@ import ( // - PUT/GET objects // ************************************************ func TestSmokeSuite(t *testing.T) { + if installer.SkipTestSuite(installer.CephTestSuite) { + t.Skip() + } + s := new(SmokeSuite) defer func(s *SmokeSuite) { HandlePanics(recover(), s.op, s.T) diff --git a/tests/integration/upgrade_test.go b/tests/integration/upgrade_test.go index 541b59eb5..5212fa314 100644 --- a/tests/integration/upgrade_test.go +++ b/tests/integration/upgrade_test.go @@ -40,6 +40,10 @@ import ( // - One mon in the cluster // ************************************************ func TestUpgradeSuite(t *testing.T) { + if installer.SkipTestSuite(installer.CephTestSuite) { + t.Skip() + } + s := new(UpgradeSuite) defer func(s *UpgradeSuite) { HandlePanics(recover(), s.op, s.T) diff --git a/tests/integration/zblock_mount_unmount_test.go b/tests/integration/zblock_mount_unmount_test.go index 0fefc37b1..83762190d 100644 --- a/tests/integration/zblock_mount_unmount_test.go +++ b/tests/integration/zblock_mount_unmount_test.go @@ -60,6 +60,10 @@ import ( // The error is "the server does not allow this method on the requested resource (post cephclusters.ceph.rook.io)". // Everything appears to have been cleaned up successfully in this test, so it is still unclear what is causing the issue between tests. func TestBlockMountUnMountSuite(t *testing.T) { + if installer.SkipTestSuite(installer.CephTestSuite) { + t.Skip() + } + s := new(BlockMountUnMountSuite) defer func(s *BlockMountUnMountSuite) { HandlePanics(recover(), s.op, s.T) From 807326049ee26cce92fdb4b15b155a1b53d2b698 Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 16 Jul 2019 22:17:23 -0600 Subject: [PATCH 2/3] fix the links in the PR checklist The relative links in the PR template are broken so we need to specify full paths. Signed-off-by: travisn --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a3ce745c9..7fbae61fa 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,12 +9,12 @@ Resolves # **Checklist:** -- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/v1.0/development-flow.html#submitting-a-pull-request) +- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/master/development-flow.html#submitting-a-pull-request) - [ ] Documentation has been updated, if necessary. - [ ] Unit tests have been added, if necessary. - [ ] Integration tests have been added, if necessary. - [ ] Pending release notes updated with breaking and/or notable changes, if necessary. - [ ] Upgrade from previous release is tested and upgrade user guide is updated, if necessary. - [ ] Code generation (`make codegen`) has been run to update object specifications, if necessary. -- [ ] Comments have been added or updated based on the standards set in [CONTRIBUTING.md](../CONTRIBUTING.md#comments) -- [ ] Add the flag for skipping the CI if this PR does not require a build. See [here](/INSTALL.md#skip-ci) for more details. +- [ ] Comments have been added or updated based on the standards set in [CONTRIBUTING.md](https://github.com/rook/rook/blob/master/CONTRIBUTING.md#comments) +- [ ] Add the flag for skipping the CI if this PR does not require a build. See [here](https://github.com/rook/rook/blob/master/INSTALL.md#skip-ci) for more details. From 74a3eb61800a8b7d2d01afdbafe192a8e90419ae Mon Sep 17 00:00:00 2001 From: travisn Date: Wed, 17 Jul 2019 23:23:01 -0600 Subject: [PATCH 3/3] Rename the ceph integration test entry points With the storage providers growing, we should make it clear in the source which tests belong to which storage provider. This change renames the ceph integration tests. Signed-off-by: travisn --- .../integration/{base_block_test.go => ceph_base_block_test.go} | 0 .../{base_deploy_test.go => ceph_base_deploy_test.go} | 0 tests/integration/{base_file_test.go => ceph_base_file_test.go} | 0 .../{base_object_test.go => ceph_base_object_test.go} | 0 .../{block_create_test.go => ceph_block_create_test.go} | 2 +- ...stem_mountuser_test.go => ceph_filesystem_mountuser_test.go} | 0 tests/integration/{helm_test.go => ceph_helm_test.go} | 2 +- tests/integration/{mon_test.go => ceph_mon_test.go} | 0 .../{multi_cluster_test.go => ceph_multi_cluster_test.go} | 2 +- tests/integration/{pool_test.go => ceph_pool_test.go} | 0 tests/integration/{smoke_test.go => ceph_smoke_test.go} | 2 +- tests/integration/{upgrade_test.go => ceph_upgrade_test.go} | 2 +- ..._mount_unmount_test.go => ceph_zblock_mount_unmount_test.go} | 2 +- 13 files changed, 6 insertions(+), 6 deletions(-) rename tests/integration/{base_block_test.go => ceph_base_block_test.go} (100%) rename tests/integration/{base_deploy_test.go => ceph_base_deploy_test.go} (100%) rename tests/integration/{base_file_test.go => ceph_base_file_test.go} (100%) rename tests/integration/{base_object_test.go => ceph_base_object_test.go} (100%) rename tests/integration/{block_create_test.go => ceph_block_create_test.go} (99%) rename tests/integration/{filesystem_mountuser_test.go => ceph_filesystem_mountuser_test.go} (100%) rename tests/integration/{helm_test.go => ceph_helm_test.go} (98%) rename tests/integration/{mon_test.go => ceph_mon_test.go} (100%) rename tests/integration/{multi_cluster_test.go => ceph_multi_cluster_test.go} (99%) rename tests/integration/{pool_test.go => ceph_pool_test.go} (100%) rename tests/integration/{smoke_test.go => ceph_smoke_test.go} (98%) rename tests/integration/{upgrade_test.go => ceph_upgrade_test.go} (99%) rename tests/integration/{zblock_mount_unmount_test.go => ceph_zblock_mount_unmount_test.go} (99%) diff --git a/tests/integration/base_block_test.go b/tests/integration/ceph_base_block_test.go similarity index 100% rename from tests/integration/base_block_test.go rename to tests/integration/ceph_base_block_test.go diff --git a/tests/integration/base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go similarity index 100% rename from tests/integration/base_deploy_test.go rename to tests/integration/ceph_base_deploy_test.go diff --git a/tests/integration/base_file_test.go b/tests/integration/ceph_base_file_test.go similarity index 100% rename from tests/integration/base_file_test.go rename to tests/integration/ceph_base_file_test.go diff --git a/tests/integration/base_object_test.go b/tests/integration/ceph_base_object_test.go similarity index 100% rename from tests/integration/base_object_test.go rename to tests/integration/ceph_base_object_test.go diff --git a/tests/integration/block_create_test.go b/tests/integration/ceph_block_create_test.go similarity index 99% rename from tests/integration/block_create_test.go rename to tests/integration/ceph_block_create_test.go index bc0418eab..97e65f11e 100644 --- a/tests/integration/block_create_test.go +++ b/tests/integration/ceph_block_create_test.go @@ -32,7 +32,7 @@ import ( // Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes -func TestBlockCreateSuite(t *testing.T) { +func TestCephBlockCreateSuite(t *testing.T) { if installer.SkipTestSuite(installer.CephTestSuite) { t.Skip() } diff --git a/tests/integration/filesystem_mountuser_test.go b/tests/integration/ceph_filesystem_mountuser_test.go similarity index 100% rename from tests/integration/filesystem_mountuser_test.go rename to tests/integration/ceph_filesystem_mountuser_test.go diff --git a/tests/integration/helm_test.go b/tests/integration/ceph_helm_test.go similarity index 98% rename from tests/integration/helm_test.go rename to tests/integration/ceph_helm_test.go index 576f1ee9e..ccd39fb92 100644 --- a/tests/integration/helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -45,7 +45,7 @@ var ( // Object // - Create the object store via the CRD // *************************************************** -func TestHelmSuite(t *testing.T) { +func TestCephHelmSuite(t *testing.T) { if installer.SkipTestSuite(installer.CephTestSuite) { t.Skip() } diff --git a/tests/integration/mon_test.go b/tests/integration/ceph_mon_test.go similarity index 100% rename from tests/integration/mon_test.go rename to tests/integration/ceph_mon_test.go diff --git a/tests/integration/multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go similarity index 99% rename from tests/integration/multi_cluster_test.go rename to tests/integration/ceph_multi_cluster_test.go index 7ef22baa6..43f431509 100644 --- a/tests/integration/multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -44,7 +44,7 @@ import ( // Object // - Create the object store via the CRD // ************************************************************* -func TestMultiClusterDeploySuite(t *testing.T) { +func TestCephMultiClusterDeploySuite(t *testing.T) { if installer.SkipTestSuite(installer.CephTestSuite) { t.Skip() } diff --git a/tests/integration/pool_test.go b/tests/integration/ceph_pool_test.go similarity index 100% rename from tests/integration/pool_test.go rename to tests/integration/ceph_pool_test.go diff --git a/tests/integration/smoke_test.go b/tests/integration/ceph_smoke_test.go similarity index 98% rename from tests/integration/smoke_test.go rename to tests/integration/ceph_smoke_test.go index 0c8dd3635..bd21d530c 100644 --- a/tests/integration/smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -49,7 +49,7 @@ import ( // - Create/delete users // - PUT/GET objects // ************************************************ -func TestSmokeSuite(t *testing.T) { +func TestCephSmokeSuite(t *testing.T) { if installer.SkipTestSuite(installer.CephTestSuite) { t.Skip() } diff --git a/tests/integration/upgrade_test.go b/tests/integration/ceph_upgrade_test.go similarity index 99% rename from tests/integration/upgrade_test.go rename to tests/integration/ceph_upgrade_test.go index 5212fa314..1ed43a205 100644 --- a/tests/integration/upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -39,7 +39,7 @@ import ( // Monitors // - One mon in the cluster // ************************************************ -func TestUpgradeSuite(t *testing.T) { +func TestCephUpgradeSuite(t *testing.T) { if installer.SkipTestSuite(installer.CephTestSuite) { t.Skip() } diff --git a/tests/integration/zblock_mount_unmount_test.go b/tests/integration/ceph_zblock_mount_unmount_test.go similarity index 99% rename from tests/integration/zblock_mount_unmount_test.go rename to tests/integration/ceph_zblock_mount_unmount_test.go index 83762190d..f5e62b252 100644 --- a/tests/integration/zblock_mount_unmount_test.go +++ b/tests/integration/ceph_zblock_mount_unmount_test.go @@ -59,7 +59,7 @@ import ( // There is an issue on k8s 1.7 where the CRD controller will frequently fail to create a cluster after this suite is run. // The error is "the server does not allow this method on the requested resource (post cephclusters.ceph.rook.io)". // Everything appears to have been cleaned up successfully in this test, so it is still unclear what is causing the issue between tests. -func TestBlockMountUnMountSuite(t *testing.T) { +func TestCephBlockMountUnMountSuite(t *testing.T) { if installer.SkipTestSuite(installer.CephTestSuite) { t.Skip() }