Merge pull request #3467 from travisn/provider-integration-tests

Allow executing integration tests for specific storage providers

.github/PULL_REQUEST_TEMPLATE.md

@@ -9,12 +9,12 @@ Resolves #
 
 **Checklist:**
 
-- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/v1.0/development-flow.html#submitting-a-pull-request)
+- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/master/development-flow.html#submitting-a-pull-request)
 - [ ] Documentation has been updated, if necessary.
 - [ ] Unit tests have been added, if necessary.
 - [ ] Integration tests have been added, if necessary.
 - [ ] Pending release notes updated with breaking and/or notable changes, if necessary.
 - [ ] Upgrade from previous release is tested and upgrade user guide is updated, if necessary.
 - [ ] Code generation (`make codegen`) has been run to update object specifications, if necessary.
-- [ ] Comments have been added or updated based on the standards set in [CONTRIBUTING.md](../CONTRIBUTING.md#comments)
-- [ ] Add the flag for skipping the CI if this PR does not require a build. See [here](/INSTALL.md#skip-ci) for more details.
+- [ ] Comments have been added or updated based on the standards set in [CONTRIBUTING.md](https://github.com/rook/rook/blob/master/CONTRIBUTING.md#comments)
+- [ ] Add the flag for skipping the CI if this PR does not require a build. See [here](https://github.com/rook/rook/blob/master/INSTALL.md#skip-ci) for more details.

INSTALL.md

@@ -165,3 +165,21 @@ This option should **not** be used if there are any changes to `.go` files, CI s
 
 In case there are known CI failures that are blocking a merge, project maintainers can add this flag
 to unblock the CI and merge the PR. This should be used with extreme care so regressions are not introduced.
+
+### [test storage-provider]
+
+Jenkins will only run the tests for an individual storage provider if specified. There is a keyword for each
+storage provider that has integration tests:
+- `[test cassandra]`
+- `[test ceph]`
+- `[test cockroachdb]`
+- `[test edgefs]`
+- `[test nfs]`
+
+Since the majority of effort for a PR generally focuses on a single storage provider it will save time
+in the CI if you only trigger the tests for your provider. If this key phrase is not found in the PR,
+all tests will be executed.
+
+These options should **not** be used if there are any changes to code which is shared with other
+storage providers. If there is any risk of affecting another storage provider, all tests should
+be executed in the CI.

Jenkinsfile

@@ -27,12 +27,22 @@ pipeline {
                     if (body.contains("[skip tests]")) {
                          env.shouldTest = "false"
                     }
-                    if (body.contains("[smoke only]")) {
-                          env.smokeOnly = "true"
-                    }
                     if (body.contains("[all logs]")) {
                           env.getLogs = "all"
                     }
+                    // extract which specific storage provider to test
+                    if (body.contains("[test cassandra]")) {
+                        env.testProvider = "cassandra"
+                    } else if (body.contains("[test ceph]")) {
+                        env.testProvider = "ceph"
+                    } else if (body.contains("[test cockroachdb]")) {
+                        env.testProvider = "cockroachdb"
+                    } else if (body.contains("[test edgefs]")) {
+                        env.testProvider = "edgefs"
+                    } else if (body.contains("[test nfs]")) {
+                        env.testProvider = "nfs"
+                    }
+                    echo ("integration test provider: ${env.testProvider}")
                 }
             }
         }
@@ -159,24 +169,14 @@ def RunIntegrationTest(k, v) {
                           export KUBECONFIG=$HOME/admin.conf
                           tests/scripts/helm.sh up'''
                     try{
-                        if ("${env.smokeOnly}" == "true") {
-                            echo "Running Smoke Tests"
-                            sh '''#!/bin/bash
-                                  set -o pipefail
-                                  export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \
-                                      KUBECONFIG=$HOME/admin.conf
-                                  kubectl config view
-                                  _output/tests/linux_amd64/integration -test.v -test.timeout 1800s -test.run SmokeSuite --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log'''
-                        }
-                        else {
                         echo "Running full regression"
                         sh '''#!/bin/bash
                               set -o pipefail
                               export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \
-                                  KUBECONFIG=$HOME/admin.conf
+                                  KUBECONFIG=$HOME/admin.conf \
+                                  STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+'''
                               kubectl config view
                               _output/tests/linux_amd64/integration -test.v -test.timeout 7200s --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log'''
-                         }
                     }
                     finally{
                         sh "journalctl -u kubelet > _output/tests/kubelet_${v}.log"

PendingReleaseNotes.md

@@ -9,6 +9,7 @@ an example usage
 - Added K8s 1.15 to the test matrix and removed K8s 1.10 from the test matrix.
 - OwnerReferences are created with the fully qualified `apiVersion` such that the references will work properly on OpenShift.
 - Linear disk device can now be used for Ceph OSDs.
+- The integration tests can be triggered for specific storage providers rather than always running all tests. See the [dev guide](INSTALL.md#test-storage-provider) for more details.
 
 ### Ceph
 

tests/framework/installer/installer.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/coreos/pkg/capnslog"
@@ -33,6 +34,12 @@ const (
 	VersionMaster = "master"
 	// Version tag for Rook v0.9
 	Version1_0 = "v1.0.1"
+	// test suite names
+	CassandraTestSuite   = "cassandra"
+	CephTestSuite        = "ceph"
+	CockroachDBTestSuite = "cockroachdb"
+	EdgeFSTestSuite      = "edgefs"
+	NFSTestSuite         = "nfs"
 )
 
 var (
@@ -53,6 +60,21 @@ type TestSuite interface {
 	Teardown()
 }
 
+func SkipTestSuite(name string) bool {
+	testsToRun := os.Getenv("STORAGE_PROVIDER_TESTS")
+	if testsToRun == "" {
+		// run all test suites
+		return false
+	}
+	if strings.EqualFold(testsToRun, name) {
+		// this suite was requested
+		return false
+	}
+
+	logger.Infof("skipping test suite since only %s should be tested rather than %s", testsToRun, name)
+	return true
+}
+
 func init() {
 	// this default will only work if running kubernetes on the local machine
 	baseTestDir, _ = os.Getwd()

tests/integration/cassandra_test.go

@@ -52,6 +52,10 @@ type CassandraSuite struct {
 
 // TestCassandraSuite initiates the CassandraSuite
 func TestCassandraSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CassandraTestSuite) {
+		t.Skip()
+	}
+
 	s := new(CassandraSuite)
 	defer func(s *CassandraSuite) {
 		r := recover()

tests/integration/ceph_block_create_test.go

@@ -32,7 +32,11 @@ import (
 
 // Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes
 
-func TestBlockCreateSuite(t *testing.T) {
+func TestCephBlockCreateSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(BlockCreateSuite)
 	defer func(s *BlockCreateSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/ceph_helm_test.go

@@ -45,7 +45,11 @@ var (
 // Object
 // - Create the object store via the CRD
 // ***************************************************
-func TestHelmSuite(t *testing.T) {
+func TestCephHelmSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(HelmSuite)
 	defer func(s *HelmSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/ceph_multi_cluster_test.go

@@ -44,7 +44,11 @@ import (
 // Object
 // - Create the object store via the CRD
 // *************************************************************
-func TestMultiClusterDeploySuite(t *testing.T) {
+func TestCephMultiClusterDeploySuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(MultiClusterDeploySuite)
 	defer func(s *MultiClusterDeploySuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/ceph_smoke_test.go

@@ -49,7 +49,11 @@ import (
 // - Create/delete users
 // - PUT/GET objects
 // ************************************************
-func TestSmokeSuite(t *testing.T) {
+func TestCephSmokeSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(SmokeSuite)
 	defer func(s *SmokeSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/ceph_upgrade_test.go

@@ -39,7 +39,11 @@ import (
 // Monitors
 // - One mon in the cluster
 // ************************************************
-func TestUpgradeSuite(t *testing.T) {
+func TestCephUpgradeSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(UpgradeSuite)
 	defer func(s *UpgradeSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/ceph_zblock_mount_unmount_test.go

@@ -59,7 +59,11 @@ import (
 // There is an issue on k8s 1.7 where the CRD controller will frequently fail to create a cluster after this suite is run.
 // The error is "the server does not allow this method on the requested resource (post cephclusters.ceph.rook.io)".
 // Everything appears to have been cleaned up successfully in this test, so it is still unclear what is causing the issue between tests.
-func TestBlockMountUnMountSuite(t *testing.T) {
+func TestCephBlockMountUnMountSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(BlockMountUnMountSuite)
 	defer func(s *BlockMountUnMountSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/cockroachdb_test.go

@@ -40,6 +40,10 @@ import (
 //   - 25% cache, 25% maxSQLMemory
 // ************************************************
 func TestCockroachDBSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CockroachDBTestSuite) {
+		t.Skip()
+	}
+
 	s := new(CockroachDBSuite)
 	defer func(s *CockroachDBSuite) {
 		HandlePanics(recover(), s, s.T)

tests/integration/edgefs_test.go

@@ -45,6 +45,10 @@ type EdgefsSuite struct {
 }
 
 func TestEdgefsSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.EdgeFSTestSuite) {
+		t.Skip()
+	}
+
 	s := new(EdgefsSuite)
 	defer func(s *EdgefsSuite) {
 		HandlePanics(recover(), s, s.T)

tests/integration/nfs_test.go

@@ -38,6 +38,10 @@ import (
 //   - Mount a NFS export and write data to it and verify
 // *******************************************************
 func TestNfsSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.NFSTestSuite) {
+		t.Skip()
+	}
+
 	s := new(NfsSuite)
 	defer func(s *NfsSuite) {
 		HandlePanics(recover(), s, s.T)