diff --git a/Jenkinsfile b/Jenkinsfile index 8e6e12abc..cf305b6c8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -197,10 +197,14 @@ def RunIntegrationTest(k, v) { set -o pipefail export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \ KUBECONFIG=$HOME/admin.conf \ + TEST_HELM_PATH=/tmp/rook-tests-scripts-helm/linux-amd64/helm \ + TEST_ENV_NAME='''+"${k}"+''' \ + TEST_BASE_DIR="WORKING_DIR" \ + TEST_LOG_COLLECTION_LEVEL='''+"${env.getLogs}"+''' \ STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+''' \ TEST_ARGUMENTS='''+"${env.testArgs}"+''' kubectl config view - _output/tests/linux_amd64/integration -test.v -test.timeout 7200s --base_test_dir "" --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log''' + _output/tests/linux_amd64/integration -test.v -test.timeout 7200s 2>&1 | tee _output/tests/integrationTests.log''' } finally{ sh "journalctl -u kubelet > _output/tests/kubelet_${v}.log" diff --git a/tests/README.md b/tests/README.md index a71fadb75..fa4537b13 100644 --- a/tests/README.md +++ b/tests/README.md @@ -71,14 +71,14 @@ Use [helm.sh](/tests/scripts/helm.sh) to install Helm and set up Rook charts def these scripts should be run from project root. e.g., `tests/script/kubeadm.sh up`. **NOTE**: If Helm is not available in your `PATH`, Helm will be downloaded to a temporary directory (`/tmp/rook-tests-scripts-helm`) and used from that directory. -The temporary directory the Helm binary is in needs to be given to the `integration` binary calls. The flag for that is `--helm=HELM_BINARY`, e.g., `--helm=/tmp/rook-tests-scripts-helm/linux-amd64/helm`. ## Run Tests From the project root do the following: -#### 1. Build rook: + +### 1. Build rook: Run `make build` -#### 2. Start Kubernetes +### 2. Start Kubernetes Using one of the following: - Using Kubeadm @@ -94,24 +94,22 @@ tests/scripts/minikube.sh helm tests/scripts/helm.sh up ``` -#### 3. Run integration tests: -Integration tests can be run using tests binary `_output/tests/${platform}/integration` that is generated during build time e.g.: -``` -_output/tests/${platform}/integration -test.v -``` +### 3. Run integration tests: -### Running Tests with parameters. -#### To run all integration tests run +Some settings are available to run the tests under different environments. The settings are all configured with environment variables. +See [environment.go](/tests/framework/installer/environment.go) for the available environment variables. + +To run all integration tests: ``` go test -v -timeout 1800s github.com/rook/rook/tests/integration ``` -#### To run a specific suite (uses regex) +To run a specific suite (uses regex): ``` go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration ``` -#### To run specific tests inside a suite: +To run specific tests inside a suite: ``` go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration -testify.m TestRookClusterInstallation_SmokeTest ``` diff --git a/tests/framework/installer/cassandra_installer.go b/tests/framework/installer/cassandra_installer.go index c9b73aca8..32587c2d4 100644 --- a/tests/framework/installer/cassandra_installer.go +++ b/tests/framework/installer/cassandra_installer.go @@ -162,10 +162,10 @@ func (ci *CassandraInstaller) UninstallCassandra(systemNamespace string, namespa } func (ci *CassandraInstaller) GatherAllCassandraLogs(systemNamespace, namespace, testName string) { - if !ci.T().Failed() && Env.Logs != "all" { + if !ci.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from Cassandra Cluster %s", namespace) - ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType) - ci.k8sHelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) + ci.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index b50099ad0..55f80a01b 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -333,10 +333,11 @@ func (h *CephInstaller) GetRookExternalClusterMonSecret(namespace string) (*v1.S } func (h *CephInstaller) initTestDir(namespace string) (string, error) { - h.hostPathToDelete = path.Join(baseTestDir, "rook-test") + h.hostPathToDelete = path.Join(baseTestDir(), "rook-test") testDir := path.Join(h.hostPathToDelete, namespace) - if createBaseTestDir { + // skip the test dir creation if we are not running under "/data" + if baseTestDir() != "/data" { // Create the test dir on the local host if err := os.MkdirAll(testDir, 0777); err != nil { return "", err @@ -401,7 +402,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") { logger.Error("rook-ceph-operator is not running") - h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", Env.HostType) + h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", testEnvName()) logger.Error("rook-ceph-operator is not Running, abort!") return false, err } @@ -664,21 +665,21 @@ func (h *CephInstaller) verifyDirCleanup(node, dir string) error { } func (h *CephInstaller) CollectOperatorLog(suiteName, testName, namespace string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } name := fmt.Sprintf("%s_%s", suiteName, testName) - h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, Env.HostType) + h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, testEnvName()) } func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("gathering all logs from the test") for _, namespace := range namespaces { - h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType) - h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, Env.HostType) + h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) + h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, testEnvName()) } } @@ -703,7 +704,7 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH h := &CephInstaller{ Manifests: NewCephManifests(rookVersion), k8shelper: k8shelp, - helmHelper: utils.NewHelmHelper(Env.Helm), + helmHelper: utils.NewHelmHelper(testHelmPath()), useHelm: useHelm, k8sVersion: version.String(), CephVersion: cephVersion, diff --git a/tests/framework/installer/cockroachdb_installer.go b/tests/framework/installer/cockroachdb_installer.go index 9b91f5c53..bf9b8166c 100644 --- a/tests/framework/installer/cockroachdb_installer.go +++ b/tests/framework/installer/cockroachdb_installer.go @@ -150,10 +150,10 @@ func (h *CockroachDBInstaller) UninstallCockroachDB(systemNamespace, namespace s } func (h *CockroachDBInstaller) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType) - h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/framework/installer/edgefs_installer.go b/tests/framework/installer/edgefs_installer.go index 6c4d363da..fe7492545 100644 --- a/tests/framework/installer/edgefs_installer.go +++ b/tests/framework/installer/edgefs_installer.go @@ -152,10 +152,10 @@ func (h *EdgefsInstaller) UninstallEdgefs(systemNamespace, namespace string) { } func (h *EdgefsInstaller) GatherAllEdgefsLogs(systemNamespace, namespace, testName string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from edgefs cluster %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType) - h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/framework/installer/environment.go b/tests/framework/installer/environment.go index b02bd0e2f..e593af28d 100644 --- a/tests/framework/installer/environment.go +++ b/tests/framework/installer/environment.go @@ -17,37 +17,43 @@ limitations under the License. package installer import ( - "flag" + "os" ) -// EnvironmentManifest contains information about system under test -type EnvironmentManifest struct { - HostType string - Helm string - RookImageName string - ToolboxImageName string - BaseTestDir string - LoadVolumeNumber int - LoadConcurrentRuns int - LoadTime int - LoadSize string - EnableChaos bool - Logs string +// testEnvName gets the name of the test environment. In the CI it is "aws_1.18.x" or similar. +func testEnvName() string { + return getEnvVarWithDefault("TEST_ENV_NAME", "localhost") } -var Env EnvironmentManifest - -func init() { - Env = EnvironmentManifest{} - flag.StringVar(&Env.HostType, "host_type", "localhost", "Host where tests are run eg - localhost, GCE or AWS") - flag.StringVar(&Env.Helm, "helm", "helm", "Path to helm binary") - flag.StringVar(&Env.RookImageName, "rook_image", "rook/ceph", "Docker image name for the rook container to install, must be in docker hub or local environment") - flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/ceph", "Docker image name of the toolbox container to install, must be in docker hub or local environment") - flag.StringVar(&Env.BaseTestDir, "base_test_dir", "/data", "Base test directory, for use only when kubernetes master is running on localhost") - flag.IntVar(&Env.LoadConcurrentRuns, "load_parallel_runs", 20, "number of routines for load test") - flag.IntVar(&Env.LoadVolumeNumber, "load_volumes", 1, "number of volumes(file,object or block) to be created for load test") - flag.IntVar(&Env.LoadTime, "load_time", 1800, "number of seconds each thread perform operations for") - flag.StringVar(&Env.LoadSize, "load_size", "medium", "load size for each thread performing operations - small,medium or large.") - flag.BoolVar(&Env.EnableChaos, "enable_chaos", false, "used to determine if random pods in a namespace are to be killed during load test.") - flag.StringVar(&Env.Logs, "logs", "", "Gather rook logs, eg - all") +// testHelmPath gets the helm path +func testHelmPath() string { + return getEnvVarWithDefault("TEST_HELM_PATH", "helm") +} + +// TestLogCollectionLevel gets whether to collect all logs +func TestLogCollectionLevel() string { + return getEnvVarWithDefault("TEST_LOG_COLLECTION_LEVEL", "") +} + +// testStorageProvider gets the storage provider for which tests should be run +func testStorageProvider() string { + return getEnvVarWithDefault("STORAGE_PROVIDER_TESTS", "") +} + +// baseTestDir gets the base test directory +func baseTestDir() string { + // If the base test directory is actively set to empty (as in CI), we use the current working directory. + val := getEnvVarWithDefault("TEST_BASE_DIR", "/data") + if val == "WORKING_DIR" { + val, _ = os.Getwd() + } + return val +} + +func getEnvVarWithDefault(env, defaultValue string) string { + val := os.Getenv(env) + if val == "" { + return defaultValue + } + return val } diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index 637f7feff..50780753b 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -18,7 +18,6 @@ package installer import ( "fmt" - "os" "strings" "testing" @@ -41,10 +40,6 @@ const ( ) var ( - // ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. ** - baseTestDir string - createBaseTestDir = true - // ** end of Variables to modify logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer") createArgs = []string{"create", "-f"} createFromStdinArgs = append(createArgs, "-") @@ -58,7 +53,7 @@ type TestSuite interface { } func SkipTestSuite(name string) bool { - testsToRun := os.Getenv("STORAGE_PROVIDER_TESTS") + testsToRun := testStorageProvider() // jenkins passes "null" if the env var is not set. if testsToRun == "" || testsToRun == "null" { // run all test suites @@ -73,17 +68,6 @@ func SkipTestSuite(name string) bool { return true } -func init() { - // If the base test directory is actively set to empty (as in CI), we use the current working directory. - baseTestDir = Env.BaseTestDir - if baseTestDir == "" { - baseTestDir, _ = os.Getwd() - } - if baseTestDir == "/data" { - createBaseTestDir = false - } -} - func SystemNamespace(namespace string) string { return fmt.Sprintf("%s-system", namespace) } diff --git a/tests/framework/installer/nfs_installer.go b/tests/framework/installer/nfs_installer.go index e4f8f0b02..8a954f6d1 100644 --- a/tests/framework/installer/nfs_installer.go +++ b/tests/framework/installer/nfs_installer.go @@ -190,10 +190,10 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) { // GatherAllNFSServerLogs gathers all NFS Server logs func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from NFSServer %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType) - h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/framework/installer/provisioners.go b/tests/framework/installer/provisioners.go index b96bcd9fc..203e554b0 100644 --- a/tests/framework/installer/provisioners.go +++ b/tests/framework/installer/provisioners.go @@ -66,7 +66,7 @@ func InstallHostPathProvisioner(k8shelper *utils.K8sHelper) error { err = k8shelper.WaitForLabeledPodsToRun("k8s-app=hostpath-provisioner", "kube-system") if err != nil { logger.Errorf("hostpath provisioner pod is not running: %+v", err) - k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", Env.HostType) + k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", testEnvName()) k8shelper.PrintStorageClasses(true /*detailed*/) return err } diff --git a/tests/framework/installer/yugabytedb_installer.go b/tests/framework/installer/yugabytedb_installer.go index 586c27dfb..fe64524a5 100644 --- a/tests/framework/installer/yugabytedb_installer.go +++ b/tests/framework/installer/yugabytedb_installer.go @@ -161,10 +161,10 @@ func (y *YugabyteDBInstaller) RemoveAllYugabyteDBResources(systemNS, namespace s } func (y *YugabyteDBInstaller) GatherAllLogs(systemNS, namespace, testName string) { - if !y.T().Failed() && Env.Logs != "all" { + if !y.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from yugabytedb cluster %s", namespace) - y.k8sHelper.GetLogsFromNamespace(systemNS, testName, Env.HostType) - y.k8sHelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + y.k8sHelper.GetLogsFromNamespace(systemNS, testName, testEnvName()) + y.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 97f268c01..a89650222 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -213,7 +213,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) { // Gather logs before Ceph upgrade to help with debugging - if installer.Env.Logs == "all" { + if installer.TestLogCollectionLevel() == "all" { s.k8sh.PrintPodDescribe(s.namespace) } n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix