tests: convert cli params to env vars

The command line flags parsing inside an init() method was causing
issues at runtime when trying to add certain new features, which
is a limitation of the flag parsing. We now move away from using
any command line flags in the tests and only use environment variables
to simplify adding new parameters to the tests.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>

Jenkinsfile

@@ -197,10 +197,14 @@ def RunIntegrationTest(k, v) {
                               set -o pipefail
                               export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \
                                   KUBECONFIG=$HOME/admin.conf \
+                                  TEST_HELM_PATH=/tmp/rook-tests-scripts-helm/linux-amd64/helm \
+                                  TEST_ENV_NAME='''+"${k}"+''' \
+                                  TEST_BASE_DIR="WORKING_DIR" \
+                                  TEST_LOG_COLLECTION_LEVEL='''+"${env.getLogs}"+''' \
                                   STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+''' \
                                   TEST_ARGUMENTS='''+"${env.testArgs}"+'''
                               kubectl config view
-                              _output/tests/linux_amd64/integration -test.v -test.timeout 7200s --base_test_dir "" --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log'''
+                              _output/tests/linux_amd64/integration -test.v -test.timeout 7200s 2>&1 | tee _output/tests/integrationTests.log'''
                     }
                     finally{
                         sh "journalctl -u kubelet > _output/tests/kubelet_${v}.log"

tests/README.md

@@ -71,14 +71,14 @@ Use [helm.sh](/tests/scripts/helm.sh) to install Helm and set up Rook charts def
 these scripts should be run from project root. e.g., `tests/script/kubeadm.sh up`.
 
 **NOTE**: If Helm is not available in your `PATH`, Helm will be downloaded to a temporary directory (`/tmp/rook-tests-scripts-helm`) and used from that directory.
-The temporary directory the Helm binary is in needs to be given to the `integration` binary calls. The flag for that is `--helm=HELM_BINARY`, e.g., `--helm=/tmp/rook-tests-scripts-helm/linux-amd64/helm`.
 
 ## Run Tests
 From the project root do the following:
-#### 1. Build rook:
+
+### 1. Build rook:
 Run `make build`
 
-#### 2. Start Kubernetes
+### 2. Start Kubernetes
 Using one of the following:
 
 - Using Kubeadm
@@ -94,24 +94,22 @@ tests/scripts/minikube.sh helm
 tests/scripts/helm.sh up
 ```
 
-#### 3. Run integration tests:
-Integration tests can be run using tests binary `_output/tests/${platform}/integration` that is generated during build time e.g.:
-```
-_output/tests/${platform}/integration -test.v
-```
+### 3. Run integration tests:
 
-### Running Tests with parameters.
-#### To run all integration tests run
+Some settings are available to run the tests under different environments. The settings are all configured with environment variables.
+See [environment.go](/tests/framework/installer/environment.go) for the available environment variables.
+
+To run all integration tests:
 ```
 go test -v -timeout 1800s github.com/rook/rook/tests/integration
 ```
 
-#### To run a specific suite (uses regex)
+To run a specific suite (uses regex):
 ```
 go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration
 ```
 
-#### To run specific tests inside a suite:
+To run specific tests inside a suite:
 ```
 go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration -testify.m TestRookClusterInstallation_SmokeTest
 ```

tests/framework/installer/cassandra_installer.go

@@ -162,10 +162,10 @@ func (ci *CassandraInstaller) UninstallCassandra(systemNamespace string, namespa
 }
 
 func (ci *CassandraInstaller) GatherAllCassandraLogs(systemNamespace, namespace, testName string) {
-	if !ci.T().Failed() && Env.Logs != "all" {
+	if !ci.T().Failed() && TestLogCollectionLevel() != "all" {
 		return
 	}
 	logger.Infof("Gathering all logs from Cassandra Cluster %s", namespace)
-	ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType)
-	ci.k8sHelper.GetLogsFromNamespace(namespace, testName, Env.HostType)
+	ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName())
+	ci.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName())
 }

tests/framework/installer/ceph_installer.go

@@ -333,10 +333,11 @@ func (h *CephInstaller) GetRookExternalClusterMonSecret(namespace string) (*v1.S
 }
 
 func (h *CephInstaller) initTestDir(namespace string) (string, error) {
-	h.hostPathToDelete = path.Join(baseTestDir, "rook-test")
+	h.hostPathToDelete = path.Join(baseTestDir(), "rook-test")
 	testDir := path.Join(h.hostPathToDelete, namespace)
 
-	if createBaseTestDir {
+	// skip the test dir creation if we are not running under "/data"
+	if baseTestDir() != "/data" {
 		// Create the test dir on the local host
 		if err := os.MkdirAll(testDir, 0777); err != nil {
 			return "", err
@@ -401,7 +402,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st
 	}
 	if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") {
 		logger.Error("rook-ceph-operator is not running")
-		h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", Env.HostType)
+		h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", testEnvName())
 		logger.Error("rook-ceph-operator is not Running, abort!")
 		return false, err
 	}
@@ -664,21 +665,21 @@ func (h *CephInstaller) verifyDirCleanup(node, dir string) error {
 }
 
 func (h *CephInstaller) CollectOperatorLog(suiteName, testName, namespace string) {
-	if !h.T().Failed() && Env.Logs != "all" {
+	if !h.T().Failed() && TestLogCollectionLevel() != "all" {
 		return
 	}
 	name := fmt.Sprintf("%s_%s", suiteName, testName)
-	h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, Env.HostType)
+	h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, testEnvName())
 }
 
 func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) {
-	if !h.T().Failed() && Env.Logs != "all" {
+	if !h.T().Failed() && TestLogCollectionLevel() != "all" {
 		return
 	}
 	logger.Infof("gathering all logs from the test")
 	for _, namespace := range namespaces {
-		h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType)
-		h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, Env.HostType)
+		h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName())
+		h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, testEnvName())
 	}
 }
 
@@ -703,7 +704,7 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH
 	h := &CephInstaller{
 		Manifests:       NewCephManifests(rookVersion),
 		k8shelper:       k8shelp,
-		helmHelper:      utils.NewHelmHelper(Env.Helm),
+		helmHelper:      utils.NewHelmHelper(testHelmPath()),
 		useHelm:         useHelm,
 		k8sVersion:      version.String(),
 		CephVersion:     cephVersion,

tests/framework/installer/cockroachdb_installer.go

@@ -150,10 +150,10 @@ func (h *CockroachDBInstaller) UninstallCockroachDB(systemNamespace, namespace s
 }
 
 func (h *CockroachDBInstaller) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) {
-	if !h.T().Failed() && Env.Logs != "all" {
+	if !h.T().Failed() && TestLogCollectionLevel() != "all" {
 		return
 	}
 	logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace)
-	h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType)
-	h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType)
+	h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName())
+	h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName())
 }

tests/framework/installer/edgefs_installer.go

@@ -152,10 +152,10 @@ func (h *EdgefsInstaller) UninstallEdgefs(systemNamespace, namespace string) {
 }
 
 func (h *EdgefsInstaller) GatherAllEdgefsLogs(systemNamespace, namespace, testName string) {
-	if !h.T().Failed() && Env.Logs != "all" {
+	if !h.T().Failed() && TestLogCollectionLevel() != "all" {
 		return
 	}
 	logger.Infof("Gathering all logs from edgefs cluster %s", namespace)
-	h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType)
-	h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType)
+	h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName())
+	h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName())
 }

tests/framework/installer/environment.go

@@ -17,37 +17,43 @@ limitations under the License.
 package installer
 
 import (
-	"flag"
+	"os"
 )
 
-// EnvironmentManifest contains information about system under test
-type EnvironmentManifest struct {
-	HostType           string
-	Helm               string
-	RookImageName      string
-	ToolboxImageName   string
-	BaseTestDir        string
-	LoadVolumeNumber   int
-	LoadConcurrentRuns int
-	LoadTime           int
-	LoadSize           string
-	EnableChaos        bool
-	Logs               string
+// testEnvName gets the name of the test environment. In the CI it is "aws_1.18.x" or similar.
+func testEnvName() string {
+	return getEnvVarWithDefault("TEST_ENV_NAME", "localhost")
 }
 
-var Env EnvironmentManifest
-
-func init() {
-	Env = EnvironmentManifest{}
-	flag.StringVar(&Env.HostType, "host_type", "localhost", "Host where tests are run eg - localhost, GCE or AWS")
-	flag.StringVar(&Env.Helm, "helm", "helm", "Path to helm binary")
-	flag.StringVar(&Env.RookImageName, "rook_image", "rook/ceph", "Docker image name for the rook container to install, must be in docker hub or local environment")
-	flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/ceph", "Docker image name of the toolbox container to install, must be in docker hub or local environment")
-	flag.StringVar(&Env.BaseTestDir, "base_test_dir", "/data", "Base test directory, for use only when kubernetes master is running on localhost")
-	flag.IntVar(&Env.LoadConcurrentRuns, "load_parallel_runs", 20, "number of routines for load test")
-	flag.IntVar(&Env.LoadVolumeNumber, "load_volumes", 1, "number of volumes(file,object or block) to be created for load test")
-	flag.IntVar(&Env.LoadTime, "load_time", 1800, "number of seconds each thread perform operations for")
-	flag.StringVar(&Env.LoadSize, "load_size", "medium", "load size for each thread performing operations - small,medium or large.")
-	flag.BoolVar(&Env.EnableChaos, "enable_chaos", false, "used to determine if random pods in a namespace are to be killed during load test.")
-	flag.StringVar(&Env.Logs, "logs", "", "Gather rook logs, eg - all")
+// testHelmPath gets the helm path
+func testHelmPath() string {
+	return getEnvVarWithDefault("TEST_HELM_PATH", "helm")
+}
+
+// TestLogCollectionLevel gets whether to collect all logs
+func TestLogCollectionLevel() string {
+	return getEnvVarWithDefault("TEST_LOG_COLLECTION_LEVEL", "")
+}
+
+// testStorageProvider gets the storage provider for which tests should be run
+func testStorageProvider() string {
+	return getEnvVarWithDefault("STORAGE_PROVIDER_TESTS", "")
+}
+
+// baseTestDir gets the base test directory
+func baseTestDir() string {
+	// If the base test directory is actively set to empty (as in CI), we use the current working directory.
+	val := getEnvVarWithDefault("TEST_BASE_DIR", "/data")
+	if val == "WORKING_DIR" {
+		val, _ = os.Getwd()
+	}
+	return val
+}
+
+func getEnvVarWithDefault(env, defaultValue string) string {
+	val := os.Getenv(env)
+	if val == "" {
+		return defaultValue
+	}
+	return val
 }

tests/framework/installer/installer.go

@@ -18,7 +18,6 @@ package installer
 
 import (
 	"fmt"
-	"os"
 	"strings"
 	"testing"
 
@@ -41,10 +40,6 @@ const (
 )
 
 var (
-	// ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. **
-	baseTestDir       string
-	createBaseTestDir = true
-	// ** end of Variables to modify
 	logger              = capnslog.NewPackageLogger("github.com/rook/rook", "installer")
 	createArgs          = []string{"create", "-f"}
 	createFromStdinArgs = append(createArgs, "-")
@@ -58,7 +53,7 @@ type TestSuite interface {
 }
 
 func SkipTestSuite(name string) bool {
-	testsToRun := os.Getenv("STORAGE_PROVIDER_TESTS")
+	testsToRun := testStorageProvider()
 	// jenkins passes "null" if the env var is not set.
 	if testsToRun == "" || testsToRun == "null" {
 		// run all test suites
@@ -73,17 +68,6 @@ func SkipTestSuite(name string) bool {
 	return true
 }
 
-func init() {
-	// If the base test directory is actively set to empty (as in CI), we use the current working directory.
-	baseTestDir = Env.BaseTestDir
-	if baseTestDir == "" {
-		baseTestDir, _ = os.Getwd()
-	}
-	if baseTestDir == "/data" {
-		createBaseTestDir = false
-	}
-}
-
 func SystemNamespace(namespace string) string {
 	return fmt.Sprintf("%s-system", namespace)
 }

tests/framework/installer/nfs_installer.go

@@ -190,10 +190,10 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) {
 
 // GatherAllNFSServerLogs gathers all NFS Server logs
 func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) {
-	if !h.T().Failed() && Env.Logs != "all" {
+	if !h.T().Failed() && TestLogCollectionLevel() != "all" {
 		return
 	}
 	logger.Infof("Gathering all logs from NFSServer %s", namespace)
-	h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType)
-	h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType)
+	h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName())
+	h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName())
 }

tests/framework/installer/provisioners.go

@@ -66,7 +66,7 @@ func InstallHostPathProvisioner(k8shelper *utils.K8sHelper) error {
 	err = k8shelper.WaitForLabeledPodsToRun("k8s-app=hostpath-provisioner", "kube-system")
 	if err != nil {
 		logger.Errorf("hostpath provisioner pod is not running: %+v", err)
-		k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", Env.HostType)
+		k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", testEnvName())
 		k8shelper.PrintStorageClasses(true /*detailed*/)
 		return err
 	}

tests/framework/installer/yugabytedb_installer.go

@@ -161,10 +161,10 @@ func (y *YugabyteDBInstaller) RemoveAllYugabyteDBResources(systemNS, namespace s
 }
 
 func (y *YugabyteDBInstaller) GatherAllLogs(systemNS, namespace, testName string) {
-	if !y.T().Failed() && Env.Logs != "all" {
+	if !y.T().Failed() && TestLogCollectionLevel() != "all" {
 		return
 	}
 	logger.Infof("Gathering all logs from yugabytedb cluster %s", namespace)
-	y.k8sHelper.GetLogsFromNamespace(systemNS, testName, Env.HostType)
-	y.k8sHelper.GetLogsFromNamespace(namespace, testName, Env.HostType)
+	y.k8sHelper.GetLogsFromNamespace(systemNS, testName, testEnvName())
+	y.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName())
 }

tests/integration/ceph_upgrade_test.go

@@ -213,7 +213,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
 
 func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) {
 	// Gather logs before Ceph upgrade to help with debugging
-	if installer.Env.Logs == "all" {
+	if installer.TestLogCollectionLevel() == "all" {
 		s.k8sh.PrintPodDescribe(s.namespace)
 	}
 	n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix