From c4e712bbd2f8c44c15adee0d754f2da233c5042d Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Mon, 13 Apr 2020 16:10:39 -0600 Subject: [PATCH] tests: convert cli params to env vars The command line flags parsing inside an init() method was causing issues at runtime when trying to add certain new features, which is a limitation of the flag parsing. We now move away from using any command line flags in the tests and only use environment variables to simplify adding new parameters to the tests. Signed-off-by: Travis Nielsen --- Jenkinsfile | 6 +- tests/README.md | 22 +++---- .../installer/cassandra_installer.go | 6 +- tests/framework/installer/ceph_installer.go | 19 +++--- .../installer/cockroachdb_installer.go | 6 +- tests/framework/installer/edgefs_installer.go | 6 +- tests/framework/installer/environment.go | 64 ++++++++++--------- tests/framework/installer/installer.go | 18 +----- tests/framework/installer/nfs_installer.go | 6 +- tests/framework/installer/provisioners.go | 2 +- .../installer/yugabytedb_installer.go | 6 +- tests/integration/ceph_upgrade_test.go | 2 +- 12 files changed, 78 insertions(+), 85 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8e6e12abc..cf305b6c8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -197,10 +197,14 @@ def RunIntegrationTest(k, v) { set -o pipefail export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \ KUBECONFIG=$HOME/admin.conf \ + TEST_HELM_PATH=/tmp/rook-tests-scripts-helm/linux-amd64/helm \ + TEST_ENV_NAME='''+"${k}"+''' \ + TEST_BASE_DIR="WORKING_DIR" \ + TEST_LOG_COLLECTION_LEVEL='''+"${env.getLogs}"+''' \ STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+''' \ TEST_ARGUMENTS='''+"${env.testArgs}"+''' kubectl config view - _output/tests/linux_amd64/integration -test.v -test.timeout 7200s --base_test_dir "" --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log''' + _output/tests/linux_amd64/integration -test.v -test.timeout 7200s 2>&1 | tee _output/tests/integrationTests.log''' } finally{ sh "journalctl -u kubelet > _output/tests/kubelet_${v}.log" diff --git a/tests/README.md b/tests/README.md index a71fadb75..fa4537b13 100644 --- a/tests/README.md +++ b/tests/README.md @@ -71,14 +71,14 @@ Use [helm.sh](/tests/scripts/helm.sh) to install Helm and set up Rook charts def these scripts should be run from project root. e.g., `tests/script/kubeadm.sh up`. **NOTE**: If Helm is not available in your `PATH`, Helm will be downloaded to a temporary directory (`/tmp/rook-tests-scripts-helm`) and used from that directory. -The temporary directory the Helm binary is in needs to be given to the `integration` binary calls. The flag for that is `--helm=HELM_BINARY`, e.g., `--helm=/tmp/rook-tests-scripts-helm/linux-amd64/helm`. ## Run Tests From the project root do the following: -#### 1. Build rook: + +### 1. Build rook: Run `make build` -#### 2. Start Kubernetes +### 2. Start Kubernetes Using one of the following: - Using Kubeadm @@ -94,24 +94,22 @@ tests/scripts/minikube.sh helm tests/scripts/helm.sh up ``` -#### 3. Run integration tests: -Integration tests can be run using tests binary `_output/tests/${platform}/integration` that is generated during build time e.g.: -``` -_output/tests/${platform}/integration -test.v -``` +### 3. Run integration tests: -### Running Tests with parameters. -#### To run all integration tests run +Some settings are available to run the tests under different environments. The settings are all configured with environment variables. +See [environment.go](/tests/framework/installer/environment.go) for the available environment variables. + +To run all integration tests: ``` go test -v -timeout 1800s github.com/rook/rook/tests/integration ``` -#### To run a specific suite (uses regex) +To run a specific suite (uses regex): ``` go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration ``` -#### To run specific tests inside a suite: +To run specific tests inside a suite: ``` go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration -testify.m TestRookClusterInstallation_SmokeTest ``` diff --git a/tests/framework/installer/cassandra_installer.go b/tests/framework/installer/cassandra_installer.go index c9b73aca8..32587c2d4 100644 --- a/tests/framework/installer/cassandra_installer.go +++ b/tests/framework/installer/cassandra_installer.go @@ -162,10 +162,10 @@ func (ci *CassandraInstaller) UninstallCassandra(systemNamespace string, namespa } func (ci *CassandraInstaller) GatherAllCassandraLogs(systemNamespace, namespace, testName string) { - if !ci.T().Failed() && Env.Logs != "all" { + if !ci.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from Cassandra Cluster %s", namespace) - ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType) - ci.k8sHelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) + ci.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index b50099ad0..55f80a01b 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -333,10 +333,11 @@ func (h *CephInstaller) GetRookExternalClusterMonSecret(namespace string) (*v1.S } func (h *CephInstaller) initTestDir(namespace string) (string, error) { - h.hostPathToDelete = path.Join(baseTestDir, "rook-test") + h.hostPathToDelete = path.Join(baseTestDir(), "rook-test") testDir := path.Join(h.hostPathToDelete, namespace) - if createBaseTestDir { + // skip the test dir creation if we are not running under "/data" + if baseTestDir() != "/data" { // Create the test dir on the local host if err := os.MkdirAll(testDir, 0777); err != nil { return "", err @@ -401,7 +402,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") { logger.Error("rook-ceph-operator is not running") - h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", Env.HostType) + h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", testEnvName()) logger.Error("rook-ceph-operator is not Running, abort!") return false, err } @@ -664,21 +665,21 @@ func (h *CephInstaller) verifyDirCleanup(node, dir string) error { } func (h *CephInstaller) CollectOperatorLog(suiteName, testName, namespace string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } name := fmt.Sprintf("%s_%s", suiteName, testName) - h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, Env.HostType) + h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, testEnvName()) } func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("gathering all logs from the test") for _, namespace := range namespaces { - h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType) - h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, Env.HostType) + h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) + h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, testEnvName()) } } @@ -703,7 +704,7 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH h := &CephInstaller{ Manifests: NewCephManifests(rookVersion), k8shelper: k8shelp, - helmHelper: utils.NewHelmHelper(Env.Helm), + helmHelper: utils.NewHelmHelper(testHelmPath()), useHelm: useHelm, k8sVersion: version.String(), CephVersion: cephVersion, diff --git a/tests/framework/installer/cockroachdb_installer.go b/tests/framework/installer/cockroachdb_installer.go index 9b91f5c53..bf9b8166c 100644 --- a/tests/framework/installer/cockroachdb_installer.go +++ b/tests/framework/installer/cockroachdb_installer.go @@ -150,10 +150,10 @@ func (h *CockroachDBInstaller) UninstallCockroachDB(systemNamespace, namespace s } func (h *CockroachDBInstaller) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType) - h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/framework/installer/edgefs_installer.go b/tests/framework/installer/edgefs_installer.go index 6c4d363da..fe7492545 100644 --- a/tests/framework/installer/edgefs_installer.go +++ b/tests/framework/installer/edgefs_installer.go @@ -152,10 +152,10 @@ func (h *EdgefsInstaller) UninstallEdgefs(systemNamespace, namespace string) { } func (h *EdgefsInstaller) GatherAllEdgefsLogs(systemNamespace, namespace, testName string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from edgefs cluster %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType) - h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/framework/installer/environment.go b/tests/framework/installer/environment.go index b02bd0e2f..e593af28d 100644 --- a/tests/framework/installer/environment.go +++ b/tests/framework/installer/environment.go @@ -17,37 +17,43 @@ limitations under the License. package installer import ( - "flag" + "os" ) -// EnvironmentManifest contains information about system under test -type EnvironmentManifest struct { - HostType string - Helm string - RookImageName string - ToolboxImageName string - BaseTestDir string - LoadVolumeNumber int - LoadConcurrentRuns int - LoadTime int - LoadSize string - EnableChaos bool - Logs string +// testEnvName gets the name of the test environment. In the CI it is "aws_1.18.x" or similar. +func testEnvName() string { + return getEnvVarWithDefault("TEST_ENV_NAME", "localhost") } -var Env EnvironmentManifest - -func init() { - Env = EnvironmentManifest{} - flag.StringVar(&Env.HostType, "host_type", "localhost", "Host where tests are run eg - localhost, GCE or AWS") - flag.StringVar(&Env.Helm, "helm", "helm", "Path to helm binary") - flag.StringVar(&Env.RookImageName, "rook_image", "rook/ceph", "Docker image name for the rook container to install, must be in docker hub or local environment") - flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/ceph", "Docker image name of the toolbox container to install, must be in docker hub or local environment") - flag.StringVar(&Env.BaseTestDir, "base_test_dir", "/data", "Base test directory, for use only when kubernetes master is running on localhost") - flag.IntVar(&Env.LoadConcurrentRuns, "load_parallel_runs", 20, "number of routines for load test") - flag.IntVar(&Env.LoadVolumeNumber, "load_volumes", 1, "number of volumes(file,object or block) to be created for load test") - flag.IntVar(&Env.LoadTime, "load_time", 1800, "number of seconds each thread perform operations for") - flag.StringVar(&Env.LoadSize, "load_size", "medium", "load size for each thread performing operations - small,medium or large.") - flag.BoolVar(&Env.EnableChaos, "enable_chaos", false, "used to determine if random pods in a namespace are to be killed during load test.") - flag.StringVar(&Env.Logs, "logs", "", "Gather rook logs, eg - all") +// testHelmPath gets the helm path +func testHelmPath() string { + return getEnvVarWithDefault("TEST_HELM_PATH", "helm") +} + +// TestLogCollectionLevel gets whether to collect all logs +func TestLogCollectionLevel() string { + return getEnvVarWithDefault("TEST_LOG_COLLECTION_LEVEL", "") +} + +// testStorageProvider gets the storage provider for which tests should be run +func testStorageProvider() string { + return getEnvVarWithDefault("STORAGE_PROVIDER_TESTS", "") +} + +// baseTestDir gets the base test directory +func baseTestDir() string { + // If the base test directory is actively set to empty (as in CI), we use the current working directory. + val := getEnvVarWithDefault("TEST_BASE_DIR", "/data") + if val == "WORKING_DIR" { + val, _ = os.Getwd() + } + return val +} + +func getEnvVarWithDefault(env, defaultValue string) string { + val := os.Getenv(env) + if val == "" { + return defaultValue + } + return val } diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index 637f7feff..50780753b 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -18,7 +18,6 @@ package installer import ( "fmt" - "os" "strings" "testing" @@ -41,10 +40,6 @@ const ( ) var ( - // ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. ** - baseTestDir string - createBaseTestDir = true - // ** end of Variables to modify logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer") createArgs = []string{"create", "-f"} createFromStdinArgs = append(createArgs, "-") @@ -58,7 +53,7 @@ type TestSuite interface { } func SkipTestSuite(name string) bool { - testsToRun := os.Getenv("STORAGE_PROVIDER_TESTS") + testsToRun := testStorageProvider() // jenkins passes "null" if the env var is not set. if testsToRun == "" || testsToRun == "null" { // run all test suites @@ -73,17 +68,6 @@ func SkipTestSuite(name string) bool { return true } -func init() { - // If the base test directory is actively set to empty (as in CI), we use the current working directory. - baseTestDir = Env.BaseTestDir - if baseTestDir == "" { - baseTestDir, _ = os.Getwd() - } - if baseTestDir == "/data" { - createBaseTestDir = false - } -} - func SystemNamespace(namespace string) string { return fmt.Sprintf("%s-system", namespace) } diff --git a/tests/framework/installer/nfs_installer.go b/tests/framework/installer/nfs_installer.go index e4f8f0b02..8a954f6d1 100644 --- a/tests/framework/installer/nfs_installer.go +++ b/tests/framework/installer/nfs_installer.go @@ -190,10 +190,10 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) { // GatherAllNFSServerLogs gathers all NFS Server logs func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) { - if !h.T().Failed() && Env.Logs != "all" { + if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from NFSServer %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, Env.HostType) - h.k8shelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/framework/installer/provisioners.go b/tests/framework/installer/provisioners.go index b96bcd9fc..203e554b0 100644 --- a/tests/framework/installer/provisioners.go +++ b/tests/framework/installer/provisioners.go @@ -66,7 +66,7 @@ func InstallHostPathProvisioner(k8shelper *utils.K8sHelper) error { err = k8shelper.WaitForLabeledPodsToRun("k8s-app=hostpath-provisioner", "kube-system") if err != nil { logger.Errorf("hostpath provisioner pod is not running: %+v", err) - k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", Env.HostType) + k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", testEnvName()) k8shelper.PrintStorageClasses(true /*detailed*/) return err } diff --git a/tests/framework/installer/yugabytedb_installer.go b/tests/framework/installer/yugabytedb_installer.go index 586c27dfb..fe64524a5 100644 --- a/tests/framework/installer/yugabytedb_installer.go +++ b/tests/framework/installer/yugabytedb_installer.go @@ -161,10 +161,10 @@ func (y *YugabyteDBInstaller) RemoveAllYugabyteDBResources(systemNS, namespace s } func (y *YugabyteDBInstaller) GatherAllLogs(systemNS, namespace, testName string) { - if !y.T().Failed() && Env.Logs != "all" { + if !y.T().Failed() && TestLogCollectionLevel() != "all" { return } logger.Infof("Gathering all logs from yugabytedb cluster %s", namespace) - y.k8sHelper.GetLogsFromNamespace(systemNS, testName, Env.HostType) - y.k8sHelper.GetLogsFromNamespace(namespace, testName, Env.HostType) + y.k8sHelper.GetLogsFromNamespace(systemNS, testName, testEnvName()) + y.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName()) } diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 97f268c01..a89650222 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -213,7 +213,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) { // Gather logs before Ceph upgrade to help with debugging - if installer.Env.Logs == "all" { + if installer.TestLogCollectionLevel() == "all" { s.k8sh.PrintPodDescribe(s.namespace) } n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix