diff --git a/tests/README.md b/tests/README.md index 545077413..fd68926a7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -142,3 +142,22 @@ To run specific tests inside a suite: ```console _output/tests/linux_amd64/integration -test.v -test.timeout 1800s -test.run CephSmokeSuite -testify.m TestARookClusterInstallation_SmokeTest ``` + +### To run tests on OpenShift environment + +- Setup OpenShift environment and export KUBECONFIG before executing the tests. +- Make sure `oc` executable file is in the PATH. +- Only `CephSmokeSuite` is currently supported on OpenShift. +- Set few environment variables: +``` +export TEST_ENV_NAME=openshift +export TEST_STORAGE_CLASS=gp2 +export TEST_BASE_DIR=/tmp +export RETRY_MAX=40 +``` + + +To run a `CephSmokeSuite` (uses regex): +```console +$ go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration +``` diff --git a/tests/framework/installer/cassandra_installer.go b/tests/framework/installer/cassandra_installer.go index 32587c2d4..587c7af03 100644 --- a/tests/framework/installer/cassandra_installer.go +++ b/tests/framework/installer/cassandra_installer.go @@ -166,6 +166,6 @@ func (ci *CassandraInstaller) GatherAllCassandraLogs(systemNamespace, namespace, return } logger.Infof("Gathering all logs from Cassandra Cluster %s", namespace) - ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) - ci.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName()) + ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, utils.TestEnvName()) + ci.k8sHelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName()) } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 0a51dffe3..2f4be7df5 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -93,10 +93,21 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { h.k8shelper.ChangeHostnames() } - err = h.startAdmissionController(namespace) + err = h.k8shelper.CreateNamespace(namespace) if err != nil { - return fmt.Errorf("Failed to start admission controllers: %v", err) + if errors.IsAlreadyExists(err) { + logger.Warningf("Namespace %q already exists!!!", namespace) + } else { + return fmt.Errorf("failed to create namespace %q. %v", namespace, err) + } } + if !utils.IsPlatformOpenShift() { + err = h.startAdmissionController(namespace) + if err != nil { + return fmt.Errorf("Failed to start admission controllers: %v", err) + } + } + rookOperator := h.Manifests.GetRookOperator(namespace) _, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...) if err != nil { @@ -175,9 +186,11 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings strin return fmt.Errorf("Failed to get Version of helm chart %v, err : %v", helmChartName, err) } - err = h.startAdmissionController(namespace) - if err != nil { - return fmt.Errorf("Failed to start admission controllers: %v", err) + if !utils.IsPlatformOpenShift() { + err = h.startAdmissionController(namespace) + if err != nil { + return fmt.Errorf("Failed to start admission controllers: %v", err) + } } err = h.helmHelper.InstallLocalRookHelmChart(helmChartName, helmDeployName, helmTag, namespace, chartSettings) @@ -465,7 +478,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") { logger.Error("rook-ceph-operator is not running") - h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", testEnvName()) + h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", utils.TestEnvName()) logger.Error("rook-ceph-operator is not Running, abort!") return false, err } @@ -496,7 +509,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } logger.Infof("installed rook operator and cluster : %s on k8s %s", namespace, h.k8sVersion) - if rookVersion != Version1_2 && h.k8shelper.VersionAtLeast("v1.15.0") { + if !utils.IsPlatformOpenShift() && rookVersion != Version1_2 && h.k8shelper.VersionAtLeast("v1.15.0") { if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", onamespace, "Running") { assert.Fail(h.T(), "admission controller is not running") } @@ -742,7 +755,7 @@ func (h *CephInstaller) CollectOperatorLog(suiteName, testName, namespace string return } name := fmt.Sprintf("%s_%s", suiteName, testName) - h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, testEnvName()) + h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, utils.TestEnvName()) } func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) { @@ -751,9 +764,9 @@ func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) } logger.Infof("gathering all logs from the test") for _, namespace := range namespaces { - h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) - h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, testEnvName()) - h.k8shelper.GetEventsFromNamespace(namespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName()) + h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, utils.TestEnvName()) + h.k8shelper.GetEventsFromNamespace(namespace, testName, utils.TestEnvName()) } } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 0891a5a92..0f17df1e2 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -22,6 +22,7 @@ import ( "strings" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/tests/framework/utils" ) const ( @@ -807,10 +808,108 @@ spec: status: {}` } +func getOpenshiftSCC(namespace string) string { + return `--- +kind: SecurityContextConstraints +# older versions of openshift have "apiVersion: v1" +apiVersion: security.openshift.io/v1 +metadata: + name: rook-ceph +allowPrivilegedContainer: true +allowHostNetwork: true +allowHostDirVolumePlugin: true +priority: +allowedCapabilities: [] +allowHostPorts: true +allowHostPID: true +allowHostIPC: true +readOnlyRootFilesystem: false +requiredDropCapabilities: [] +defaultAddCapabilities: [] +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +fsGroup: + type: MustRunAs +supplementalGroups: + type: RunAsAny +allowedFlexVolumes: + - driver: "ceph.rook.io/rook" + - driver: "ceph.rook.io/rook-ceph" +volumes: + - configMap + - downwardAPI + - emptyDir + - flexVolume + - hostPath + - persistentVolumeClaim + - projected + - secret +users: + # A user needs to be added for each rook service account. + # This assumes running in the default sample "rook-ceph" namespace. + # If other namespaces or service accounts are configured, they need to be updated here. + - system:serviceaccount:` + namespace + `:rook-ceph-system + - system:serviceaccount:` + namespace + `:default + - system:serviceaccount:` + namespace + `:rook-ceph-mgr + - system:serviceaccount:` + namespace + `:rook-ceph-osd +--- +kind: SecurityContextConstraints +# older versions of openshift have "apiVersion: v1" +apiVersion: security.openshift.io/v1 +metadata: + name: rook-ceph-csi +allowPrivilegedContainer: true +allowHostNetwork: true +allowHostDirVolumePlugin: true +priority: +allowedCapabilities: ['*'] +allowHostPorts: true +allowHostPID: true +allowHostIPC: true +readOnlyRootFilesystem: false +requiredDropCapabilities: [] +defaultAddCapabilities: [] +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +fsGroup: + type: RunAsAny +supplementalGroups: + type: RunAsAny +allowedFlexVolumes: + - driver: "ceph.rook.io/rook" + - driver: "ceph.rook.io/rook-ceph" +volumes: ['*'] +users: + # A user needs to be added for each rook service account. + # This assumes running in the default sample "rook-ceph" namespace. + # If other namespaces or service accounts are configured, they need to be updated here. + - system:serviceaccount:` + namespace + `:rook-csi-rbd-plugin-sa + - system:serviceaccount:` + namespace + `:rook-csi-rbd-provisioner-sa + - system:serviceaccount:` + namespace + `:rook-csi-cephfs-plugin-sa + - system:serviceaccount:` + namespace + `:rook-csi-cephfs-provisioner-sa +` +} + // GetRookOperator returns rook Operator manifest func (m *CephManifestsMaster) GetRookOperator(namespace string) string { - return ` -apiVersion: rbac.authorization.k8s.io/v1 + var operatorManifest string + openshiftEnv := "" + + if utils.IsPlatformOpenShift() { + openshiftEnv = ` + - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED + value: "true" + - name: FLEXVOLUME_DIR_PATH + value: "/etc/kubernetes/kubelet-plugins/volume/exec"` + operatorManifest = getOpenshiftSCC(namespace) + } + + operatorManifest = operatorManifest + `--- +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role metadata: name: rook-ceph-system @@ -1641,7 +1740,17 @@ spec: - name: ROOK_ENABLE_FLEX_DRIVER value: "true" - name: ROOK_CURRENT_NAMESPACE_ONLY - value: "false" + value: "false"` + openshiftEnv + ` + volumeMounts: + - mountPath: /var/lib/rook + name: rook-config + - mountPath: /etc/ceph + name: default-config-dir + volumes: + - name: rook-config + emptyDir: {} + - name: default-config-dir + emptyDir: {} --- kind: ConfigMap apiVersion: v1 @@ -1655,6 +1764,7 @@ data: ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" CSI_LOG_LEVEL: "5" ` + return operatorManifest } // GetClusterRoles returns rook-cluster manifest diff --git a/tests/framework/installer/cockroachdb_installer.go b/tests/framework/installer/cockroachdb_installer.go index 674dbf606..0a72b10f4 100644 --- a/tests/framework/installer/cockroachdb_installer.go +++ b/tests/framework/installer/cockroachdb_installer.go @@ -154,6 +154,6 @@ func (h *CockroachDBInstaller) GatherAllCockroachDBLogs(systemNamespace, namespa return } logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) - h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, utils.TestEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName()) } diff --git a/tests/framework/installer/edgefs_installer.go b/tests/framework/installer/edgefs_installer.go index d89731997..ab3b10808 100644 --- a/tests/framework/installer/edgefs_installer.go +++ b/tests/framework/installer/edgefs_installer.go @@ -156,6 +156,6 @@ func (h *EdgefsInstaller) GatherAllEdgefsLogs(systemNamespace, namespace, testNa return } logger.Infof("Gathering all logs from edgefs cluster %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) - h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, utils.TestEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName()) } diff --git a/tests/framework/installer/environment.go b/tests/framework/installer/environment.go index 29ee27410..b337d1f7d 100644 --- a/tests/framework/installer/environment.go +++ b/tests/framework/installer/environment.go @@ -20,11 +20,6 @@ import ( "os" ) -// testEnvName gets the name of the test environment. In the CI it is "aws_1.18.x" or similar. -func testEnvName() string { - return getEnvVarWithDefault("TEST_ENV_NAME", "localhost") -} - // testHelmPath gets the helm path func testHelmPath() string { return getEnvVarWithDefault("TEST_HELM_PATH", "helm") @@ -46,6 +41,14 @@ func TestIsOfficialBuild() bool { return getEnvVarWithDefault("TEST_IS_OFFICIAL_BUILD", "") != "false" } +func StorageClassName() string { + return getEnvVarWithDefault("TEST_STORAGE_CLASS", "") +} + +func UsePVC() bool { + return StorageClassName() != "" +} + // baseTestDir gets the base test directory func baseTestDir() string { // If the base test directory is actively set to empty (as in CI), we use the current working directory. diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index 50780753b..67aaec536 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/errors" ) @@ -69,6 +70,10 @@ func SkipTestSuite(name string) bool { } func SystemNamespace(namespace string) string { + if utils.IsPlatformOpenShift() { + logger.Infof("For openshift execution used system namespace: %s", namespace) + return namespace + } return fmt.Sprintf("%s-system", namespace) } diff --git a/tests/framework/installer/nfs_installer.go b/tests/framework/installer/nfs_installer.go index 4505ab061..f1838dc58 100644 --- a/tests/framework/installer/nfs_installer.go +++ b/tests/framework/installer/nfs_installer.go @@ -194,6 +194,6 @@ func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testNa return } logger.Infof("Gathering all logs from NFSServer %s", namespace) - h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName()) - h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName()) + h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, utils.TestEnvName()) + h.k8shelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName()) } diff --git a/tests/framework/installer/provisioners.go b/tests/framework/installer/provisioners.go index c9295a119..68f9b282b 100644 --- a/tests/framework/installer/provisioners.go +++ b/tests/framework/installer/provisioners.go @@ -66,7 +66,7 @@ func InstallHostPathProvisioner(k8shelper *utils.K8sHelper) error { err = k8shelper.WaitForLabeledPodsToRun("k8s-app=hostpath-provisioner", "kube-system") if err != nil { logger.Errorf("hostpath provisioner pod is not running: %+v", err) - k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", testEnvName()) + k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", utils.TestEnvName()) k8shelper.PrintStorageClasses(true /*detailed*/) return err } diff --git a/tests/framework/installer/yugabytedb_installer.go b/tests/framework/installer/yugabytedb_installer.go index 8793f5f4c..3d221c679 100644 --- a/tests/framework/installer/yugabytedb_installer.go +++ b/tests/framework/installer/yugabytedb_installer.go @@ -165,6 +165,6 @@ func (y *YugabyteDBInstaller) GatherAllLogs(systemNS, namespace, testName string return } logger.Infof("Gathering all logs from yugabytedb cluster %s", namespace) - y.k8sHelper.GetLogsFromNamespace(systemNS, testName, testEnvName()) - y.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName()) + y.k8sHelper.GetLogsFromNamespace(systemNS, testName, utils.TestEnvName()) + y.k8sHelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName()) } diff --git a/tests/framework/utils/env.go b/tests/framework/utils/env.go new file mode 100644 index 000000000..9d26ad41c --- /dev/null +++ b/tests/framework/utils/env.go @@ -0,0 +1,52 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "os" + "strconv" +) + +// TestEnvName gets the name of the test environment. In the CI it is "aws_1.18.x" or similar. +func TestEnvName() string { + return GetEnvVarWithDefault("TEST_ENV_NAME", "localhost") +} + +// TestRetryNumber get the max retry. Example, for OpenShift it's 40. +func TestRetryNumber() int { + count := GetEnvVarWithDefault("RETRY_MAX", "30") + number, err := strconv.Atoi(count) + if err != nil { + panic(fmt.Errorf("Error when converting to numeric value %v", err)) + } + return number +} + +// IsPlatformOpenShift check if the platform is openshift or not +func IsPlatformOpenShift() bool { + return TestEnvName() == "openshift" +} + +// GetEnvVarWithDefault get environment variable by key. +func GetEnvVarWithDefault(env, defaultValue string) string { + val := os.Getenv(env) + if val == "" { + return defaultValue + } + return val +} diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 8dc97b405..8ed1ac8e0 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -54,8 +54,6 @@ type K8sHelper struct { } const ( - // RetryLoop params for tests. - RetryLoop = 60 // RetryInterval param for test - wait time while in RetryLoop RetryInterval = 5 // TestMountPath is the path inside a test pod where storage is mounted @@ -64,6 +62,16 @@ const ( hostnameTestPrefix = "test-prefix-this-is-a-very-long-hostname-" ) +// getCmd returns kubectl or oc if env var rook_test_openshift is +// set to true +func getCmd() string { + cmd := "kubectl" + if IsPlatformOpenShift() { + cmd = "oc" + } + return cmd +} + // CreateK8sHelper creates a instance of k8sHelper func CreateK8sHelper(t func() *testing.T) (*K8sHelper, error) { executor := &exec.CommandExecutor{} @@ -87,7 +95,12 @@ func CreateK8sHelper(t func() *testing.T) (*K8sHelper, error) { return h, err } -var k8slogger = capnslog.NewPackageLogger("github.com/rook/rook", "utils") +var ( + k8slogger = capnslog.NewPackageLogger("github.com/rook/rook", "utils") + cmd = getCmd() + // RetryLoop params for tests. + RetryLoop = TestRetryNumber() +) // GetK8sServerVersion returns k8s server version under test func (k8sh *K8sHelper) GetK8sServerVersion() string { @@ -130,12 +143,12 @@ func (k8sh *K8sHelper) SetDeploymentVersion(namespace, deploymentName, container func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) { result, err := k8sh.executor.ExecuteCommandWithTimeout(15*time.Second, "kubectl", args...) if err != nil { - k8slogger.Errorf("Failed to execute: kubectl %+v : %+v. %s", args, err, result) + k8slogger.Errorf("Failed to execute: %s %+v : %+v. %s", cmd, args, err, result) if args[0] == "delete" { // allow the tests to continue if we were deleting a resource that timed out return result, nil } - return result, fmt.Errorf("Failed to run: kubectl %v : %v", args, err) + return result, fmt.Errorf("Failed to run: %s %v : %v", cmd, args, err) } return result, nil } @@ -143,15 +156,15 @@ func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) { // KubectlWithStdin is wrapper for executing kubectl commands in stdin func (k8sh *K8sHelper) KubectlWithStdin(stdin string, args ...string) (string, error) { - cmdStruct := CommandArgs{Command: "kubectl", PipeToStdIn: stdin, CmdArgs: args} + cmdStruct := CommandArgs{Command: cmd, PipeToStdIn: stdin, CmdArgs: args} cmdOut := ExecuteCommand(cmdStruct) if cmdOut.ExitCode != 0 { - k8slogger.Errorf("Failed to execute stdin: kubectl %v : %v", args, cmdOut.Err.Error()) + k8slogger.Errorf("Failed to execute stdin: %s %v : %v", cmd, args, cmdOut.Err.Error()) if strings.Contains(cmdOut.Err.Error(), "(NotFound)") || strings.Contains(cmdOut.StdErr, "(NotFound)") { return cmdOut.StdErr, errors.NewNotFound(schema.GroupResource{}, "") } - return cmdOut.StdErr, fmt.Errorf("Failed to run stdin: kubectl %v : %v", args, cmdOut.StdErr) + return cmdOut.StdErr, fmt.Errorf("Failed to run stdin: %s %v : %v", cmd, args, cmdOut.StdErr) } if cmdOut.StdOut == "" { return cmdOut.StdErr, nil @@ -170,7 +183,7 @@ func getKubeConfig(executor exec.Executor) (*rest.Config, error) { // Parse the kubectl context to get the settings for client connections var kc kubectlContext if err := json.Unmarshal([]byte(context), &kc); err != nil { - return nil, fmt.Errorf("failed to unmarshal kubectl config: %+v", err) + return nil, fmt.Errorf("failed to unmarshal %s config: %+v", cmd, err) } // find the current context @@ -282,7 +295,7 @@ func (k8sh *K8sHelper) ExecWithRetry(retries int, namespace, podName, command st time.Sleep(3 * time.Second) } } - return "", fmt.Errorf("kubectl exec command %s failed on pod %s in namespace %s. %+v", command, podName, namespace, err) + return "", fmt.Errorf("%s exec command %s failed on pod %s in namespace %s. %+v", cmd, command, podName, namespace, err) } // ResourceOperationFromTemplate performs a kubectl action from a template file after replacing its context diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index b2dcaeba9..80a8ce6c5 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -84,9 +84,9 @@ func (suite *SmokeSuite) SetupSuite() { smokeTestCluster := TestCluster{ namespace: suite.namespace, storeType: "bluestore", - storageClassName: "", + storageClassName: installer.StorageClassName(), useHelm: false, - usePVC: false, + usePVC: installer.UsePVC(), mons: 3, rbdMirrorWorkers: 1, rookCephCleanup: true, @@ -118,7 +118,9 @@ func (suite *SmokeSuite) TestFileStorage_SmokeTest() { } func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { - runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace) + if !utils.IsPlatformOpenShift() { + runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace) + } } // Test to make sure all rook components are installed and Running