ceph: integration test on OpenShift

this commit enables the rook integration
testing capability on openshift. currently
this commit enable CephSmokeSuite testing
only.

Signed-off-by: subhamkrai <subhamkumarrai03@gmail.com>
This commit is contained in:
subhamkrai committed 2020-09-18 14:30:57 +05:30
1 parent 204ffd4065
commit 4c11e45155
14 files changed
+260 -43

No files matched your search

+19
View File
@@ -142,3 +142,22 @@ To run specific tests inside a suite:
```console
_output/tests/linux_amd64/integration -test.v -test.timeout 1800s -test.run CephSmokeSuite -testify.m TestARookClusterInstallation_SmokeTest
```
### To run tests on OpenShift environment
- Setup OpenShift environment and export KUBECONFIG before executing the tests.
- Make sure `oc` executable file is in the PATH.
- Only `CephSmokeSuite` is currently supported on OpenShift.
- Set few environment variables:
```
export TEST_ENV_NAME=openshift
export TEST_STORAGE_CLASS=gp2
export TEST_BASE_DIR=/tmp
export RETRY_MAX=40
```
To run a `CephSmokeSuite` (uses regex):
```console
$ go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration
```
@@ -166,6 +166,6 @@ func (ci *CassandraInstaller) GatherAllCassandraLogs(systemNamespace, namespace,
return
}
logger.Infof("Gathering all logs from Cassandra Cluster %s", namespace)
ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName())
ci.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName())
ci.k8sHelper.GetLogsFromNamespace(systemNamespace, testName, utils.TestEnvName())
ci.k8sHelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName())
}
+24 -11
View File
@@ -93,10 +93,21 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) {
h.k8shelper.ChangeHostnames()
}
err = h.startAdmissionController(namespace)
err = h.k8shelper.CreateNamespace(namespace)
if err != nil {
return fmt.Errorf("Failed to start admission controllers: %v", err)
if errors.IsAlreadyExists(err) {
logger.Warningf("Namespace %q already exists!!!", namespace)
} else {
return fmt.Errorf("failed to create namespace %q. %v", namespace, err)
}
}
if !utils.IsPlatformOpenShift() {
err = h.startAdmissionController(namespace)
if err != nil {
return fmt.Errorf("Failed to start admission controllers: %v", err)
}
}
rookOperator := h.Manifests.GetRookOperator(namespace)
_, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...)
if err != nil {
@@ -175,9 +186,11 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings strin
return fmt.Errorf("Failed to get Version of helm chart %v, err : %v", helmChartName, err)
}
err = h.startAdmissionController(namespace)
if err != nil {
return fmt.Errorf("Failed to start admission controllers: %v", err)
if !utils.IsPlatformOpenShift() {
err = h.startAdmissionController(namespace)
if err != nil {
return fmt.Errorf("Failed to start admission controllers: %v", err)
}
}
err = h.helmHelper.InstallLocalRookHelmChart(helmChartName, helmDeployName, helmTag, namespace, chartSettings)
@@ -465,7 +478,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st
}
if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") {
logger.Error("rook-ceph-operator is not running")
h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", testEnvName())
h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", utils.TestEnvName())
logger.Error("rook-ceph-operator is not Running, abort!")
return false, err
}
@@ -496,7 +509,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st
}
logger.Infof("installed rook operator and cluster : %s on k8s %s", namespace, h.k8sVersion)
if rookVersion != Version1_2 && h.k8shelper.VersionAtLeast("v1.15.0") {
if !utils.IsPlatformOpenShift() && rookVersion != Version1_2 && h.k8shelper.VersionAtLeast("v1.15.0") {
if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", onamespace, "Running") {
assert.Fail(h.T(), "admission controller is not running")
}
@@ -742,7 +755,7 @@ func (h *CephInstaller) CollectOperatorLog(suiteName, testName, namespace string
return
}
name := fmt.Sprintf("%s_%s", suiteName, testName)
h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, testEnvName())
h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, utils.TestEnvName())
}
func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) {
@@ -751,9 +764,9 @@ func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string)
}
logger.Infof("gathering all logs from the test")
for _, namespace := range namespaces {
h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName())
h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, testEnvName())
h.k8shelper.GetEventsFromNamespace(namespace, testName, testEnvName())
h.k8shelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName())
h.k8shelper.GetPodDescribeFromNamespace(namespace, testName, utils.TestEnvName())
h.k8shelper.GetEventsFromNamespace(namespace, testName, utils.TestEnvName())
}
}
+113 -3
View File
@@ -22,6 +22,7 @@ import (
"strings"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/utils"
)
const (
@@ -807,10 +808,108 @@ spec:
status: {}`
}
func getOpenshiftSCC(namespace string) string {
return `---
kind: SecurityContextConstraints
# older versions of openshift have "apiVersion: v1"
apiVersion: security.openshift.io/v1
metadata:
name: rook-ceph
allowPrivilegedContainer: true
allowHostNetwork: true
allowHostDirVolumePlugin: true
priority:
allowedCapabilities: []
allowHostPorts: true
allowHostPID: true
allowHostIPC: true
readOnlyRootFilesystem: false
requiredDropCapabilities: []
defaultAddCapabilities: []
runAsUser:
type: RunAsAny
seLinuxContext:
type: MustRunAs
fsGroup:
type: MustRunAs
supplementalGroups:
type: RunAsAny
allowedFlexVolumes:
- driver: "ceph.rook.io/rook"
- driver: "ceph.rook.io/rook-ceph"
volumes:
- configMap
- downwardAPI
- emptyDir
- flexVolume
- hostPath
- persistentVolumeClaim
- projected
- secret
users:
# A user needs to be added for each rook service account.
# This assumes running in the default sample "rook-ceph" namespace.
# If other namespaces or service accounts are configured, they need to be updated here.
- system:serviceaccount:` + namespace + `:rook-ceph-system
- system:serviceaccount:` + namespace + `:default
- system:serviceaccount:` + namespace + `:rook-ceph-mgr
- system:serviceaccount:` + namespace + `:rook-ceph-osd
---
kind: SecurityContextConstraints
# older versions of openshift have "apiVersion: v1"
apiVersion: security.openshift.io/v1
metadata:
name: rook-ceph-csi
allowPrivilegedContainer: true
allowHostNetwork: true
allowHostDirVolumePlugin: true
priority:
allowedCapabilities: ['*']
allowHostPorts: true
allowHostPID: true
allowHostIPC: true
readOnlyRootFilesystem: false
requiredDropCapabilities: []
defaultAddCapabilities: []
runAsUser:
type: RunAsAny
seLinuxContext:
type: RunAsAny
fsGroup:
type: RunAsAny
supplementalGroups:
type: RunAsAny
allowedFlexVolumes:
- driver: "ceph.rook.io/rook"
- driver: "ceph.rook.io/rook-ceph"
volumes: ['*']
users:
# A user needs to be added for each rook service account.
# This assumes running in the default sample "rook-ceph" namespace.
# If other namespaces or service accounts are configured, they need to be updated here.
- system:serviceaccount:` + namespace + `:rook-csi-rbd-plugin-sa
- system:serviceaccount:` + namespace + `:rook-csi-rbd-provisioner-sa
- system:serviceaccount:` + namespace + `:rook-csi-cephfs-plugin-sa
- system:serviceaccount:` + namespace + `:rook-csi-cephfs-provisioner-sa
`
}
// GetRookOperator returns rook Operator manifest
func (m *CephManifestsMaster) GetRookOperator(namespace string) string {
return `
apiVersion: rbac.authorization.k8s.io/v1
var operatorManifest string
openshiftEnv := ""
if utils.IsPlatformOpenShift() {
openshiftEnv = `
- name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
value: "true"
- name: FLEXVOLUME_DIR_PATH
value: "/etc/kubernetes/kubelet-plugins/volume/exec"`
operatorManifest = getOpenshiftSCC(namespace)
}
operatorManifest = operatorManifest + `---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: rook-ceph-system
@@ -1641,7 +1740,17 @@ spec:
- name: ROOK_ENABLE_FLEX_DRIVER
value: "true"
- name: ROOK_CURRENT_NAMESPACE_ONLY
value: "false"
value: "false"` + openshiftEnv + `
volumeMounts:
- mountPath: /var/lib/rook
name: rook-config
- mountPath: /etc/ceph
name: default-config-dir
volumes:
- name: rook-config
emptyDir: {}
- name: default-config-dir
emptyDir: {}
---
kind: ConfigMap
apiVersion: v1
@@ -1655,6 +1764,7 @@ data:
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
CSI_LOG_LEVEL: "5"
`
return operatorManifest
}
// GetClusterRoles returns rook-cluster manifest
@@ -154,6 +154,6 @@ func (h *CockroachDBInstaller) GatherAllCockroachDBLogs(systemNamespace, namespa
return
}
logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace)
h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName())
h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName())
h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, utils.TestEnvName())
h.k8shelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName())
}
@@ -156,6 +156,6 @@ func (h *EdgefsInstaller) GatherAllEdgefsLogs(systemNamespace, namespace, testNa
return
}
logger.Infof("Gathering all logs from edgefs cluster %s", namespace)
h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName())
h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName())
h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, utils.TestEnvName())
h.k8shelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName())
}
+8 -5
View File
@@ -20,11 +20,6 @@ import (
"os"
)
// testEnvName gets the name of the test environment. In the CI it is "aws_1.18.x" or similar.
func testEnvName() string {
return getEnvVarWithDefault("TEST_ENV_NAME", "localhost")
}
// testHelmPath gets the helm path
func testHelmPath() string {
return getEnvVarWithDefault("TEST_HELM_PATH", "helm")
@@ -46,6 +41,14 @@ func TestIsOfficialBuild() bool {
return getEnvVarWithDefault("TEST_IS_OFFICIAL_BUILD", "") != "false"
}
func StorageClassName() string {
return getEnvVarWithDefault("TEST_STORAGE_CLASS", "")
}
func UsePVC() bool {
return StorageClassName() != ""
}
// baseTestDir gets the base test directory
func baseTestDir() string {
// If the base test directory is actively set to empty (as in CI), we use the current working directory.
+5
View File
@@ -22,6 +22,7 @@ import (
"testing"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/api/errors"
)
@@ -69,6 +70,10 @@ func SkipTestSuite(name string) bool {
}
func SystemNamespace(namespace string) string {
if utils.IsPlatformOpenShift() {
logger.Infof("For openshift execution used system namespace: %s", namespace)
return namespace
}
return fmt.Sprintf("%s-system", namespace)
}
+2 -2
View File
@@ -194,6 +194,6 @@ func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testNa
return
}
logger.Infof("Gathering all logs from NFSServer %s", namespace)
h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, testEnvName())
h.k8shelper.GetLogsFromNamespace(namespace, testName, testEnvName())
h.k8shelper.GetLogsFromNamespace(systemNamespace, testName, utils.TestEnvName())
h.k8shelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName())
}
+1 -1
View File
@@ -66,7 +66,7 @@ func InstallHostPathProvisioner(k8shelper *utils.K8sHelper) error {
err = k8shelper.WaitForLabeledPodsToRun("k8s-app=hostpath-provisioner", "kube-system")
if err != nil {
logger.Errorf("hostpath provisioner pod is not running: %+v", err)
k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", testEnvName())
k8shelper.GetPodDescribeFromNamespace("kube-system", "hostPathProvisioner", utils.TestEnvName())
k8shelper.PrintStorageClasses(true /*detailed*/)
return err
}
@@ -165,6 +165,6 @@ func (y *YugabyteDBInstaller) GatherAllLogs(systemNS, namespace, testName string
return
}
logger.Infof("Gathering all logs from yugabytedb cluster %s", namespace)
y.k8sHelper.GetLogsFromNamespace(systemNS, testName, testEnvName())
y.k8sHelper.GetLogsFromNamespace(namespace, testName, testEnvName())
y.k8sHelper.GetLogsFromNamespace(systemNS, testName, utils.TestEnvName())
y.k8sHelper.GetLogsFromNamespace(namespace, testName, utils.TestEnvName())
}
+52
View File
@@ -0,0 +1,52 @@
/*
Copyright 2020 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package utils
import (
"fmt"
"os"
"strconv"
)
// TestEnvName gets the name of the test environment. In the CI it is "aws_1.18.x" or similar.
func TestEnvName() string {
return GetEnvVarWithDefault("TEST_ENV_NAME", "localhost")
}
// TestRetryNumber get the max retry. Example, for OpenShift it's 40.
func TestRetryNumber() int {
count := GetEnvVarWithDefault("RETRY_MAX", "30")
number, err := strconv.Atoi(count)
if err != nil {
panic(fmt.Errorf("Error when converting to numeric value %v", err))
}
return number
}
// IsPlatformOpenShift check if the platform is openshift or not
func IsPlatformOpenShift() bool {
return TestEnvName() == "openshift"
}
// GetEnvVarWithDefault get environment variable by key.
func GetEnvVarWithDefault(env, defaultValue string) string {
val := os.Getenv(env)
if val == "" {
return defaultValue
}
return val
}
+23 -10
View File
@@ -54,8 +54,6 @@ type K8sHelper struct {
}
const (
// RetryLoop params for tests.
RetryLoop = 60
// RetryInterval param for test - wait time while in RetryLoop
RetryInterval = 5
// TestMountPath is the path inside a test pod where storage is mounted
@@ -64,6 +62,16 @@ const (
hostnameTestPrefix = "test-prefix-this-is-a-very-long-hostname-"
)
// getCmd returns kubectl or oc if env var rook_test_openshift is
// set to true
func getCmd() string {
cmd := "kubectl"
if IsPlatformOpenShift() {
cmd = "oc"
}
return cmd
}
// CreateK8sHelper creates a instance of k8sHelper
func CreateK8sHelper(t func() *testing.T) (*K8sHelper, error) {
executor := &exec.CommandExecutor{}
@@ -87,7 +95,12 @@ func CreateK8sHelper(t func() *testing.T) (*K8sHelper, error) {
return h, err
}
var k8slogger = capnslog.NewPackageLogger("github.com/rook/rook", "utils")
var (
k8slogger = capnslog.NewPackageLogger("github.com/rook/rook", "utils")
cmd = getCmd()
// RetryLoop params for tests.
RetryLoop = TestRetryNumber()
)
// GetK8sServerVersion returns k8s server version under test
func (k8sh *K8sHelper) GetK8sServerVersion() string {
@@ -130,12 +143,12 @@ func (k8sh *K8sHelper) SetDeploymentVersion(namespace, deploymentName, container
func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) {
result, err := k8sh.executor.ExecuteCommandWithTimeout(15*time.Second, "kubectl", args...)
if err != nil {
k8slogger.Errorf("Failed to execute: kubectl %+v : %+v. %s", args, err, result)
k8slogger.Errorf("Failed to execute: %s %+v : %+v. %s", cmd, args, err, result)
if args[0] == "delete" {
// allow the tests to continue if we were deleting a resource that timed out
return result, nil
}
return result, fmt.Errorf("Failed to run: kubectl %v : %v", args, err)
return result, fmt.Errorf("Failed to run: %s %v : %v", cmd, args, err)
}
return result, nil
}
@@ -143,15 +156,15 @@ func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) {
// KubectlWithStdin is wrapper for executing kubectl commands in stdin
func (k8sh *K8sHelper) KubectlWithStdin(stdin string, args ...string) (string, error) {
cmdStruct := CommandArgs{Command: "kubectl", PipeToStdIn: stdin, CmdArgs: args}
cmdStruct := CommandArgs{Command: cmd, PipeToStdIn: stdin, CmdArgs: args}
cmdOut := ExecuteCommand(cmdStruct)
if cmdOut.ExitCode != 0 {
k8slogger.Errorf("Failed to execute stdin: kubectl %v : %v", args, cmdOut.Err.Error())
k8slogger.Errorf("Failed to execute stdin: %s %v : %v", cmd, args, cmdOut.Err.Error())
if strings.Contains(cmdOut.Err.Error(), "(NotFound)") || strings.Contains(cmdOut.StdErr, "(NotFound)") {
return cmdOut.StdErr, errors.NewNotFound(schema.GroupResource{}, "")
}
return cmdOut.StdErr, fmt.Errorf("Failed to run stdin: kubectl %v : %v", args, cmdOut.StdErr)
return cmdOut.StdErr, fmt.Errorf("Failed to run stdin: %s %v : %v", cmd, args, cmdOut.StdErr)
}
if cmdOut.StdOut == "" {
return cmdOut.StdErr, nil
@@ -170,7 +183,7 @@ func getKubeConfig(executor exec.Executor) (*rest.Config, error) {
// Parse the kubectl context to get the settings for client connections
var kc kubectlContext
if err := json.Unmarshal([]byte(context), &kc); err != nil {
return nil, fmt.Errorf("failed to unmarshal kubectl config: %+v", err)
return nil, fmt.Errorf("failed to unmarshal %s config: %+v", cmd, err)
}
// find the current context
@@ -282,7 +295,7 @@ func (k8sh *K8sHelper) ExecWithRetry(retries int, namespace, podName, command st
time.Sleep(3 * time.Second)
}
}
return "", fmt.Errorf("kubectl exec command %s failed on pod %s in namespace %s. %+v", command, podName, namespace, err)
return "", fmt.Errorf("%s exec command %s failed on pod %s in namespace %s. %+v", cmd, command, podName, namespace, err)
}
// ResourceOperationFromTemplate performs a kubectl action from a template file after replacing its context
+5 -3
View File
@@ -84,9 +84,9 @@ func (suite *SmokeSuite) SetupSuite() {
smokeTestCluster := TestCluster{
namespace: suite.namespace,
storeType: "bluestore",
storageClassName: "",
storageClassName: installer.StorageClassName(),
useHelm: false,
usePVC: false,
usePVC: installer.UsePVC(),
mons: 3,
rbdMirrorWorkers: 1,
rookCephCleanup: true,
@@ -118,7 +118,9 @@ func (suite *SmokeSuite) TestFileStorage_SmokeTest() {
}
func (suite *SmokeSuite) TestObjectStorage_SmokeTest() {
runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
if !utils.IsPlatformOpenShift() {
runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
}
}
// Test to make sure all rook components are installed and Running