feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #972 from dangula/pr_test_multiClusterDeploy

Browse Source 
creating tests for multi cluster rook install
This commit is contained in:
Travis Nielsen authored and GitHub committed 2017-09-13 16:39:21 -07:00
2 parents bf74619fa5 9e08afb015
commit 772a138e13
27 files changed
+1006 -841
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Jenkinsfile
Vendored
+1 -1
View File
@@ -29,7 +29,7 @@ pipeline {
def data = [
"aws_ci": "v1.6.7",
"gce_ci": "v1.7.4"
"gce_ci": "v1.7.5"
]
testruns = [:]
for (kv in mapToList(data)) {
tests/README.md
+21 -6
View File
@@ -22,7 +22,7 @@ The framework also provides scripts to install Kubernetes. There are two scripts
2. Kubeadm : run [kubeadm.sh](/tests/scripts/kubeadm.sh) to setup a single-node K8s cluster using kubeadm (recommended for ubuntu)
These start-up scripts are designed to start any version of kubernetes, Based on `KUBE_VERSION` environment variable
By default they start kubernetes version v1.7.4.(except minikube which starts kubernetes v1.7.2)
By default they start kubernetes version v1.7.5.
e.g. To kubernetes version v1.6.7
```
export KUBE_VERSION=v1.6.7
@@ -31,8 +31,13 @@ tests/scripts/kubeadm.sh up
To stop the cluster : `tests/scripts/kubeadm.sh clean` or `tests/scripts/miniube.sh clean`
All of the above scripts are designed to start kubernetes, make sure rbd is working, and copy the latest rook and toolbox images
to the kubernetes containers.
To copy rook images generated by build into minikube vm, run the following commands after `minikube.sh up`
```
tests/scripts/minikube.sh update
tests/scripts/minikube.sh helm
```
kubeadm.sh starts kubernetes on the host, so all the images generated by build should be available to the cluster.
### Install Helm
Use [helm.sh](/tests/scripts/helm.sh) to install helm and set up rook charts defined under `_output/charts` (generated by build).
@@ -46,9 +51,19 @@ should be run from project root. eg* `tests/script/kubeadm.sh up`
From the root do the following:
1. Build rook: `make build`
2. Start kubernetes using one of the following:
- `tests/scripts/minikube.sh up`
- `tests/scripts/kubeadm.sh up`
- `tests/scripts/helm.sh up`
- using Kubeadm
```
tests/scripts/kubeadm.sh up
tests/scripts/helm.sh up
```
- using minikube
```
tests/scripts/minikube.sh up
tests/scripts/minikube.sh update
tests/scripts/minikube.sh helm
tests/scripts/helm.sh up
```
3. Run integration tests: `make test-integration`
tests/framework/clients/object.go
+2 -2
View File
@@ -34,8 +34,8 @@ func CreateObjectOperation(rookRestClient contracts.RestAPIOperator) *ObjectOper
//ObjectCreate Function to create a object store in rook
//Input paramatres -None
//Output - output returned by rook Rest API client
func (ro *ObjectOperation) ObjectCreate() (string, error) {
store := model.ObjectStore{Name: "default", RGWReplicas: 1}
func (ro *ObjectOperation) ObjectCreate(name string, replicaCount int32) (string, error) {
store := model.ObjectStore{Name: name, RGWReplicas: replicaCount}
store.DataConfig.ReplicationConfig.Size = 1
store.MetadataConfig.ReplicationConfig.Size = 1
return ro.restClient.CreateObjectStore(store)
tests/framework/clients/rest_client.go
+16 -12
View File
@@ -34,23 +34,28 @@ type RestAPIClient struct {
}
//CreateRestAPIClient Create Rook REST API client
func CreateRestAPIClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper) *RestAPIClient {
func CreateRestAPIClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper, namespace string) *RestAPIClient {
var endpoint string
switch {
case platform == enums.Kubernetes:
//Start rook_api_external server via nodePort if not it not already running.
_, err := k8sHelper.GetService("rook-api-external")
externalAPIservice := "rook-api-external-" + namespace
_, err := k8sHelper.GetService(externalAPIservice, namespace)
if err != nil {
_, err = k8sHelper.ResourceOperation("create", getExternalAPIServiceDefinition())
_, err = k8sHelper.ResourceOperation("create", getExternalAPIServiceDefinition(namespace))
if err != nil {
panic(fmt.Errorf("failed to kubectl create -f - %v: %+v", getExternalAPIServiceDefinition(), err))
panic(fmt.Errorf("failed to kubectl create -f - %v: %+v", getExternalAPIServiceDefinition(namespace), err))
}
}
apiIP, err := k8sHelper.GetPodHostID("rook-api", "rook")
apiIP, err := k8sHelper.GetPodHostID("rook-api", namespace)
if err != nil {
panic(fmt.Errorf("Host Ip for Rook-api service not found. %+v", err))
}
endpoint = "http://" + apiIP + ":30002"
nodePort, err := k8sHelper.GetServiceNodePort(externalAPIservice, namespace)
if err != nil {
panic(fmt.Errorf("port for external Rook-api service not found. %+v", err))
}
endpoint = fmt.Sprintf("http://%s:%s", apiIP, nodePort)
case platform == enums.StandAlone:
endpoint = "http://localhost:8124"
default:
@@ -190,24 +195,23 @@ func (a *RestAPIClient) DeleteObjectUser(id string) error {
}
func getExternalAPIServiceDefinition() string {
func getExternalAPIServiceDefinition(namespace string) string {
return `apiVersion: v1
kind: Service
metadata:
name: rook-api-external
namespace: rook
name: rook-api-external-` + namespace + `
namespace: ` + namespace + `
labels:
app: rook-api
rook_cluster: rook
rook_cluster: ` + namespace + `
spec:
ports:
- name: rook-api
port: 8124
protocol: TCP
nodePort: 30002
selector:
app: rook-api
rook_cluster: rook
rook_cluster: ` + namespace + `
sessionAffinity: None
type: NodePort
`
tests/framework/clients/test_client.go
+2 -2
View File
@@ -44,12 +44,12 @@ const (
)
//CreateTestClient creates new instance of test client for a platform
func CreateTestClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper) (*TestClient, error) {
func CreateTestClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper, namespace string) (*TestClient, error) {
var blockClient contracts.BlockOperator
var fsClient contracts.FileSystemOperator
var objectClient contracts.ObjectOperator
var poolClient contracts.PoolOperator
rookRestClient := CreateRestAPIClient(platform, k8sHelper)
rookRestClient := CreateRestAPIClient(platform, k8sHelper, namespace)
switch {
case platform == enums.Kubernetes:
tests/framework/contracts/operations.go
+1 -1
View File
@@ -42,7 +42,7 @@ type FileSystemOperator interface {
//ObjectOperator - interface for rook object operations
type ObjectOperator interface {
ObjectCreate() (string, error)
ObjectCreate(name string, replicaCount int32) (string, error)
ObjectBucketList() ([]model.ObjectBucket, error)
ObjectConnection() (*model.ObjectStoreConnectInfo, error)
ObjectCreateUser(userid string, displayname string) (*model.ObjectUser, error)
tests/framework/installer/install_data.go
+10 -7
View File
@@ -27,7 +27,8 @@ func NewK8sInstallData() *InstallData {
return &InstallData{}
}
func (i *InstallData) getRookOperator(k8sVersion string) string {
//GetRookOperator returns rook Operator manifest
func (i *InstallData) GetRookOperator(k8sVersion string) string {
if strings.Contains(k8sVersion, "v1.5") {
return `apiVersion: extensions/v1beta1
@@ -170,17 +171,18 @@ spec:
}
func (i *InstallData) getRookCluster() string {
//GetRookCluster returns rook-cluster manifest
func (i *InstallData) GetRookCluster(namespace string) string {
return `apiVersion: v1
kind: Namespace
metadata:
name: rook
name: ` + namespace + `
---
apiVersion: rook.io/v1alpha1
kind: Cluster
metadata:
name: rook
namespace: rook
name: ` + namespace + `
namespace: ` + namespace + `
spec:
versionTag: master
dataDirHostPath:
@@ -241,12 +243,13 @@ spec:
# deviceFilter: "^sd."`
}
func (i *InstallData) getRookToolBox() string {
//GetRookToolBox returns rook-toolbox manifest
func (i *InstallData) GetRookToolBox(namespace string) string {
return `apiVersion: v1
kind: Pod
metadata:
name: rook-tools
namespace: rook
namespace: ` + namespace + `
spec:
containers:
- name: rook-tools
tests/framework/installer/install_helper.go
+55 -94
View File
@@ -59,8 +59,10 @@ func init() {
//CreateK8sRookOperator creates rook-operator via kubectl
func (h *InstallHelper) CreateK8sRookOperator() (err error) {
logger.Infof("Starting Rook Operator")
//creating clusterrolebinding for kubeadm env.
h.k8shelper.CreateAnonSystemClusterBinding()
rookOperator := h.installData.getRookOperator(h.k8shelper.GetK8sServerVersion())
rookOperator := h.installData.GetRookOperator(h.k8shelper.GetK8sServerVersion())
_, err = h.k8shelper.KubectlWithStdin(rookOperator, createArgs...)
@@ -84,10 +86,10 @@ func (h *InstallHelper) CreateK8sRookOperator() (err error) {
return nil
}
//CreateK8sRookOperatorViaHelm creates rook operator via Execute chart named local/rook present in local repo
func (h *InstallHelper) CreateK8sRookOperatorViaHelm() (err error) {
//CreateK8sRookOperatorViaHelm creates rook operator via Helm chart named local/rook present in local repo
func (h *InstallHelper) CreateK8sRookOperatorViaHelm(namespace string) (err error) {
//creating clusterrolebinding for kubeadm env.
h.k8shelper.Kubectl([]string{"create", "clusterrolebinding", "anon-user-access", "--clusterrole", "cluster-admin", "--user", "system:anonymous"}...)
h.k8shelper.CreateAnonSystemClusterBinding()
helmTag, err := h.helmHelper.GetLocalRookHelmChartVersion(helmChartName)
@@ -95,7 +97,7 @@ func (h *InstallHelper) CreateK8sRookOperatorViaHelm() (err error) {
return fmt.Errorf("Failed to get Version of helm chart %v, err : %v", helmChartName, err)
}
err = h.helmHelper.InstallLocalRookHelmChart(helmChartName, helmDeployName, helmTag, "rook")
err = h.helmHelper.InstallLocalRookHelmChart(helmChartName, helmDeployName, helmTag, namespace)
if err != nil {
return fmt.Errorf("failed toinstall rook operator via helm, err : %v", err)
@@ -117,10 +119,10 @@ func (h *InstallHelper) CreateK8sRookOperatorViaHelm() (err error) {
}
//CreateK8sRookToolbox creates rook-tools via kubectl
func (h *InstallHelper) CreateK8sRookToolbox() (err error) {
func (h *InstallHelper) CreateK8sRookToolbox(clusterNamespace string) (err error) {
logger.Infof("Starting Rook toolbox")
rookToolbox := h.installData.getRookToolBox()
rookToolbox := h.installData.GetRookToolBox(clusterNamespace)
_, err = h.k8shelper.KubectlWithStdin(rookToolbox, createArgs...)
@@ -128,7 +130,7 @@ func (h *InstallHelper) CreateK8sRookToolbox() (err error) {
return fmt.Errorf("Failed to create rook-toolbox pod : %v ", err)
}
if !h.k8shelper.IsPodRunningInNamespace("rook-tools") {
if !h.k8shelper.IsPodRunning("rook-tools", clusterNamespace) {
return fmt.Errorf("Rook Toolbox couldn't start")
}
logger.Infof("Rook Toolbox started")
@@ -137,10 +139,10 @@ func (h *InstallHelper) CreateK8sRookToolbox() (err error) {
}
//CreateK8sRookCluster creates rook cluster via kubectl
func (h *InstallHelper) CreateK8sRookCluster() (err error) {
func (h *InstallHelper) CreateK8sRookCluster(clusterNamespace string) (err error) {
logger.Infof("Starting Rook Cluster")
rookCluster := h.installData.getRookCluster()
rookCluster := h.installData.GetRookCluster(clusterNamespace)
_, err = h.k8shelper.KubectlWithStdin(rookCluster, createArgs...)
@@ -148,7 +150,7 @@ func (h *InstallHelper) CreateK8sRookCluster() (err error) {
return fmt.Errorf("Failed to create rook cluster : %v ", err)
}
if !h.k8shelper.IsServiceUpInNameSpace("rook-api") {
if !h.k8shelper.IsServiceUp("rook-api", clusterNamespace) {
logger.Infof("Rook Cluster couldn't start")
} else {
logger.Infof("Rook Cluster started")
@@ -158,9 +160,7 @@ func (h *InstallHelper) CreateK8sRookCluster() (err error) {
}
//InstallRookOnK8s installs rook on k8s
func (h *InstallHelper) InstallRookOnK8s() (err error) {
//creating clusterrolebinding for kubeadm env.
h.k8shelper.Kubectl([]string{"create", "clusterrolebinding", "anon-user-access", "--clusterrole", "cluster-admin", "--user", "system:anonymous"}...)
func (h *InstallHelper) InstallRookOnK8s(clusterNamespace string) (err error) {
//flag used for local debuggin purpose, when rook is pre-installed
skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true")
@@ -191,7 +191,7 @@ func (h *InstallHelper) InstallRookOnK8s() (err error) {
time.Sleep(10 * time.Second)
//Create rook cluster
err = h.CreateK8sRookCluster()
err = h.CreateK8sRookCluster(clusterNamespace)
if err != nil {
panic(err)
}
@@ -199,16 +199,16 @@ func (h *InstallHelper) InstallRookOnK8s() (err error) {
time.Sleep(5 * time.Second)
//Create rook client
err = h.CreateK8sRookToolbox()
err = h.CreateK8sRookToolbox(clusterNamespace)
if err != nil {
panic(err)
}
logger.Infof("installed rook on k8s %s", h.Env.K8sVersion)
logger.Infof("installed rook operator and cluster : %s on k8s %s", clusterNamespace, h.Env.K8sVersion)
return nil
}
//UninstallRookFromK8s uninstalls rook from k8s
func (h *InstallHelper) UninstallRookFromK8s() {
func (h *InstallHelper) UninstallRookFromK8s(clusterNamespace string, helmInstalled bool) {
//flag used for local debugging purpose, when rook is pre-installed
skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true")
if skipRookInstall {
@@ -226,17 +226,24 @@ func (h *InstallHelper) UninstallRookFromK8s() {
if err != nil {
panic(err)
}
rookOperator := h.installData.getRookOperator(k8sVersion)
if helmInstalled {
err = h.helmHelper.DeleteLocalRookHelmChart(helmDeployName)
if err != nil {
panic(err)
}
} else {
rookOperator := h.installData.GetRookOperator(k8sVersion)
_, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteArgs...)
_, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteArgs...)
if err != nil {
panic(err)
}
}
_, err = k8sHelp.DeleteResource([]string{"-n", clusterNamespace, "cluster", clusterNamespace})
if err != nil {
panic(err)
}
_, err = k8sHelp.DeleteResource([]string{"-n", "rook", "cluster", "rook"})
if err != nil {
panic(err)
}
_, err = k8sHelp.DeleteResource([]string{"-n", "rook", "serviceaccount", "rook-api"})
_, err = k8sHelp.DeleteResource([]string{"-n", clusterNamespace, "serviceaccount", "rook-api"})
if err != nil {
panic(err)
}
@@ -262,103 +269,57 @@ func (h *InstallHelper) UninstallRookFromK8s() {
panic(err)
}
}
_, err = k8sHelp.DeleteResource([]string{"secret", "rook-rook-user"})
_, err = k8sHelp.DeleteResource([]string{"secret", clusterNamespace + "-rook-user"})
if err != nil {
panic(err)
}
_, err = k8sHelp.DeleteResource([]string{"namespace", "rook"})
_, err = k8sHelp.DeleteResource([]string{"namespace", clusterNamespace})
if err != nil {
panic(err)
}
isRookUninstalled := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", "rook")
isNameSpaceDeleted := k8sHelp.WaitUntilNameSpaceIsDeleted("rook")
isRookUninstalled := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterNamespace)
isNameSpaceDeleted := k8sHelp.WaitUntilNameSpaceIsDeleted(clusterNamespace)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
time.Sleep(time.Second * 10)
if isRookUninstalled && isNameSpaceDeleted {
logger.Infof("Rook uninstalled successfully")
logger.Infof("Rook cluster %s uninstalled successfully", clusterNamespace)
return
}
panic(fmt.Errorf("Rook not uninstalled"))
panic(fmt.Errorf("Rook cluster %s not uninstalled", clusterNamespace))
}
//UninstallRookFromK8sViaHelm uninstalls rook from k8s and deletes helm deploy
func (h *InstallHelper) UninstallRookFromK8sViaHelm() {
//flag used for local debugging purpose, when rook is pre-installed
skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true")
if skipRookInstall {
return
}
//CleanupCluster deletes a rook cluster for a namespace
func (h *InstallHelper) CleanupCluster(clusterName string, kv *utilversion.Version) {
serverVersion, err := h.k8shelper.Clientset.Discovery().ServerVersion()
if err != nil {
panic(err)
}
kubeVersion := utilversion.MustParseSemantic(serverVersion.GitVersion)
logger.Infof("Uninstalling Rook")
k8sHelp, err := utils.CreateK8sHelper()
logger.Infof("Uninstalling All Rook Clusters - %s", clusterName)
_, err := h.k8shelper.DeleteResource([]string{"-n", clusterName, "cluster", clusterName})
if err != nil {
logger.Errorf("Rook Cluster %s cannot be deleted,err -> %v", clusterName, err)
panic(err)
}
err = h.helmHelper.DeleteLocalRookHelmChart(helmDeployName)
_, err = h.k8shelper.DeleteResource([]string{"-n", clusterName, "serviceaccount", "rook-api"})
if err != nil {
logger.Errorf("rook-api service account in namespace %s cannot be deleted,err -> %v", clusterName, err)
panic(err)
}
_, err = k8sHelp.DeleteResource([]string{"-n", "rook", "cluster", "rook"})
if err != nil {
panic(err)
}
_, err = k8sHelp.DeleteResource([]string{"-n", "rook", "serviceaccount", "rook-api"})
if err != nil {
panic(err)
}
if kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.6.0")) {
_, err = k8sHelp.DeleteResource([]string{"clusterrole", "rook-api"})
if err != nil {
panic(err)
}
_, err = k8sHelp.DeleteResource([]string{"clusterrolebinding", "rook-api"})
if err != nil {
panic(err)
}
}
if kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.7.0")) {
_, err = k8sHelp.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io"})
if err != nil {
panic(err)
}
} else {
_, err = k8sHelp.DeleteResource([]string{"thirdpartyresources", "cluster.rook.io", "pool.rook.io"})
if err != nil {
panic(err)
}
}
_, err = k8sHelp.DeleteResource([]string{"secret", "rook-rook-user"})
if err != nil {
panic(err)
}
_, err = k8sHelp.DeleteResource([]string{"namespace", "rook"})
_, err = h.k8shelper.DeleteResource([]string{"secret", clusterName + "-rook-user"})
if err != nil {
logger.Errorf("rook-user secret in namespace %s cannot be deleted,err -> %v", clusterName, err)
panic(err)
}
isRookUninstalled := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", "rook")
isNameSpaceDeleted := k8sHelp.WaitUntilNameSpaceIsDeleted("rook")
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
time.Sleep(time.Second * 10)
if isRookUninstalled && isNameSpaceDeleted {
logger.Infof("Rook uninstalled successfully")
return
_, err = h.k8shelper.DeleteResource([]string{"namespace", clusterName})
if err != nil {
logger.Errorf("namespace %s cannot be deleted,err -> %v", clusterName, err)
panic(err)
}
panic(fmt.Errorf("Rook not uninstalled"))
h.k8shelper.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterName)
h.k8shelper.WaitUntilNameSpaceIsDeleted(clusterName)
}
//NewK8sRookhelper creates new instance of InstallHelper
tests/framework/utils/helm_helper.go
+16
View File
@@ -1,3 +1,19 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package utils
import (
tests/framework/utils/k8s_helper.go
+86 -147
View File
@@ -29,7 +29,6 @@ import (
"strconv"
"github.com/coreos/pkg/capnslog"
"github.com/jmoiron/jsonq"
"github.com/rook/rook/pkg/util/exec"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -207,22 +206,6 @@ type kuserContext struct {
} `json:"user"`
}
//GetMonIP returns IP address for a ceph mon pod
func (k8sh *K8sHelper) GetMonIP(mon string) (string, error) {
//kubectl -n rook get pod mon0 -o json|jq ".status.podIP"|
cmdArgs := []string{"-n", "rook", "get", "pod", mon, "-o", "json"}
result, err := k8sh.Kubectl(cmdArgs...)
if err == nil {
data := map[string]interface{}{}
dec := json.NewDecoder(strings.NewReader(result))
dec.Decode(&data)
jq := jsonq.NewQuery(data)
ip, _ := jq.String("status", "podIP")
return fmt.Sprintf("%s:6790", ip), nil
}
return "", fmt.Errorf("Error Getting Monitor IP : %v", err)
}
//ResourceOperationFromTemplate performs a kubectl action from a template file after replacing its context
func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinition string, config map[string]string) (string, error) {
@@ -285,18 +268,18 @@ func (k8sh *K8sHelper) GetResource(args []string) (string, error) {
}
//GetMonitorServices returns all ceph mon pod names
func (k8sh *K8sHelper) GetMonitorServices() (map[string]string, error) {
func (k8sh *K8sHelper) GetMonitorServices(namespace string) (map[string]string, error) {
listOpts := metav1.ListOptions{LabelSelector: "app=rook-ceph-mon"}
args := []string{"-n", "rook", "get", "svc", "-l", "app=rook-ceph-mon", "--no-headers=true"}
result, err := k8sh.Kubectl(args...)
podList, err := k8sh.Clientset.Services(namespace).List(listOpts)
if err != nil {
return nil, fmt.Errorf("Failed to find mon services. %v", err)
logger.Errorf("Cannot get rook monitor pods in namespace %s, err: %v", namespace, err)
return nil, fmt.Errorf("Cannot get rook monitor pods in namespace %s, err: %v", namespace, err)
}
mons := []string{}
for _, svc := range podList.Items {
mons = append(mons, svc.Spec.ClusterIP)
// Get the IP address from the 2nd position in the line
mons, err := parseMonEndpoints(result)
if err != nil {
return nil, err
}
if len(mons) != 3 {
return nil, fmt.Errorf("Unexpected monitors: %+v", mons)
@@ -309,143 +292,71 @@ func (k8sh *K8sHelper) GetMonitorServices() (map[string]string, error) {
}, nil
}
func parseMonEndpoints(input string) ([]string, error) {
lines := strings.Split(input, "\n")
mons := []string{}
for _, line := range lines {
if len(line) == 0 {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
return nil, fmt.Errorf("Missing ip for mon service. %s", line)
}
mons = append(mons, strings.TrimSpace(fields[1]))
}
return mons, nil
}
//IsPodRunning returns true if a Pod is running status or goes to Running status within 90s else returns false
func (k8sh *K8sHelper) IsPodRunning(name string) bool {
args := []string{"get", "pod", name, "-o", "jsonpath='{.status.phase}'"}
func (k8sh *K8sHelper) IsPodRunning(name string, namespace string) bool {
getOpts := metav1.GetOptions{}
inc := 0
for inc < RetryLoop {
result, err := k8sh.Kubectl(args...)
pod, err := k8sh.Clientset.Pods(namespace).Get(name, getOpts)
if err == nil {
if strings.Contains(result, "Running") {
if pod.Status.Phase == "Running" {
return true
}
logger.Infof("Pod %s status: %s", name, result)
}
time.Sleep(RetryInterval * time.Second)
inc++
time.Sleep(RetryInterval * time.Second)
logger.Infof("waiting for pod %s in namespace %s to be running", name, namespace)
}
logger.Infof("Giving up waiting for pod %s to be running", name)
return false
}
//IsPodRunningInNamespace returns true if a Pod in a namespace is running status or goes to Running
// status within 90s else returns false
func (k8sh *K8sHelper) IsPodRunningInNamespace(name string) bool {
args := []string{"get", "pods", "-n", "rook", name, "-o", "jsonpath='{.status.phase}'"}
inc := 0
for inc < RetryLoop {
result, err := k8sh.Kubectl(args...)
if err == nil {
if strings.Contains(result, "Running") {
return true
}
logger.Infof("Pod %s status: %s", name, result)
}
time.Sleep(RetryInterval * time.Second)
inc++
}
logger.Infof("Giving up waiting for pod %s to be running", name)
logger.Infof("Giving up waiting for pod %s in namespace %s to be running", name, namespace)
return false
}
//IsPodTerminated returns true if a Pod is terminated status or goes to Terminated status
// within 90s else returns false\
func (k8sh *K8sHelper) IsPodTerminated(name string) bool {
args := []string{"get", "pods", name}
func (k8sh *K8sHelper) IsPodTerminated(name string, namespace string) bool {
getOpts := metav1.GetOptions{}
inc := 0
for inc < RetryLoop {
_, err := k8sh.Kubectl(args...)
pod, err := k8sh.Clientset.Pods(namespace).Get(name, getOpts)
if err != nil {
k8slogger.Infof("Pod in default namespace terminated: " + name)
k8slogger.Infof("Pod %s in namespace %s terminated ", name, namespace)
return true
}
k8slogger.Infof("waiting for Pod %s in namespace %s to terminated, status : %v", name, namespace, pod.Status.Phase)
time.Sleep(RetryInterval * time.Second)
inc++
}
k8slogger.Infof("Pod in default namespace did not terminated: " + name)
return false
}
//IsPodTerminatedInNamespace returns true if a Pod in a namespace is terminated status
// or goes to Terminated status within 90s else returns false\
func (k8sh *K8sHelper) IsPodTerminatedInNamespace(name string) bool {
args := []string{"get", "-n", "rook", "pods", name}
inc := 0
for inc < RetryLoop {
_, err := k8sh.Kubectl(args...)
if err != nil {
k8slogger.Infof("Pod in rook namespace terminated: " + name)
return true
}
time.Sleep(RetryInterval * time.Second)
inc++
}
k8slogger.Infof("Pod in rook namespace did not terminated: " + name)
k8slogger.Infof("Pod %s in namespace %s did not terminated", name, namespace)
return false
}
//IsServiceUp returns true if a service is up or comes up within 40s, else returns false
func (k8sh *K8sHelper) IsServiceUp(name string) bool {
args := []string{"get", "svc", name}
func (k8sh *K8sHelper) IsServiceUp(name string, namespace string) bool {
getOpts := metav1.GetOptions{}
inc := 0
for inc < RetryLoop {
_, err := k8sh.Kubectl(args...)
_, err := k8sh.Clientset.Services(namespace).Get(name, getOpts)
if err == nil {
k8slogger.Infof("Service in default namespace is up: " + name)
k8slogger.Infof("Service: %s in namespace: %s is up", name, namespace)
return true
}
k8slogger.Infof("waiting for Service %s in namespace %s ", name, namespace)
time.Sleep(RetryInterval * time.Second)
inc++
}
k8slogger.Infof("Service in default namespace is not up: " + name)
return false
}
//IsServiceUpInNameSpace returns true if a service in a namespace is up or comes up within 40s, else returns false
func (k8sh *K8sHelper) IsServiceUpInNameSpace(name string) bool {
args := []string{"get", "svc", "-n", "rook", name}
inc := 0
for inc < RetryLoop {
_, err := k8sh.Kubectl(args...)
if err == nil {
return true
}
time.Sleep(RetryInterval * time.Second)
inc++
}
k8slogger.Infof("Service in rook namespace is not up: " + name)
k8slogger.Infof("Giving up waiting for service: %s in namespace %s ", name, namespace)
return false
}
//GetService returns output from "kubectl get svc $NAME" command
func (k8sh *K8sHelper) GetService(servicename string) (string, error) {
args := []string{"get", "svc", "-n", "rook", servicename}
result, err := k8sh.Kubectl(args...)
func (k8sh *K8sHelper) GetService(servicename string, namespace string) (*v1.Service, error) {
getOpts := metav1.GetOptions{}
result, err := k8sh.Clientset.Services(namespace).Get(servicename, getOpts)
if err != nil {
return "", fmt.Errorf("Cannot find service %v -- %v", servicename, err)
return nil, fmt.Errorf("Cannot find service %s in namespace %s, err-- %v", servicename, namespace, err)
}
return result, nil
}
@@ -500,28 +411,30 @@ func (k8sh *K8sHelper) GetPodDetails(podNamePattern string, namespace string) (s
//GetPodHostID returns HostIP address of a pod
func (k8sh *K8sHelper) GetPodHostID(podNamePattern string, namespace string) (string, error) {
output, err := k8sh.GetPodDetails(podNamePattern, namespace)
listOpts := metav1.ListOptions{LabelSelector: "app=" + podNamePattern}
podList, err := k8sh.Clientset.Pods(namespace).List(listOpts)
if err != nil {
return "", err
logger.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
}
podNames := strings.Split(output, "\n")
if len(podNames) == 0 {
return "", fmt.Errorf("pod %s not found", podNamePattern)
if podList.Size() < 1 {
logger.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
}
return podList.Items[0].Status.HostIP, nil
}
//get host Ip of the pod
args := []string{"get", podNames[0], "-o", "jsonpath='{.status.hostIP}'"}
if namespace != "" {
args = append(args, []string{"-n", namespace}...)
//GetServiceNodePort returns nodeProt of service
func (k8sh *K8sHelper) GetServiceNodePort(serviceName string, namespace string) (string, error) {
getOpts := metav1.GetOptions{}
svc, err := k8sh.Clientset.Services(namespace).Get(serviceName, getOpts)
if err != nil {
logger.Errorf("Cannot get service : %v in namespace %v, err: %v", serviceName, namespace, err)
return "", fmt.Errorf("Cannot get serice : %v in namespace %v, err: %v", serviceName, namespace, err)
}
result, err := k8sh.Kubectl(args...)
if err == nil {
hostIP := strings.Replace(result, "'", "", -1)
return strings.TrimSpace(hostIP), nil
}
return "", fmt.Errorf("Error Getting Monitor IP -- %v", err)
np := svc.Spec.Ports[0].NodePort
return strconv.FormatInt(int64(np), 10), nil
}
//IsStorageClassPresent returns true if storageClass is present, if not false
@@ -578,19 +491,30 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m
return false
}
if podList.Size() < minExpected {
if len(podList.Items) < minExpected {
logger.Errorf("Expected at least %d pods with name %v, actual count %d", minExpected, podName, podList.Size())
return false
}
for _, pod := range podList.Items {
if !(pod.Status.Phase == v1.PodPhase(expectedPhase)) {
logger.Errorf("Expected pod %v to be in %v phase, actual phase %v ", pod.Name, expectedPhase, pod.Status.Phase)
return false
inc := 0
for inc < RetryLoop {
r := true
pl, _ := k8sh.Clientset.Pods(namespace).List(listOpts)
for _, pod := range pl.Items {
if !(pod.Status.Phase == v1.PodPhase(expectedPhase)) {
r = false
logger.Infof("waiting for pod %v to be in %s Phase, currently in %v Phase", pod.Name, expectedPhase, pod.Status.Phase)
}
}
}
if r {
return true
}
inc++
time.Sleep(RetryInterval * time.Second)
return true
}
logger.Errorf("All pods with app Name %v not in %v phase ", podName, expectedPhase)
return false
}
@@ -665,14 +589,18 @@ func (k8sh *K8sHelper) WaitUntilNameSpaceIsDeleted(namespace string) bool {
}
//GetRGWServiceURL returns URL of ceph RGW service in the cluster
func (k8sh *K8sHelper) GetRGWServiceURL() (string, error) {
hostip, err := k8sh.GetPodHostID("rook-ceph-rgw", "rook")
func (k8sh *K8sHelper) GetRGWServiceURL(servicename string, namespace string) (string, error) {
hostip, err := k8sh.GetPodHostID("rook-ceph-rgw", namespace)
if err != nil {
panic(fmt.Errorf("RGW pods not found/object store possibly not started"))
}
//TODO - Get nodePort stop hardcoding
endpoint := hostip + ":30001"
nodePort, err := k8sh.GetServiceNodePort(servicename, namespace)
if err != nil {
panic(fmt.Errorf("RGW pods not found/object store possibly not started"))
}
endpoint := hostip + ":" + nodePort
return endpoint, err
}
@@ -720,7 +648,7 @@ func (k8sh *K8sHelper) GetRookLogs(podAppName string, namespace string, testName
continue
}
}
fileName := testName + "_" + podName + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".log"
fileName := fmt.Sprintf("%s_%s_%s_%d.log", testName, podName, namespace, time.Now().Unix())
fpath = path.Join(fpath, fileName)
file, err := os.Create(fpath)
if err != nil {
@@ -736,3 +664,14 @@ func (k8sh *K8sHelper) GetRookLogs(podAppName string, namespace string, testName
}
}
}
//CreateAnonSystemClusterBinding Creates anon-user-access clusterrolebinding for cluster-admin role - used by kubeadm env.
func (k8sh *K8sHelper) CreateAnonSystemClusterBinding() {
args := []string{"create", "clusterrolebinding", "anon-user-access", "--clusterrole", "cluster-admin", "--user", "system:anonymous"}
_, err := k8sh.Kubectl(args...)
if err != nil {
logger.Warningf("anon-user-access not created")
return
}
logger.Warningf("anon-user-access created")
}
tests/integration/block_createBlock_api_test.go
+3 -3
View File
@@ -55,10 +55,10 @@ func (s *BlockImageCreateSuite) SetupSuite() {
s.installer = installer.NewK8sRookhelper(kh.Clientset)
err = s.installer.InstallRookOnK8s()
err = s.installer.InstallRookOnK8s("rook")
require.NoError(s.T(), err)
s.testClient, err = clients.CreateTestClient(enums.Kubernetes, kh)
s.testClient, err = clients.CreateTestClient(enums.Kubernetes, kh, "rook")
require.Nil(s.T(), err)
s.rc = s.testClient.GetRestAPIClient()
@@ -158,6 +158,6 @@ func (s *BlockImageCreateSuite) TearDownTest() {
}
func (s *BlockImageCreateSuite) TearDownSuite() {
s.installer.UninstallRookFromK8s()
s.installer.UninstallRookFromK8s("rook", false)
}
tests/integration/block_createBlock_k8s_test.go
+3 -3
View File
@@ -58,10 +58,10 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() {
s.installer = installer.NewK8sRookhelper(s.kh.Clientset)
err = s.installer.InstallRookOnK8s()
err = s.installer.InstallRookOnK8s("rook")
require.NoError(s.T(), err)
s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh)
s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh, "rook")
require.Nil(s.T(), err)
s.bc = s.testClient.GetBlockClient()
@@ -238,5 +238,5 @@ func (s *K8sBlockImageCreateSuite) isPVCBound(name string) bool {
}
func (s *K8sBlockImageCreateSuite) TearDownSuite() {
s.installer.UninstallRookFromK8s()
s.installer.UninstallRookFromK8s("rook", false)
}
tests/integration/block_e2e_k8s_test.go
+3 -3
View File
@@ -59,10 +59,10 @@ func (s *K8sBlockEnd2EndIntegrationSuite) SetupSuite() {
s.installer = installer.NewK8sRookhelper(s.kh.Clientset)
err = s.installer.InstallRookOnK8s()
err = s.installer.InstallRookOnK8s("rook")
require.NoError(s.T(), err)
s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh)
s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh, "rook")
require.Nil(s.T(), err)
s.bc = s.testClient.GetBlockClient()
@@ -225,5 +225,5 @@ func (s *K8sBlockEnd2EndIntegrationSuite) TearDownTest() {
func (s *K8sBlockEnd2EndIntegrationSuite) TearDownSuite() {
s.installer.UninstallRookFromK8s()
s.installer.UninstallRookFromK8s("rook", false)
}
tests/longhaul/block_on_k8s_test.go
+2 -2
View File
@@ -45,11 +45,11 @@ func (s *K8sBlockLongHaulSuite) SetupSuite() {
s.installer = installer.NewK8sRookhelper(s.kh.Clientset)
if !s.kh.IsRookInstalled() {
err = s.installer.InstallRookOnK8s()
err = s.installer.InstallRookOnK8s("rook")
require.NoError(s.T(), err)
}
s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh)
s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh, "rook")
require.Nil(s.T(), err)
s.bc = s.testClient.GetBlockClient()
tests/scripts/kubeadm-install.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash +e
KUBE_VERSION=${1:-"v1.7.4"}
KUBE_VERSION=${1:-"v1.7.5"}
null_str=
KUBE_INSTALL_VERSION="${KUBE_VERSION/v/$null_str}"-00
tests/scripts/kubeadm.sh
+1 -1
View File
@@ -5,7 +5,7 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
tarname=image.tar
tarfile=${WORK_DIR}/tests/${tarname}
export KUBE_VERSION=${KUBE_VERSION:-"v1.7.4"}
export KUBE_VERSION=${KUBE_VERSION:-"v1.7.5"}
install() {
tests/scripts/minikube.sh
+4 -2
View File
@@ -4,7 +4,7 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${scriptdir}/../../build/common.sh"
tarname=image.tar
tarfile=${WORK_DIR}/tests/${tarname}
tarfile=$(pwd)/_output/tests/${tarname}
ssh_opts=(
-o StrictHostKeyChecking=no
@@ -52,7 +52,7 @@ copy_image_to_cluster() {
}
# configure dind-cluster
KUBE_VERSION=${KUBE_VERSION:-"v1.7.2"}
KUBE_VERSION=${KUBE_VERSION:-"v1.7.5"}
case "${1:-}" in
up)
@@ -65,6 +65,8 @@ case "${1:-}" in
copy_image_to_cluster ${BUILD_REGISTRY}/rook-amd64 rook/rook:master
copy_image_to_cluster ${BUILD_REGISTRY}/toolbox-amd64 rook/toolbox:master
copy_image_to_cluster ceph/base ceph/base
;;
down)
minikube stop
tests/smoke/base_block_test.go
+277
View File
@@ -0,0 +1,277 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package smoke
import (
"time"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"strings"
)
var (
blockMountPath = "/tmp/rook1"
blockPodName = "block-test"
)
// Smoke Test for Block Storage - Test check the following operations on Block Storage in order
//Create,Mount,Write,Read,Unmount and Delete.
func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
poolName := "replicapool"
storageClassName := "rook-block"
blockName := "block-pv-claim"
defer blockTestDataCleanUp(helper, k8sh, namespace, poolName, storageClassName, blockName)
logger.Infof("Block Storage End to End Integration Test - create, mount, write to, read from, and unmount")
logger.Infof("Running on Rook Cluster %s", namespace)
rbc := helper.GetBlockClient()
logger.Infof("Step 0 : Get Initial List Block")
initBlockImages, _ := rbc.BlockList()
logger.Infof("step 1: Create block storage")
cbErr := blockStorageOperation(helper, k8sh, namespace, poolName, storageClassName, blockName, "create")
require.Nil(s.T(), cbErr)
require.True(s.T(), retryBlockImageCountCheck(helper, k8sh, len(initBlockImages), 1), "Make sure a new block is created")
logger.Infof("Block Storage created successfully")
require.True(s.T(), k8sh.WaitUntilPVCIsBound(blockName), "Make sure PVC is Bound")
logger.Infof("step 2: Mount block storage")
_, mtErr := rbc.BlockMap(getBlockPodDefintion(blockName), blockMountPath)
require.Nil(s.T(), mtErr)
require.True(s.T(), k8sh.IsPodRunning(blockPodName, defaultNamespace), "make sure block-test pod is in running state")
logger.Infof("Block Storage Mounted successfully")
logger.Infof("step 3: Write to block storage")
_, wtErr := rbc.BlockWrite(blockPodName, blockMountPath, "Smoke Test Data form Block storage", "bsFile1", "")
require.Nil(s.T(), wtErr)
logger.Infof("Write to Block storage successfully")
logger.Infof("step 4: Read from block storage")
read, rErr := rbc.BlockRead(blockPodName, blockMountPath, "bsFile1", "")
require.Nil(s.T(), rErr)
require.Contains(s.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged")
logger.Infof("Read from Block storage successfully")
logger.Infof("step 5: Unmount block storage")
_, unmtErr := rbc.BlockUnmap(getBlockPodDefintion(blockName), blockMountPath)
require.Nil(s.T(), unmtErr)
require.True(s.T(), k8sh.IsPodTerminated(blockPodName, defaultNamespace), "make sure block-test pod is terminated")
logger.Infof("Block Storage unmounted successfully")
logger.Infof("step 6: Deleting block storage")
dbErr := blockStorageOperation(helper, k8sh, namespace, poolName, storageClassName, blockName, "delete")
require.Nil(s.T(), dbErr)
require.True(s.T(), retryBlockImageCountCheck(helper, k8sh, len(initBlockImages), 0), "Make sure a block is deleted")
logger.Infof("Block Storage deleted successfully")
}
func runBlockE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterNamespace string) {
logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc")
logger.Infof("Running on Rook Cluster %s", clusterNamespace)
//Check initial number of blocks
defer blockTestDataCleanUp(helper, k8sh, clusterNamespace, "rookpool", "rook-block", "test-block")
bc := helper.GetBlockClient()
initialBlocks, err := bc.BlockList()
require.Nil(s.T(), err)
initBlockCount := len(initialBlocks)
logger.Infof("step 1: Create StorageClass and pool")
sc := map[string]string{}
storageclass := getBlockPoolAndStorageClassDefinition(k8sh, clusterNamespace, "rookpool", "rook-block")
res1, err := k8sh.ResourceOperationFromTemplate("create", storageclass, sc)
require.Contains(s.T(), res1, "pool \"rookpool\" created", "Make sure test pool is created")
require.Contains(s.T(), res1, "storageclass \"rook-block\" created", "Make sure storageclass is created")
require.NoError(s.T(), err)
// see https://github.com/rook/rook/issues/767
time.Sleep(10 * time.Second)
logger.Infof("step 2: Create pvc")
pvc := getPvcDefinition("test-block-claim", "rook-block")
res2, err := k8sh.ResourceOperationFromTemplate("create", pvc, sc)
require.Contains(s.T(), res2, "persistentvolumeclaim \"test-block-claim\" created", "Make sure pvc is created")
require.NoError(s.T(), err)
require.True(s.T(), isPVCBound(k8sh, "test-block-claim"))
//Make sure new block is created
b, _ := bc.BlockList()
require.Equal(s.T(), initBlockCount+1, len(b), "Make sure new block image is created")
//Delete pvc and storageclass
_, err = k8sh.ResourceOperationFromTemplate("delete", storageclass, sc)
require.NoError(s.T(), err)
_, err = k8sh.ResourceOperationFromTemplate("delete", pvc, sc)
require.NoError(s.T(), err)
time.Sleep(2 * time.Second)
b, _ = bc.BlockList()
require.Equal(s.T(), initBlockCount, len(b), "Make sure new block image is deleted")
}
func blockTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace string, poolname string, storageclassname string, blockname string) {
logger.Infof("Cleaning up block storage")
helper.GetBlockClient().BlockUnmap(getBlockPodDefintion(blockname), blockMountPath)
blockStorageOperation(helper, k8sh, namespace, poolname, storageclassname, blockname, "delete")
cleanUpDymanicBlockStorage(helper)
}
func blockStorageOperation(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace string, poolname string, storageclassname string, blockname string, action string) error {
poolStorageClassDef := getBlockPoolAndStorageClassDefinition(k8sh, namespace, poolname, storageclassname)
pvcDef := getPvcDefinition(blockname, storageclassname)
k8sh.ResourceOperation(action, poolStorageClassDef)
// see https://github.com/rook/rook/issues/767
time.Sleep(10 * time.Second)
_, err := k8sh.ResourceOperation(action, pvcDef)
return err
}
// periodically checking if block image count has changed to expected value
// When creating pvc in k8s platform, it may take some time for the block Image to be bounded
func retryBlockImageCountCheck(helper *clients.TestClient, k8sh *utils.K8sHelper, imageCount, expectedChange int) bool {
inc := 0
for inc < utils.RetryLoop {
logger.Infof("Getting list of blocks (expecting %d)", (imageCount + expectedChange))
blockImages, _ := helper.GetBlockClient().BlockList()
if imageCount+expectedChange == len(blockImages) {
return true
}
time.Sleep(time.Second * utils.RetryInterval)
inc++
}
return false
}
//CleanUpDymanicBlockStorage is helper method to clean up bock storage created by tests
func cleanUpDymanicBlockStorage(helper *clients.TestClient) {
// Delete storage pool, storage class and pvc
blockImagesList, _ := helper.GetBlockClient().BlockList()
for _, blockImage := range blockImagesList {
helper.GetRestAPIClient().DeleteBlockImage(blockImage)
}
}
func getBlockPoolAndStorageClassDefinition(k8sh *utils.K8sHelper, namespace string, poolName string, storageclassName string) string {
k8sVersion := k8sh.GetK8sServerVersion()
if strings.Contains(k8sVersion, "v1.5") {
return `apiVersion: rook.io/v1alpha1
kind: Pool
metadata:
name: ` + poolName + `
namespace: ` + namespace + `
spec:
replication:
size: 1
# For an erasure-coded pool, comment out the replication size above and uncomment the following settings.
# Make sure you have enough OSDs to support the replica size or erasure code chunks.
#erasureCode:
# codingChunks: 2
# dataChunks: 2
---
apiVersion: storage.k8s.io/v1beta1
kind: StorageClass
metadata:
name: ` + storageclassName + `
provisioner: rook.io/block
parameters:
pool: ` + poolName + `
clusterName: ` + namespace + `
clusterNamespace: ` + namespace
}
return `apiVersion: rook.io/v1alpha1
kind: Pool
metadata:
name: ` + poolName + `
namespace: ` + namespace + `
spec:
replication:
size: 1
# For an erasure-coded pool, comment out the replication count above and uncomment the following settings.
# Make sure you have enough OSDs to support the replica count or erasure code chunks.
#erasureCode:
# codingChunks: 2
# dataChunks: 2
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ` + storageclassName + `
provisioner: rook.io/block
parameters:
pool: ` + poolName + `
clusterName: ` + namespace + `
clusterNamespace: ` + namespace
}
func getPvcDefinition(blockName string, storageclassName string) string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ` + blockName + `
spec:
storageClassName: ` + storageclassName + `
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1M`
}
func getBlockPodDefintion(blockName string) string {
return `apiVersion: v1
kind: Pod
metadata:
name: block-test
spec:
containers:
- image: busybox
name: block-test1
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
volumeMounts:
- name: block-persistent-storage
mountPath: /tmp/rook1
volumes:
- name: block-persistent-storage
persistentVolumeClaim:
claimName: ` + blockName + `
restartPolicy: Never`
}
func isPVCBound(k8sh *utils.K8sHelper, name string) bool {
inc := 0
for inc < utils.RetryLoop {
status, _ := k8sh.GetPVCStatus(name)
if strings.TrimRight(status, "\n") == "'Bound'" {
return true
}
time.Sleep(time.Second * utils.RetryInterval)
inc++
}
return false
}
tests/smoke/base_deploy_test.go
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package smoke
import (
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
var (
logger = capnslog.NewPackageLogger("github.com/rook/rook", "integrationTest")
defaultNamespace = "default"
defaultRookNamespace = "test-rook"
clusterNamespace1 = "test-cluster-1"
clusterNamespace2 = "test-cluster-2"
)
//Test to make sure all rook components are installed and Running
func checkIfRookClusterIsInstalled(k8sh *utils.K8sHelper, s suite.Suite, oNamespace string, cNamespace string) {
logger.Infof("Make sure all Pods in Rook Cluster %s are running", cNamespace)
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-operator", oNamespace, 1, "Running"),
"Make sure there is 1 rook-operator present in Running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-api", cNamespace, 1, "Running"),
"Make sure there is 1 rook-api present in Running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", cNamespace, 1, "Running"),
"Make sure there is 1 rook-ceph-mgr present in Running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-osd", cNamespace, 1, "Running"),
"Make sure there is at lest 1 rook-ceph-osd present in Running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mon", cNamespace, 3, "Running"),
"Make sure there are 3 rook-ceph-mon present in Running state")
}
func gatherAllRookLogs(k8sh *utils.K8sHelper, s suite.Suite, oNamespace string, cNamespace string) {
logger.Infof("Gathering all logs from Rook Cluster %s", cNamespace)
k8sh.GetRookLogs("rook-operator", oNamespace, s.T().Name())
k8sh.GetRookLogs("rook-api", cNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-mgr", cNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-mon", cNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-osd", cNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-rgw", cNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-mds", cNamespace, s.T().Name())
}
tests/smoke/base_file_test.go
+152
View File
@@ -0,0 +1,152 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package smoke
import (
"fmt"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
var (
fileMountPath = "/tmp/rookfs"
filePodName = "file-test"
)
// Smoke Test for File System Storage - Test check the following operations on FileSystem Storage in order
//Create,Mount,Write,Read,Unmount and Delete.
func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, fileSystemName string) {
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, fileSystemName, fileMountPath)
logger.Infof("Running on Rook Cluster %s", namespace)
logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount")
rfc := helper.GetFileSystemClient()
logger.Infof("Step 1: Create file System")
_, fscErr := rfc.FSCreate(fileSystemName)
require.Nil(s.T(), fscErr)
fileSystemList, _ := rfc.FSList()
require.Equal(s.T(), 1, len(fileSystemList), "There should one shared file system present")
filesystemData := fileSystemList[0]
require.Equal(s.T(), fileSystemName, filesystemData.Name, "make sure filesystem name matches")
logger.Infof("File system created")
logger.Infof("Step 2: Mount file System")
mtfsErr := podWithFileSystem(k8sh, s, filePodName, namespace, fileSystemName, fileMountPath, "create")
require.Nil(s.T(), mtfsErr)
require.True(s.T(), k8sh.IsPodRunning(filePodName, defaultRookNamespace), "make sure file-test pod is in running state")
logger.Infof("File system mounted successfully")
logger.Infof("Step 3: Write to file system")
_, wfsErr := rfc.FSWrite(filePodName, fileMountPath, "Smoke Test Data for file system storage", "fsFile1", namespace)
require.Nil(s.T(), wfsErr)
logger.Infof("Write to file system successful")
logger.Infof("Step 4: Read from file system")
read, rdErr := rfc.FSRead(filePodName, fileMountPath, "fsFile1", namespace)
require.Nil(s.T(), rdErr)
require.Contains(s.T(), read, "Smoke Test Data for file system storage", "make sure content of the files is unchanged")
logger.Infof("Read from file system successful")
logger.Infof("Step 5: UnMount file System")
umtfsErr := podWithFileSystem(k8sh, s, filePodName, namespace, fileSystemName, fileMountPath, "delete")
require.Nil(s.T(), umtfsErr)
require.True(s.T(), k8sh.IsPodTerminated(filePodName, defaultRookNamespace), "make sure file-test pod is terminated")
logger.Infof("File system mounted successfully")
logger.Infof("Step 6: Deleting file storage")
rfc.FSDelete(fileSystemName)
//Delete is not deleting filesystem - known issue
//require.Nil(suite.T(), fsd_err)
logger.Infof("File system deleted")
}
//Test File System Creation on Rook that was installed on a custom namespace i.e. Namespace != "rook"
func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, fileSystemName string) {
logger.Infof("File Storage End to End Integration Test - create FileSystem and make sure mds pod is running")
logger.Infof("Running on Rook Cluster %s", namespace)
fc := helper.GetFileSystemClient()
logger.Infof("Step 1: Create file System")
_, fscErr := fc.FSCreate(fileSystemName)
require.Nil(s.T(), fscErr)
fileSystemList, _ := fc.FSList()
require.Equal(s.T(), 1, len(fileSystemList), "There should one shared file system present")
logger.Infof("Step 2: Make sure rook-ceph-mds pod is running")
require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-mds", namespace, "Running"),
"Make sure rook-ceph-mds is in running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mds", namespace, 1, "Running"),
"Make sure there is 1 rook-ceph-mds present in Running state")
}
func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, fileSystemName string, fileMountPath string) {
logger.Infof("Cleaning up file system")
podWithFileSystem(k8sh, s, podname, namespace, fileSystemName, fileMountPath, "delete")
helper.GetFileSystemClient().FSDelete(fileSystemName)
}
func podWithFileSystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error {
mons, err := k8sh.GetMonitorServices(namespace)
require.Nil(s.T(), err)
logger.Infof("mountFileStorage: Mons: %+v", mons)
_, err = k8sh.ResourceOperationFromTemplate(action, getFileSystemTestPod(podname, namespace, filesystemName, fileMountPath), mons)
if err != nil {
return fmt.Errorf("failed to %s pod -- %s. %+v", action, getFileSystemTestPod(podname, namespace, filesystemName, fileMountPath), err)
}
return nil
}
func getFileSystemTestPod(podname string, namespace string, filesystemName string, fileMountPath string) string {
return `apiVersion: v1
kind: Pod
metadata:
name: ` + podname + `
namespace: ` + namespace + `
spec:
containers:
- name: ` + podname + `
image: busybox
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
env:
volumeMounts:
- mountPath: "` + fileMountPath + `"
name: ` + filesystemName + `
volumes:
- name: ` + filesystemName + `
cephfs:
monitors:
- {{.mon0}}
- {{.mon1}}
- {{.mon2}}
user: admin
secretRef:
name: rook-admin
readOnly: false
restartPolicy: Never
`
}
tests/smoke/object_test.go → tests/smoke/base_object_test.go
+70 -45
View File
@@ -23,8 +23,11 @@ import (
"time"
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
var (
@@ -39,97 +42,119 @@ var (
// Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order
//Create object store, Create User, Connect to Object Store, Create Bucket, Read/Write/Delete to bucket,Delete Bucket and
//Delete user
func (suite *SmokeSuite) TestObjectStorage_SmokeTest() {
defer suite.objectTestDataCleanUp()
oc := suite.helper.GetObjectClient()
func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
defer objectTestDataCleanUp(helper, k8sh, namespace)
oc := helper.GetObjectClient()
logger.Infof("Object Storage Smoke Test - Create Object Store, User,Bucket and read/write to bucket")
logger.Infof("Object Storage End To End Integration Test - Create Object Store, User,Bucket and read/write to bucket")
logger.Infof("Running on Rook Cluster %s", namespace)
logger.Infof("Step 0 : Create Object Store")
cobsErr := suite.createObjectStore()
require.Nil(suite.T(), cobsErr)
cobsErr := createObjectStore(helper, k8sh, namespace, "default", 1)
require.Nil(s.T(), cobsErr)
logger.Infof("Object store created successfully")
logger.Infof("Step 1 : Create Object Store User")
initialUsers, _ := oc.ObjectListUser()
_, cosuErr := oc.ObjectCreateUser(userid, userdisplayname)
require.Nil(suite.T(), cosuErr)
require.Nil(s.T(), cosuErr)
usersAfterCrate, _ := oc.ObjectListUser()
require.Equal(suite.T(), len(initialUsers)+1, len(usersAfterCrate), "Make sure user list count is increaded by 1")
require.Equal(s.T(), len(initialUsers)+1, len(usersAfterCrate), "Make sure user list count is increaded by 1")
getuserData, guErr := oc.ObjectGetUser(userid)
require.Nil(suite.T(), guErr)
require.Equal(suite.T(), userid, getuserData.UserID, "Check user id returned")
require.Equal(suite.T(), userdisplayname, *getuserData.DisplayName, "Check user name returned")
require.Nil(s.T(), guErr)
require.Equal(s.T(), userid, getuserData.UserID, "Check user id returned")
require.Equal(s.T(), userdisplayname, *getuserData.DisplayName, "Check user name returned")
logger.Infof("Object store user created successfully")
logger.Infof("Step 2 : Get connection information")
conninfo, conninfoError := oc.ObjectGetUser(userid)
require.Nil(suite.T(), conninfoError)
s3endpoint, _ := suite.k8sh.GetRGWServiceURL()
require.Nil(s.T(), conninfoError)
rgwExternalServiceName := "rgw-external-" + namespace
s3endpoint, _ := k8sh.GetRGWServiceURL(rgwExternalServiceName, defaultRookNamespace)
s3client := utils.CreateNewS3Helper(s3endpoint, *conninfo.AccessKey, *conninfo.SecretKey)
logger.Infof("Step 3 : Create bucket")
initialBuckets, _ := oc.ObjectBucketList()
s3client.CreateBucket(bucketname)
BucketsAfterCreate, _ := oc.ObjectBucketList()
require.Equal(suite.T(), len(initialBuckets)+1, len(BucketsAfterCreate), "Make sure new bucket is created")
require.Equal(s.T(), len(initialBuckets)+1, len(BucketsAfterCreate), "Make sure new bucket is created")
bkt, _ := getBucket(bucketname, BucketsAfterCreate)
require.Equal(suite.T(), bucketname, bkt.Name)
require.Equal(suite.T(), userid, bkt.Owner)
require.Equal(s.T(), bucketname, bkt.Name)
require.Equal(s.T(), userid, bkt.Owner)
logger.Infof("Bucket created in Object store successfully")
logger.Infof("Step 4 : Put Object on bucket")
initObjSize, initObjNum, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterCreate)
require.Equal(suite.T(), uint64(0), initObjSize)
require.Equal(suite.T(), uint64(0), initObjNum)
require.Equal(s.T(), uint64(0), initObjSize)
require.Equal(s.T(), uint64(0), initObjNum)
_, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType)
require.Nil(suite.T(), poErr)
require.Nil(s.T(), poErr)
BucketsAfterPut, _ := oc.ObjectBucketList()
ObjSize, ObjNum, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterPut)
require.NotEmpty(suite.T(), ObjSize)
require.Equal(suite.T(), uint64(1), ObjNum)
require.NotEmpty(s.T(), ObjSize)
require.Equal(s.T(), uint64(1), ObjNum)
logger.Infof("Object Created on bucket successfully")
logger.Infof("Step 5 : Put Object from bucket")
read, err := s3client.GetObjectInBucket(bucketname, objectKey)
require.Nil(suite.T(), err)
require.Equal(suite.T(), objBody, read)
require.Nil(s.T(), err)
require.Equal(s.T(), objBody, read)
logger.Infof("Object retrieved from bucket successfully")
logger.Infof("Step 6 : Delete Object on bucket")
_, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey)
require.Nil(suite.T(), delobjErr)
require.Nil(s.T(), delobjErr)
BucketsAfterOjbDelete, _ := oc.ObjectBucketList()
ObjSize1, ObjNum1, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterOjbDelete)
require.Equal(suite.T(), uint64(0), ObjSize1)
require.Equal(suite.T(), uint64(0), ObjNum1)
require.Equal(s.T(), uint64(0), ObjSize1)
require.Equal(s.T(), uint64(0), ObjNum1)
logger.Infof("Object deleted on bucket successfully")
logger.Infof("Step 6 : Delete bucket")
_, bkdelErr := s3client.DeleteBucket(bucketname)
require.Nil(suite.T(), bkdelErr)
require.Nil(s.T(), bkdelErr)
BucketsAfterDelete, _ := oc.ObjectBucketList()
require.Equal(suite.T(), len(initialBuckets), len(BucketsAfterDelete), "Make sure new bucket is deleted")
require.Equal(s.T(), len(initialBuckets), len(BucketsAfterDelete), "Make sure new bucket is deleted")
logger.Infof("Bucket deleted successfully")
logger.Infof("Step 7 : Delete User")
usersBeforeDelete, _ := oc.ObjectListUser()
oc.ObjectDeleteUser(userid)
usersAfterDelete, _ := oc.ObjectListUser()
require.Equal(suite.T(), len(usersBeforeDelete)-1, len(usersAfterDelete), "Make sure user list count is reducd by 1")
require.Equal(s.T(), len(usersBeforeDelete)-1, len(usersAfterDelete), "Make sure user list count is reducd by 1")
logger.Infof("Object store user deleted successfully")
}
func (suite *SmokeSuite) objectTestDataCleanUp() {
//Test Object StoreCreation on Rook that was installed via helm
func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, name string, replicaSize int) {
logger.Infof("Object Storage End To End Integration Test - Create Object Store and check if rgw service is Running")
logger.Infof("Running on Rook Cluster %s", namespace)
logger.Infof("Step 1 : Create Object Store")
oc := helper.GetObjectClient()
oc.ObjectCreate(name, int32(replicaSize))
logger.Infof("Step 2 : check rook-ceph-rgw service status and count")
require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", namespace, "Running"),
"Make sure rook-ceph-rgw is in running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"),
"Make sure all rook-ceph-rgw pods are in Running state")
require.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace))
}
func objectTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace string) {
rgwExternalServiceName := "rgw-external-" + namespace
logger.Infof("Cleaning up object store")
oc := suite.helper.GetObjectClient()
oc := helper.GetObjectClient()
userinfo, err := oc.ObjectGetUser(userid)
if err != nil {
return //when user is not found
}
s3endpoint, _ := suite.k8sh.GetRGWServiceURL()
s3endpoint, _ := k8sh.GetRGWServiceURL(rgwExternalServiceName, namespace)
s3client := utils.CreateNewS3Helper(s3endpoint, *userinfo.AccessKey, *userinfo.SecretKey)
s3client.DeleteObjectInBucket(bucketname, objectKey)
s3client.DeleteBucket(bucketname)
@@ -148,14 +173,15 @@ func getBucket(bucketname string, bucketList []model.ObjectBucket) (model.Object
return model.ObjectBucket{}, errors.New("Bucket not found")
}
func (suite *SmokeSuite) createObjectStore() error {
suite.helper.GetObjectClient().ObjectCreate()
func createObjectStore(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace string, name string, replicaSize int32) error {
rgwExternalServiceName := "rgw-external-" + namespace
helper.GetObjectClient().ObjectCreate(name, replicaSize)
time.Sleep(time.Second * 2) //wait for rgw service to to started
if suite.k8sh.IsServiceUpInNameSpace("rook-ceph-rgw-default") {
_, err := suite.k8sh.GetService("rgw-external")
if k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace) {
_, err := k8sh.GetService(rgwExternalServiceName, namespace)
if err != nil {
suite.k8sh.KubectlWithStdin(getRGWExternalServiceDef(), []string{"create", "-f", "-"}...)
if !suite.k8sh.IsServiceUpInNameSpace("rgw-external") {
k8sh.KubectlWithStdin(getRGWExternalServiceDef(namespace), []string{"create", "-f", "-"}...)
if !k8sh.IsServiceUp(rgwExternalServiceName, defaultRookNamespace) {
logger.Infof("Couldn't start RGW external serivce")
return fmt.Errorf("Cannot expose rgw servie for external user")
}
@@ -177,24 +203,23 @@ func getBucketSizeAndObjectes(bucketname string, bucketList []model.ObjectBucket
}
//Expose RGW service
func getRGWExternalServiceDef() string {
func getRGWExternalServiceDef(namespace string) string {
return `apiVersion: v1
kind: Service
metadata:
name: rgw-external
namespace: rook
name: rgw-external-` + namespace + `
namespace: ` + namespace + `
labels:
app: rook-ceph-rgw
rook_cluster: rook
rook_cluster: ` + namespace + `
spec:
ports:
- name: rook-ceph-rgw
port: 53390
protocol: TCP
nodePort: 30001
selector:
app: rook-ceph-rgw
rook_cluster: rook
rook_cluster: ` + namespace + `
sessionAffinity: None
type: NodePort
`
tests/smoke/block_test.go
-210
View File
@@ -1,210 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package smoke
import (
"time"
"strings"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
)
var (
pvcName = "block-pv-claim"
blockMountPath = "/tmp/rook1"
blockPodName = "block-test"
)
// Smoke Test for Block Storage - Test check the following operations on Block Storage in order
//Create,Mount,Write,Read,Unmount and Delete.
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
defer suite.blockTestDataCleanUp()
logger.Infof("Block Storage Smoke Test - create, mount, write to, read from, and unmount")
rbc := suite.helper.GetBlockClient()
logger.Infof("Step 0 : Get Initial List Block")
initBlockImages, _ := rbc.BlockList()
logger.Infof("step 1: Create block storage")
cbErr := suite.blockStorageOperation("create")
require.Nil(suite.T(), cbErr)
require.True(suite.T(), suite.retryBlockImageCountCheck(len(initBlockImages), 1), "Make sure a new block is created")
logger.Infof("Block Storage created successfully")
require.True(suite.T(), suite.k8sh.WaitUntilPVCIsBound(pvcName), "Make sure PVC is Bound")
logger.Infof("step 2: Mount block storage")
_, mtErr := rbc.BlockMap(getBlockPodDefintion(), blockMountPath)
require.Nil(suite.T(), mtErr)
require.True(suite.T(), suite.k8sh.IsPodRunning(blockPodName), "make sure block-test pod is in running state")
logger.Infof("Block Storage Mounted successfully")
logger.Infof("step 3: Write to block storage")
_, wtErr := rbc.BlockWrite(blockPodName, blockMountPath, "Smoke Test Data form Block storage", "bsFile1", "")
require.Nil(suite.T(), wtErr)
logger.Infof("Write to Block storage successfully")
logger.Infof("step 4: Read from block storage")
read, rErr := rbc.BlockRead(blockPodName, blockMountPath, "bsFile1", "")
require.Nil(suite.T(), rErr)
require.Contains(suite.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged")
logger.Infof("Read from Block storage successfully")
logger.Infof("step 5: Unmount block storage")
_, unmtErr := rbc.BlockUnmap(getBlockPodDefintion(), blockMountPath)
require.Nil(suite.T(), unmtErr)
require.True(suite.T(), suite.k8sh.IsPodTerminated(blockPodName), "make sure block-test pod is terminated")
logger.Infof("Block Storage unmounted successfully")
logger.Infof("step 6: Deleting block storage")
dbErr := suite.blockStorageOperation("delete")
require.Nil(suite.T(), dbErr)
require.True(suite.T(), suite.retryBlockImageCountCheck(len(initBlockImages), 0), "Make sure a block is deleted")
logger.Infof("Block Storage deleted successfully")
}
func (suite *SmokeSuite) blockTestDataCleanUp() {
logger.Infof("Cleaning up block storage")
suite.helper.GetBlockClient().BlockUnmap(getBlockPodDefintion(), blockMountPath)
suite.blockStorageOperation("delete")
suite.cleanUpDymanicBlockStorage()
}
func (suite *SmokeSuite) blockStorageOperation(action string) error {
poolStorageClassDef := suite.getBlockPoolAndStorageClassDefinition()
pvcDef := getPvcDefiniton()
suite.k8sh.ResourceOperation(action, poolStorageClassDef)
// see https://github.com/rook/rook/issues/767
time.Sleep(10 * time.Second)
_, err := suite.k8sh.ResourceOperation(action, pvcDef)
return err
}
// periodically checking if block image count has changed to expected value
// When creating pvc in k8s platform, it may take some time for the block Image to be bounded
func (suite *SmokeSuite) retryBlockImageCountCheck(imageCount, expectedChange int) bool {
inc := 0
for inc < utils.RetryLoop {
logger.Infof("Getting list of blocks (expecting %d)", (imageCount + expectedChange))
blockImages, _ := suite.helper.GetBlockClient().BlockList()
if imageCount+expectedChange == len(blockImages) {
return true
}
time.Sleep(time.Second * utils.RetryInterval)
inc++
}
return false
}
//CleanUpDymanicBlockStorage is helper method to clean up bock storage created by tests
func (suite *SmokeSuite) cleanUpDymanicBlockStorage() {
// Delete storage pool, storage class and pvc
blockImagesList, _ := suite.helper.GetBlockClient().BlockList()
for _, blockImage := range blockImagesList {
suite.helper.GetRestAPIClient().DeleteBlockImage(blockImage)
}
}
func (suite *SmokeSuite) getBlockPoolAndStorageClassDefinition() string {
k8sVersion := suite.k8sh.GetK8sServerVersion()
if strings.Contains(k8sVersion, "v1.5") {
return `apiVersion: rook.io/v1alpha1
kind: Pool
metadata:
name: replicapool
namespace: rook
spec:
replication:
size: 1
# For an erasure-coded pool, comment out the replication size above and uncomment the following settings.
# Make sure you have enough OSDs to support the replica size or erasure code chunks.
#erasureCode:
# codingChunks: 2
# dataChunks: 2
---
apiVersion: storage.k8s.io/v1beta1
kind: StorageClass
metadata:
name: rook-block
provisioner: rook.io/block
parameters:
pool: replicapool`
}
return `apiVersion: rook.io/v1alpha1
kind: Pool
metadata:
name: replicapool
namespace: rook
spec:
replication:
size: 1
# For an erasure-coded pool, comment out the replication count above and uncomment the following settings.
# Make sure you have enough OSDs to support the replica count or erasure code chunks.
#erasureCode:
# codingChunks: 2
# dataChunks: 2
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rook-block
provisioner: rook.io/block
parameters:
pool: replicapool`
}
func getPvcDefiniton() string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: block-pv-claim
annotations:
volume.beta.kubernetes.io/storage-class: rook-block
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1M`
}
func getBlockPodDefintion() string {
return `apiVersion: v1
kind: Pod
metadata:
name: block-test
spec:
containers:
- image: busybox
name: block-test1
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
volumeMounts:
- name: block-persistent-storage
mountPath: /tmp/rook1
volumes:
- name: block-persistent-storage
persistentVolumeClaim:
claimName: block-pv-claim
restartPolicy: Never`
}
tests/smoke/file_test.go
-128
View File
@@ -1,128 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package smoke
import (
"fmt"
"github.com/stretchr/testify/require"
)
var (
fileSystemName = "testfs"
fileMountPath = "/tmp/rookfs"
filePodName = "file-test"
filePodNamespace = "rook"
)
// Smoke Test for File System Storage - Test check the following operations on FileSystem Storage in order
//Create,Mount,Write,Read,Unmount and Delete.
func (suite *SmokeSuite) TestFileStorage_SmokeTest() {
defer suite.fileTestDataCleanUp()
logger.Infof("File Storage Smoke Test - create, mount, write to, read from, and unmount")
rfc := suite.helper.GetFileSystemClient()
logger.Infof("Step 1: Create file System")
_, fscErr := rfc.FSCreate(fileSystemName)
require.Nil(suite.T(), fscErr)
fileSystemList, _ := rfc.FSList()
require.Equal(suite.T(), 1, len(fileSystemList), "There should one shared file system present")
filesystemData := fileSystemList[0]
require.Equal(suite.T(), "testfs", filesystemData.Name, "make sure filesystem name matches")
logger.Infof("File system created")
logger.Infof("Step 2: Mount file System")
mtfsErr := suite.podWithFileSystem("create")
require.Nil(suite.T(), mtfsErr)
require.True(suite.T(), suite.k8sh.IsPodRunningInNamespace(filePodName), "make sure file-test pod is in running state")
logger.Infof("File system mounted successfully")
logger.Infof("Step 3: Write to file system")
_, wfsErr := rfc.FSWrite(filePodName, fileMountPath, "Smoke Test Data for file system storage", "fsFile1", filePodNamespace)
require.Nil(suite.T(), wfsErr)
logger.Infof("Write to file system successful")
logger.Infof("Step 4: Read from file system")
read, rdErr := rfc.FSRead(filePodName, fileMountPath, "fsFile1", filePodNamespace)
require.Nil(suite.T(), rdErr)
require.Contains(suite.T(), read, "Smoke Test Data for file system storage", "make sure content of the files is unchanged")
logger.Infof("Read from file system successful")
logger.Infof("Step 5: UnMount file System")
umtfsErr := suite.podWithFileSystem("delete")
require.Nil(suite.T(), umtfsErr)
require.True(suite.T(), suite.k8sh.IsPodTerminatedInNamespace(filePodName), "make sure file-test pod is terminated")
logger.Infof("File system mounted successfully")
logger.Infof("Step 6: Deleting file storage")
rfc.FSDelete(fileSystemName)
//Delete is not deleting filesystem - known issue
//require.Nil(suite.T(), fsd_err)
logger.Infof("File system deleted")
}
func (suite *SmokeSuite) fileTestDataCleanUp() {
logger.Infof("Cleaning up file system")
suite.podWithFileSystem("delete")
suite.helper.GetFileSystemClient().FSDelete(fileSystemName)
}
func (suite *SmokeSuite) podWithFileSystem(action string) error {
mons, err := suite.k8sh.GetMonitorServices()
require.Nil(suite.T(), err)
logger.Infof("mountFileStorage: Mons: %+v", mons)
_, err = suite.k8sh.ResourceOperationFromTemplate(action, getFileSystemTestPod(), mons)
if err != nil {
return fmt.Errorf("failed to %s pod -- %s. %+v", action, getFileSystemTestPod(), err)
}
return nil
}
func getFileSystemTestPod() string {
return `apiVersion: v1
kind: Pod
metadata:
name: file-test
namespace: rook
spec:
containers:
- name: file-test1
image: busybox
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
env:
volumeMounts:
- mountPath: "/tmp/rookfs"
name: testfs
volumes:
- name: testfs
cephfs:
monitors:
- {{.mon0}}
- {{.mon1}}
- {{.mon2}}
user: admin
secretRef:
name: rook-admin
readOnly: false
restartPolicy: Never
`
}
tests/smoke/helm_test.go
+20 -153
View File
@@ -1,19 +1,16 @@
package smoke
import (
"strings"
"testing"
"time"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/enums"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"time"
)
var (
@@ -26,12 +23,10 @@ func TestHelmSmokeSuite(t *testing.T) {
type HelmSmokeSuite struct {
suite.Suite
helper *clients.TestClient
k8sh *utils.K8sHelper
installer *installer.InstallHelper
hh *utils.HelmHelper
pvcDef string
storageclassDef string
helper *clients.TestClient
k8sh *utils.K8sHelper
installer *installer.InstallHelper
hh *utils.HelmHelper
}
func (hs *HelmSmokeSuite) SetupSuite() {
@@ -43,181 +38,53 @@ func (hs *HelmSmokeSuite) SetupSuite() {
hs.installer = installer.NewK8sRookhelper(kh.Clientset)
err = hs.installer.CreateK8sRookOperatorViaHelm()
err = hs.installer.CreateK8sRookOperatorViaHelm(defaultRookNamespace)
require.NoError(hs.T(), err)
require.True(hs.T(), kh.IsPodInExpectedState("rook-operator", "rook", "Running"),
require.True(hs.T(), kh.IsPodInExpectedState("rook-operator", defaultRookNamespace, "Running"),
"Make sure rook-operator is in running state")
time.Sleep(10 * time.Second)
err = hs.installer.CreateK8sRookCluster()
err = hs.installer.CreateK8sRookCluster(defaultRookNamespace)
require.NoError(hs.T(), err)
err = hs.installer.CreateK8sRookToolbox()
err = hs.installer.CreateK8sRookToolbox(defaultRookNamespace)
require.NoError(hs.T(), err)
hs.helper, err = clients.CreateTestClient(enums.Kubernetes, kh)
hs.helper, err = clients.CreateTestClient(enums.Kubernetes, kh, defaultRookNamespace)
require.Nil(hs.T(), err)
hs.pvcDef = `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{.claimName}}
annotations:
volume.beta.kubernetes.io/storage-class: rook-block
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1M`
hs.storageclassDef = `apiVersion: rook.io/v1alpha1
kind: Pool
metadata:
name: {{.poolName}}
namespace: rook
spec:
replication:
size: 1
# For an erasure-coded pool, comment out the replication count above and uncomment the following settings.
# Make sure you have enough OSDs to support the replica count or erasure code chunks.
#erasureCode:
# codingChunks: 2
# dataChunks: 2
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rook-block
provisioner: rook.io/block
parameters:
pool: {{.poolName}}`
}
func (hs *HelmSmokeSuite) TearDownSuite() {
if hs.T().Failed() {
hs.k8sh.GetRookLogs("rook-operator", "default", hs.T().Name())
hs.k8sh.GetRookLogs("rook-api", "rook", hs.T().Name())
hs.k8sh.GetRookLogs("rook-ceph-mgr", "rook", hs.T().Name())
hs.k8sh.GetRookLogs("rook-ceph-mon", "rook", hs.T().Name())
hs.k8sh.GetRookLogs("rook-ceph-osd", "rook", hs.T().Name())
hs.k8sh.GetRookLogs("rook-ceph-rgw", "rook", hs.T().Name())
hs.k8sh.GetRookLogs("rook-ceph-mds", "rook", hs.T().Name())
gatherAllRookLogs(hs.k8sh, hs.Suite, defaultRookNamespace, defaultRookNamespace)
}
hs.installer.UninstallRookFromK8sViaHelm()
hs.installer.UninstallRookFromK8s(defaultRookNamespace, true)
}
//Test to make sure all rook components are installed and Running
func (hs *HelmSmokeSuite) TestRookInstallViaHelm() {
checkIfRookClusterIsInstalled(hs.k8sh, hs.Suite, defaultRookNamespace, defaultRookNamespace)
assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-operator", "default", 1, "Running"),
"Make sure there is 1 rook-operator present in Running state")
assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-api", "rook", 1, "Running"),
"Make sure there is 1 rook-api present in Running state")
assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-mgr", "rook", 1, "Running"),
"Make sure there is 1 rook-ceph-mgr present in Running state")
assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-osd", "rook", 1, "Running"),
"Make sure there is at lest 1 rook-ceph-osd present in Running state")
assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-mon", "rook", 3, "Running"),
"Make sure there are 3 rook-ceph-mon present in Running state")
}
//Test BlockCreation on Rook that was installed via Helm charts
//Test BlockCreation on Rook that was installed via Helm
func (hs *HelmSmokeSuite) TestBlockStoreOnRookInstalledViaHelm() {
//Check initial number of blocks
bc := hs.helper.GetBlockClient()
initialBlocks, err := bc.BlockList()
require.Nil(hs.T(), err)
initBlockCount := len(initialBlocks)
//deploy storageclass
sc := map[string]string{
"poolName": "helm-pool",
}
res1, err := hs.k8sh.ResourceOperationFromTemplate("create", hs.storageclassDef, sc)
require.Contains(hs.T(), res1, "pool \"helm-pool\" created", "Make sure test pool is created")
require.Contains(hs.T(), res1, "storageclass \"rook-block\" created", "Make sure storageclass is created")
require.NoError(hs.T(), err)
// see https://github.com/rook/rook/issues/767
time.Sleep(10 * time.Second)
// deploy pvc
vc := map[string]string{
"claimName": "helm-claim",
}
res2, err := hs.k8sh.ResourceOperationFromTemplate("create", hs.pvcDef, vc)
require.Contains(hs.T(), res2, "persistentvolumeclaim \"helm-claim\" created", "Make sure pvc is created")
require.NoError(hs.T(), err)
require.True(hs.T(), hs.isPVCBound("helm-claim"))
//Make sure new block is created
b, _ := bc.BlockList()
require.Equal(hs.T(), initBlockCount+1, len(b), "Make sure new block image is created")
//Delete pvc and storageclass
_, err = hs.k8sh.ResourceOperationFromTemplate("delete", hs.storageclassDef, sc)
require.NoError(hs.T(), err)
_, err = hs.k8sh.ResourceOperationFromTemplate("delete", hs.pvcDef, vc)
require.NoError(hs.T(), err)
time.Sleep(2 * time.Second)
b, _ = bc.BlockList()
require.Equal(hs.T(), initBlockCount, len(b), "Make sure new block image is deleted")
runBlockE2ETestLite(hs.helper, hs.k8sh, hs.Suite, defaultRookNamespace)
}
//Test File System Creation on Rook that was installed via Helm charts
//Test File System Creation on Rook that was installed via helm
func (hs *HelmSmokeSuite) TestFileStoreOnRookInstalledViaHelm() {
fc := hs.helper.GetFileSystemClient()
//Create file System
_, fscErr := fc.FSCreate(fileSystemName)
require.Nil(hs.T(), fscErr)
fileSystemList, _ := fc.FSList()
require.Equal(hs.T(), 1, len(fileSystemList), "There should one shared file system present")
//Make sure rook-ceph-mds pods are running
require.True(hs.T(), hs.k8sh.IsPodInExpectedState("rook-ceph-mds", "rook", "Running"),
"Make sure rook-operator is in running state")
assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-mds", "rook", 1, "Running"),
"Make sure there is 1 rook-operator present in Running state")
runFileE2ETestLite(hs.helper, hs.k8sh, hs.Suite, defaultRookNamespace, "testFs")
}
//Test Object StoreCreation on Rook that was installed via Helm charts
//Test Object StoreCreation on Rook that was installed via helm
func (hs *HelmSmokeSuite) TestObjectStoreOnRookInstalledViaHelm() {
//Create Object store
oc := hs.helper.GetObjectClient()
oc.ObjectCreate()
//Make sure rook-ceph-rgw pods are running
require.True(hs.T(), hs.k8sh.IsPodInExpectedState("rook-ceph-rgw", "rook", "Running"),
"Make sure rook-operator is in running state")
assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-rgw", "rook", 2, "Running"),
"Make sure there is 1 rook-operator present in Running state")
require.True(hs.T(), hs.k8sh.IsServiceUpInNameSpace("rook-ceph-rgw-default"))
runObjectE2ETestLite(hs.helper, hs.k8sh, hs.Suite, defaultRookNamespace, "default", 3)
}
func (hs *HelmSmokeSuite) isPVCBound(name string) bool {
inc := 0
for inc < utils.RetryLoop {
status, _ := hs.k8sh.GetPVCStatus(name)
if strings.TrimRight(status, "\n") == "'Bound'" {
return true
}
time.Sleep(time.Second * utils.RetryInterval)
inc++
}
return false
}
tests/smoke/mon_test.go
+3 -4
View File
@@ -29,9 +29,8 @@ import (
func (suite *SmokeSuite) TestMonFailover() {
logger.Infof("Mon Failover Smoke Test")
namespace := "rook"
opts := metav1.ListOptions{LabelSelector: "app=rook-ceph-mon"}
monReplicas, err := suite.k8sh.Clientset.ReplicaSets(namespace).List(opts)
monReplicas, err := suite.k8sh.Clientset.ReplicaSets(defaultRookNamespace).List(opts)
require.Nil(suite.T(), err)
assert.Equal(suite.T(), 3, len(monReplicas.Items))
@@ -39,12 +38,12 @@ func (suite *SmokeSuite) TestMonFailover() {
logger.Infof("Killing mon %s", monToKill)
propagation := metav1.DeletePropagationForeground
delOptions := &metav1.DeleteOptions{PropagationPolicy: &propagation}
err = suite.k8sh.Clientset.ReplicaSets(namespace).Delete(monToKill, delOptions)
err = suite.k8sh.Clientset.ReplicaSets(defaultRookNamespace).Delete(monToKill, delOptions)
require.Nil(suite.T(), err)
// Wait for the health check to start a new monitor
for i := 0; i < 10; i++ {
monReplicas, err := suite.k8sh.Clientset.ReplicaSets(namespace).List(opts)
monReplicas, err := suite.k8sh.Clientset.ReplicaSets(defaultRookNamespace).List(opts)
require.Nil(suite.T(), err)
// Make sure the old mon is not still alive
tests/smoke/multi_cluster_deploy_test.go
+178
View File
@@ -0,0 +1,178 @@
package smoke
import (
"strings"
"testing"
"time"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/enums"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
utilversion "k8s.io/kubernetes/pkg/util/version"
)
func TestInstallingMultipleRookCluster(t *testing.T) {
suite.Run(t, new(MultiRookClusterDeploySuite))
}
type MultiRookClusterDeploySuite struct {
suite.Suite
helper1 *clients.TestClient
helper2 *clients.TestClient
k8sh *utils.K8sHelper
installer *installer.InstallHelper
installData *installer.InstallData
}
//Deploy Multiple Rook clusters
func (mrc *MultiRookClusterDeploySuite) SetupSuite() {
kh, err := utils.CreateK8sHelper()
require.NoError(mrc.T(), err)
mrc.k8sh = kh
mrc.installer = installer.NewK8sRookhelper(kh.Clientset)
mrc.installData = installer.NewK8sInstallData()
err = mrc.installer.CreateK8sRookOperator()
require.NoError(mrc.T(), err)
require.True(mrc.T(), kh.IsPodInExpectedState("rook-operator", "default", "Running"),
"Make sure rook-operator is in running state")
time.Sleep(10 * time.Second)
err = mrc.installer.CreateK8sRookCluster(clusterNamespace1)
require.NoError(mrc.T(), err)
err = mrc.installer.CreateK8sRookToolbox(clusterNamespace1)
require.NoError(mrc.T(), err)
err = mrc.installer.CreateK8sRookCluster(clusterNamespace2)
require.NoError(mrc.T(), err)
err = mrc.installer.CreateK8sRookToolbox(clusterNamespace2)
require.NoError(mrc.T(), err)
mrc.helper1, err = clients.CreateTestClient(enums.Kubernetes, kh, clusterNamespace1)
require.Nil(mrc.T(), err)
mrc.helper2, err = clients.CreateTestClient(enums.Kubernetes, kh, clusterNamespace2)
require.Nil(mrc.T(), err)
}
func (mrc *MultiRookClusterDeploySuite) TearDownSuite() {
if mrc.T().Failed() {
gatherAllRookLogs(mrc.k8sh, mrc.Suite, defaultNamespace, clusterNamespace1)
gatherAllRookLogs(mrc.k8sh, mrc.Suite, defaultNamespace, clusterNamespace2)
}
deleteArgs := []string{"delete", "-f", "-"}
skipRookInstall := strings.EqualFold(mrc.installer.Env.SkipInstallRook, "true")
if skipRookInstall {
return
}
k8sVersion := mrc.k8sh.GetK8sServerVersion()
serverVersion, err := mrc.k8sh.Clientset.Discovery().ServerVersion()
if err != nil {
panic(err)
}
kubeVersion := utilversion.MustParseSemantic(serverVersion.GitVersion)
logger.Infof("Uninstalling All Rook Clusters")
k8sHelp, err := utils.CreateK8sHelper()
if err != nil {
panic(err)
}
rookOperator := mrc.installData.GetRookOperator(k8sVersion)
//Delete rook operator
_, err = mrc.k8sh.KubectlWithStdin(rookOperator, deleteArgs...)
if err != nil {
logger.Errorf("Rook operator cannot be deleted,err -> %v", err)
panic(err)
}
//delete rook cluster
mrc.installer.CleanupCluster(clusterNamespace1, kubeVersion)
mrc.installer.CleanupCluster(clusterNamespace2, kubeVersion)
//Delete clusterrol and clustterrolebindings
if kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.6.0")) {
_, err = k8sHelp.DeleteResource([]string{"clusterrole", "rook-api"})
if err != nil {
logger.Errorf("Clusterrole rook-api cannot be deleted")
panic(err)
}
_, err = k8sHelp.DeleteResource([]string{"clusterrolebinding", "rook-api"})
if err != nil {
logger.Errorf("clusterrolebinding rook-api cannot be deleted")
panic(err)
}
}
// Delete crd/tpr
if kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.7.0")) {
_, err = k8sHelp.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io"})
if err != nil {
logger.Errorf("crd cannot be deleted")
panic(err)
}
} else {
_, err = k8sHelp.DeleteResource([]string{"thirdpartyresources", "cluster.rook.io", "pool.rook.io"})
if err != nil {
logger.Errorf("tpr cannot be deleted")
panic(err)
}
}
isRookUninstalled1 := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterNamespace1)
isNameSpaceDeleted1 := k8sHelp.WaitUntilNameSpaceIsDeleted(clusterNamespace1)
isRookUninstalled2 := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterNamespace2)
isNameSpaceDeleted2 := k8sHelp.WaitUntilNameSpaceIsDeleted(clusterNamespace2)
mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
if isRookUninstalled1 && isNameSpaceDeleted1 && isRookUninstalled2 && isNameSpaceDeleted2 {
logger.Infof("Rook clusters %s and %s uninstalled successfully", clusterNamespace1, clusterNamespace2)
return
}
logger.Infof("Rook clusters %s and %s not uninstalled successfully", clusterNamespace1, clusterNamespace2)
}
//Test to make sure all rook components are installed and Running
func (mrc *MultiRookClusterDeploySuite) TestInstallingMultipleRookClusters() {
//Check if Rook cluster 1 is deployed successfully
checkIfRookClusterIsInstalled(mrc.k8sh, mrc.Suite, defaultNamespace, clusterNamespace1)
//Check if Rook cluster 1 is deployed successfully
checkIfRookClusterIsInstalled(mrc.k8sh, mrc.Suite, defaultNamespace, clusterNamespace2)
}
//Test Block Store Creation on multiple rook clusters
func (mrc *MultiRookClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() {
runBlockE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, clusterNamespace1)
runBlockE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, clusterNamespace2)
}
//Test FileSystem Creation on multiple rook clusters
func (mrc *MultiRookClusterDeploySuite) TestFileStoreOnMultiRookCluster() {
runFileE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, clusterNamespace1, "test-fs-1")
//TODO - Known Issues #https://github.com/rook/rook/issues/970
//runFileE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, clusterNamespace2, "test-fs-2")
}
//Test Object Store Creation on multiple rook clusters
func (mrc *MultiRookClusterDeploySuite) TestObjectStoreOnMultiRookCluster() {
runObjectE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, clusterNamespace1, "default-c1", 2)
//TODO - Known Issues #https://github.com/rook/rook/issues/970
//runObjectE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, clusterNamespace2, "default-c2", 1)
}
tests/smoke/smoke_test.go
+20 -14
View File
@@ -19,8 +19,6 @@ package smoke
import (
"testing"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/enums"
"github.com/rook/rook/tests/framework/installer"
@@ -29,8 +27,6 @@ import (
"github.com/stretchr/testify/suite"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "smoketest")
func TestSmokeSuiteK8s(t *testing.T) {
suite.Run(t, new(SmokeSuite))
}
@@ -50,22 +46,32 @@ func (suite *SmokeSuite) SetupSuite() {
suite.installer = installer.NewK8sRookhelper(kh.Clientset)
err = suite.installer.InstallRookOnK8s()
err = suite.installer.InstallRookOnK8s(defaultRookNamespace)
require.NoError(suite.T(), err)
suite.helper, err = clients.CreateTestClient(enums.Kubernetes, kh)
suite.helper, err = clients.CreateTestClient(enums.Kubernetes, kh, defaultRookNamespace)
require.Nil(suite.T(), err)
}
func (suite *SmokeSuite) TearDownSuite() {
if suite.T().Failed() {
suite.k8sh.GetRookLogs("rook-operator", "default", suite.T().Name())
suite.k8sh.GetRookLogs("rook-api", "rook", suite.T().Name())
suite.k8sh.GetRookLogs("rook-ceph-mgr", "rook", suite.T().Name())
suite.k8sh.GetRookLogs("rook-ceph-mon", "rook", suite.T().Name())
suite.k8sh.GetRookLogs("rook-ceph-osd", "rook", suite.T().Name())
suite.k8sh.GetRookLogs("rook-ceph-rgw", "rook", suite.T().Name())
suite.k8sh.GetRookLogs("rook-ceph-mds", "rook", suite.T().Name())
gatherAllRookLogs(suite.k8sh, suite.Suite, defaultNamespace, defaultRookNamespace)
}
suite.installer.UninstallRookFromK8s()
suite.installer.UninstallRookFromK8s(defaultRookNamespace, false)
}
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
runBlockE2ETest(suite.helper, suite.k8sh, suite.Suite, defaultRookNamespace)
}
func (suite *SmokeSuite) TestFileStorage_SmokeTest() {
runFileE2ETest(suite.helper, suite.k8sh, suite.Suite, defaultRookNamespace, "smoke-test-fs")
}
func (suite *SmokeSuite) TestObjectStorage_SmokeTest() {
runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, defaultRookNamespace)
}
//Test to make sure all rook components are installed and Running
func (suite *SmokeSuite) TestRookClusterInstallation_smokeTest() {
checkIfRookClusterIsInstalled(suite.k8sh, suite.Suite, defaultNamespace, defaultRookNamespace)
}