From 9e08afb0153cd3b3cefb1b900aabef163d4f675c Mon Sep 17 00:00:00 2001 From: Divesh Angula Date: Wed, 6 Sep 2017 23:36:13 -0700 Subject: [PATCH] creating tests for multi cluster rook install - refactoring basic tests so they can be reused with different rook installations --- Jenkinsfile | 2 +- tests/README.md | 27 +- tests/framework/clients/object.go | 4 +- tests/framework/clients/rest_client.go | 28 +- tests/framework/clients/test_client.go | 4 +- tests/framework/contracts/operations.go | 2 +- tests/framework/installer/install_data.go | 17 +- tests/framework/installer/install_helper.go | 149 ++++------ tests/framework/utils/helm_helper.go | 16 + tests/framework/utils/k8s_helper.go | 233 ++++++--------- .../integration/block_createBlock_api_test.go | 6 +- .../integration/block_createBlock_k8s_test.go | 6 +- tests/integration/block_e2e_k8s_test.go | 6 +- tests/longhaul/block_on_k8s_test.go | 4 +- tests/scripts/kubeadm-install.sh | 2 +- tests/scripts/kubeadm.sh | 2 +- tests/scripts/minikube.sh | 6 +- tests/smoke/base_block_test.go | 277 ++++++++++++++++++ tests/smoke/base_deploy_test.go | 59 ++++ tests/smoke/base_file_test.go | 152 ++++++++++ .../{object_test.go => base_object_test.go} | 115 +++++--- tests/smoke/block_test.go | 210 ------------- tests/smoke/file_test.go | 128 -------- tests/smoke/helm_test.go | 173 ++--------- tests/smoke/mon_test.go | 7 +- tests/smoke/multi_cluster_deploy_test.go | 178 +++++++++++ tests/smoke/smoke_test.go | 34 ++- 27 files changed, 1006 insertions(+), 841 deletions(-) create mode 100644 tests/smoke/base_block_test.go create mode 100644 tests/smoke/base_deploy_test.go create mode 100644 tests/smoke/base_file_test.go rename tests/smoke/{object_test.go => base_object_test.go} (55%) delete mode 100644 tests/smoke/block_test.go delete mode 100644 tests/smoke/file_test.go create mode 100644 tests/smoke/multi_cluster_deploy_test.go diff --git a/Jenkinsfile b/Jenkinsfile index 3ac3aab5f..fa97b8da6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { def data = [ "aws_ci": "v1.6.7", - "gce_ci": "v1.7.4" + "gce_ci": "v1.7.5" ] testruns = [:] for (kv in mapToList(data)) { diff --git a/tests/README.md b/tests/README.md index 1b1b70980..dff13c23e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,7 +22,7 @@ The framework also provides scripts to install Kubernetes. There are two scripts 2. Kubeadm : run [kubeadm.sh](/tests/scripts/kubeadm.sh) to setup a single-node K8s cluster using kubeadm (recommended for ubuntu) These start-up scripts are designed to start any version of kubernetes, Based on `KUBE_VERSION` environment variable -By default they start kubernetes version v1.7.4.(except minikube which starts kubernetes v1.7.2) +By default they start kubernetes version v1.7.5. e.g. To kubernetes version v1.6.7 ``` export KUBE_VERSION=v1.6.7 @@ -31,8 +31,13 @@ tests/scripts/kubeadm.sh up To stop the cluster : `tests/scripts/kubeadm.sh clean` or `tests/scripts/miniube.sh clean` -All of the above scripts are designed to start kubernetes, make sure rbd is working, and copy the latest rook and toolbox images -to the kubernetes containers. +To copy rook images generated by build into minikube vm, run the following commands after `minikube.sh up` +``` +tests/scripts/minikube.sh update +tests/scripts/minikube.sh helm +``` +kubeadm.sh starts kubernetes on the host, so all the images generated by build should be available to the cluster. + ### Install Helm Use [helm.sh](/tests/scripts/helm.sh) to install helm and set up rook charts defined under `_output/charts` (generated by build). @@ -46,9 +51,19 @@ should be run from project root. eg* `tests/script/kubeadm.sh up` From the root do the following: 1. Build rook: `make build` 2. Start kubernetes using one of the following: - - `tests/scripts/minikube.sh up` - - `tests/scripts/kubeadm.sh up` - - `tests/scripts/helm.sh up` + + - using Kubeadm + ``` + tests/scripts/kubeadm.sh up + tests/scripts/helm.sh up + ``` + - using minikube + ``` + tests/scripts/minikube.sh up + tests/scripts/minikube.sh update + tests/scripts/minikube.sh helm + tests/scripts/helm.sh up + ``` 3. Run integration tests: `make test-integration` diff --git a/tests/framework/clients/object.go b/tests/framework/clients/object.go index a2eb56ca9..0b7499aaf 100644 --- a/tests/framework/clients/object.go +++ b/tests/framework/clients/object.go @@ -34,8 +34,8 @@ func CreateObjectOperation(rookRestClient contracts.RestAPIOperator) *ObjectOper //ObjectCreate Function to create a object store in rook //Input paramatres -None //Output - output returned by rook Rest API client -func (ro *ObjectOperation) ObjectCreate() (string, error) { - store := model.ObjectStore{Name: "default", RGWReplicas: 1} +func (ro *ObjectOperation) ObjectCreate(name string, replicaCount int32) (string, error) { + store := model.ObjectStore{Name: name, RGWReplicas: replicaCount} store.DataConfig.ReplicationConfig.Size = 1 store.MetadataConfig.ReplicationConfig.Size = 1 return ro.restClient.CreateObjectStore(store) diff --git a/tests/framework/clients/rest_client.go b/tests/framework/clients/rest_client.go index 48099ce20..763ad0275 100644 --- a/tests/framework/clients/rest_client.go +++ b/tests/framework/clients/rest_client.go @@ -34,23 +34,28 @@ type RestAPIClient struct { } //CreateRestAPIClient Create Rook REST API client -func CreateRestAPIClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper) *RestAPIClient { +func CreateRestAPIClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper, namespace string) *RestAPIClient { var endpoint string switch { case platform == enums.Kubernetes: //Start rook_api_external server via nodePort if not it not already running. - _, err := k8sHelper.GetService("rook-api-external") + externalAPIservice := "rook-api-external-" + namespace + _, err := k8sHelper.GetService(externalAPIservice, namespace) if err != nil { - _, err = k8sHelper.ResourceOperation("create", getExternalAPIServiceDefinition()) + _, err = k8sHelper.ResourceOperation("create", getExternalAPIServiceDefinition(namespace)) if err != nil { - panic(fmt.Errorf("failed to kubectl create -f - %v: %+v", getExternalAPIServiceDefinition(), err)) + panic(fmt.Errorf("failed to kubectl create -f - %v: %+v", getExternalAPIServiceDefinition(namespace), err)) } } - apiIP, err := k8sHelper.GetPodHostID("rook-api", "rook") + apiIP, err := k8sHelper.GetPodHostID("rook-api", namespace) if err != nil { panic(fmt.Errorf("Host Ip for Rook-api service not found. %+v", err)) } - endpoint = "http://" + apiIP + ":30002" + nodePort, err := k8sHelper.GetServiceNodePort(externalAPIservice, namespace) + if err != nil { + panic(fmt.Errorf("port for external Rook-api service not found. %+v", err)) + } + endpoint = fmt.Sprintf("http://%s:%s", apiIP, nodePort) case platform == enums.StandAlone: endpoint = "http://localhost:8124" default: @@ -190,24 +195,23 @@ func (a *RestAPIClient) DeleteObjectUser(id string) error { } -func getExternalAPIServiceDefinition() string { +func getExternalAPIServiceDefinition(namespace string) string { return `apiVersion: v1 kind: Service metadata: - name: rook-api-external - namespace: rook + name: rook-api-external-` + namespace + ` + namespace: ` + namespace + ` labels: app: rook-api - rook_cluster: rook + rook_cluster: ` + namespace + ` spec: ports: - name: rook-api port: 8124 protocol: TCP - nodePort: 30002 selector: app: rook-api - rook_cluster: rook + rook_cluster: ` + namespace + ` sessionAffinity: None type: NodePort ` diff --git a/tests/framework/clients/test_client.go b/tests/framework/clients/test_client.go index 968eb7bee..490508ca2 100644 --- a/tests/framework/clients/test_client.go +++ b/tests/framework/clients/test_client.go @@ -44,12 +44,12 @@ const ( ) //CreateTestClient creates new instance of test client for a platform -func CreateTestClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper) (*TestClient, error) { +func CreateTestClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper, namespace string) (*TestClient, error) { var blockClient contracts.BlockOperator var fsClient contracts.FileSystemOperator var objectClient contracts.ObjectOperator var poolClient contracts.PoolOperator - rookRestClient := CreateRestAPIClient(platform, k8sHelper) + rookRestClient := CreateRestAPIClient(platform, k8sHelper, namespace) switch { case platform == enums.Kubernetes: diff --git a/tests/framework/contracts/operations.go b/tests/framework/contracts/operations.go index 4f54394fe..a0825c730 100644 --- a/tests/framework/contracts/operations.go +++ b/tests/framework/contracts/operations.go @@ -42,7 +42,7 @@ type FileSystemOperator interface { //ObjectOperator - interface for rook object operations type ObjectOperator interface { - ObjectCreate() (string, error) + ObjectCreate(name string, replicaCount int32) (string, error) ObjectBucketList() ([]model.ObjectBucket, error) ObjectConnection() (*model.ObjectStoreConnectInfo, error) ObjectCreateUser(userid string, displayname string) (*model.ObjectUser, error) diff --git a/tests/framework/installer/install_data.go b/tests/framework/installer/install_data.go index 85c70fa8f..f2e4eebc5 100644 --- a/tests/framework/installer/install_data.go +++ b/tests/framework/installer/install_data.go @@ -27,7 +27,8 @@ func NewK8sInstallData() *InstallData { return &InstallData{} } -func (i *InstallData) getRookOperator(k8sVersion string) string { +//GetRookOperator returns rook Operator manifest +func (i *InstallData) GetRookOperator(k8sVersion string) string { if strings.Contains(k8sVersion, "v1.5") { return `apiVersion: extensions/v1beta1 @@ -170,17 +171,18 @@ spec: } -func (i *InstallData) getRookCluster() string { +//GetRookCluster returns rook-cluster manifest +func (i *InstallData) GetRookCluster(namespace string) string { return `apiVersion: v1 kind: Namespace metadata: - name: rook + name: ` + namespace + ` --- apiVersion: rook.io/v1alpha1 kind: Cluster metadata: - name: rook - namespace: rook + name: ` + namespace + ` + namespace: ` + namespace + ` spec: versionTag: master dataDirHostPath: @@ -241,12 +243,13 @@ spec: # deviceFilter: "^sd."` } -func (i *InstallData) getRookToolBox() string { +//GetRookToolBox returns rook-toolbox manifest +func (i *InstallData) GetRookToolBox(namespace string) string { return `apiVersion: v1 kind: Pod metadata: name: rook-tools - namespace: rook + namespace: ` + namespace + ` spec: containers: - name: rook-tools diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/install_helper.go index a1ecffe15..acb25fa1d 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/install_helper.go @@ -59,8 +59,10 @@ func init() { //CreateK8sRookOperator creates rook-operator via kubectl func (h *InstallHelper) CreateK8sRookOperator() (err error) { logger.Infof("Starting Rook Operator") + //creating clusterrolebinding for kubeadm env. + h.k8shelper.CreateAnonSystemClusterBinding() - rookOperator := h.installData.getRookOperator(h.k8shelper.GetK8sServerVersion()) + rookOperator := h.installData.GetRookOperator(h.k8shelper.GetK8sServerVersion()) _, err = h.k8shelper.KubectlWithStdin(rookOperator, createArgs...) @@ -84,10 +86,10 @@ func (h *InstallHelper) CreateK8sRookOperator() (err error) { return nil } -//CreateK8sRookOperatorViaHelm creates rook operator via Execute chart named local/rook present in local repo -func (h *InstallHelper) CreateK8sRookOperatorViaHelm() (err error) { +//CreateK8sRookOperatorViaHelm creates rook operator via Helm chart named local/rook present in local repo +func (h *InstallHelper) CreateK8sRookOperatorViaHelm(namespace string) (err error) { //creating clusterrolebinding for kubeadm env. - h.k8shelper.Kubectl([]string{"create", "clusterrolebinding", "anon-user-access", "--clusterrole", "cluster-admin", "--user", "system:anonymous"}...) + h.k8shelper.CreateAnonSystemClusterBinding() helmTag, err := h.helmHelper.GetLocalRookHelmChartVersion(helmChartName) @@ -95,7 +97,7 @@ func (h *InstallHelper) CreateK8sRookOperatorViaHelm() (err error) { return fmt.Errorf("Failed to get Version of helm chart %v, err : %v", helmChartName, err) } - err = h.helmHelper.InstallLocalRookHelmChart(helmChartName, helmDeployName, helmTag, "rook") + err = h.helmHelper.InstallLocalRookHelmChart(helmChartName, helmDeployName, helmTag, namespace) if err != nil { return fmt.Errorf("failed toinstall rook operator via helm, err : %v", err) @@ -117,10 +119,10 @@ func (h *InstallHelper) CreateK8sRookOperatorViaHelm() (err error) { } //CreateK8sRookToolbox creates rook-tools via kubectl -func (h *InstallHelper) CreateK8sRookToolbox() (err error) { +func (h *InstallHelper) CreateK8sRookToolbox(clusterNamespace string) (err error) { logger.Infof("Starting Rook toolbox") - rookToolbox := h.installData.getRookToolBox() + rookToolbox := h.installData.GetRookToolBox(clusterNamespace) _, err = h.k8shelper.KubectlWithStdin(rookToolbox, createArgs...) @@ -128,7 +130,7 @@ func (h *InstallHelper) CreateK8sRookToolbox() (err error) { return fmt.Errorf("Failed to create rook-toolbox pod : %v ", err) } - if !h.k8shelper.IsPodRunningInNamespace("rook-tools") { + if !h.k8shelper.IsPodRunning("rook-tools", clusterNamespace) { return fmt.Errorf("Rook Toolbox couldn't start") } logger.Infof("Rook Toolbox started") @@ -137,10 +139,10 @@ func (h *InstallHelper) CreateK8sRookToolbox() (err error) { } //CreateK8sRookCluster creates rook cluster via kubectl -func (h *InstallHelper) CreateK8sRookCluster() (err error) { +func (h *InstallHelper) CreateK8sRookCluster(clusterNamespace string) (err error) { logger.Infof("Starting Rook Cluster") - rookCluster := h.installData.getRookCluster() + rookCluster := h.installData.GetRookCluster(clusterNamespace) _, err = h.k8shelper.KubectlWithStdin(rookCluster, createArgs...) @@ -148,7 +150,7 @@ func (h *InstallHelper) CreateK8sRookCluster() (err error) { return fmt.Errorf("Failed to create rook cluster : %v ", err) } - if !h.k8shelper.IsServiceUpInNameSpace("rook-api") { + if !h.k8shelper.IsServiceUp("rook-api", clusterNamespace) { logger.Infof("Rook Cluster couldn't start") } else { logger.Infof("Rook Cluster started") @@ -158,9 +160,7 @@ func (h *InstallHelper) CreateK8sRookCluster() (err error) { } //InstallRookOnK8s installs rook on k8s -func (h *InstallHelper) InstallRookOnK8s() (err error) { - //creating clusterrolebinding for kubeadm env. - h.k8shelper.Kubectl([]string{"create", "clusterrolebinding", "anon-user-access", "--clusterrole", "cluster-admin", "--user", "system:anonymous"}...) +func (h *InstallHelper) InstallRookOnK8s(clusterNamespace string) (err error) { //flag used for local debuggin purpose, when rook is pre-installed skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true") @@ -191,7 +191,7 @@ func (h *InstallHelper) InstallRookOnK8s() (err error) { time.Sleep(10 * time.Second) //Create rook cluster - err = h.CreateK8sRookCluster() + err = h.CreateK8sRookCluster(clusterNamespace) if err != nil { panic(err) } @@ -199,16 +199,16 @@ func (h *InstallHelper) InstallRookOnK8s() (err error) { time.Sleep(5 * time.Second) //Create rook client - err = h.CreateK8sRookToolbox() + err = h.CreateK8sRookToolbox(clusterNamespace) if err != nil { panic(err) } - logger.Infof("installed rook on k8s %s", h.Env.K8sVersion) + logger.Infof("installed rook operator and cluster : %s on k8s %s", clusterNamespace, h.Env.K8sVersion) return nil } //UninstallRookFromK8s uninstalls rook from k8s -func (h *InstallHelper) UninstallRookFromK8s() { +func (h *InstallHelper) UninstallRookFromK8s(clusterNamespace string, helmInstalled bool) { //flag used for local debugging purpose, when rook is pre-installed skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true") if skipRookInstall { @@ -226,17 +226,24 @@ func (h *InstallHelper) UninstallRookFromK8s() { if err != nil { panic(err) } - rookOperator := h.installData.getRookOperator(k8sVersion) + if helmInstalled { + err = h.helmHelper.DeleteLocalRookHelmChart(helmDeployName) + if err != nil { + panic(err) + } + } else { + rookOperator := h.installData.GetRookOperator(k8sVersion) - _, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteArgs...) + _, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteArgs...) + if err != nil { + panic(err) + } + } + _, err = k8sHelp.DeleteResource([]string{"-n", clusterNamespace, "cluster", clusterNamespace}) if err != nil { panic(err) } - _, err = k8sHelp.DeleteResource([]string{"-n", "rook", "cluster", "rook"}) - if err != nil { - panic(err) - } - _, err = k8sHelp.DeleteResource([]string{"-n", "rook", "serviceaccount", "rook-api"}) + _, err = k8sHelp.DeleteResource([]string{"-n", clusterNamespace, "serviceaccount", "rook-api"}) if err != nil { panic(err) } @@ -262,103 +269,57 @@ func (h *InstallHelper) UninstallRookFromK8s() { panic(err) } } - _, err = k8sHelp.DeleteResource([]string{"secret", "rook-rook-user"}) + _, err = k8sHelp.DeleteResource([]string{"secret", clusterNamespace + "-rook-user"}) if err != nil { panic(err) } - _, err = k8sHelp.DeleteResource([]string{"namespace", "rook"}) + _, err = k8sHelp.DeleteResource([]string{"namespace", clusterNamespace}) if err != nil { panic(err) } - isRookUninstalled := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", "rook") - isNameSpaceDeleted := k8sHelp.WaitUntilNameSpaceIsDeleted("rook") + isRookUninstalled := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterNamespace) + isNameSpaceDeleted := k8sHelp.WaitUntilNameSpaceIsDeleted(clusterNamespace) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) - time.Sleep(time.Second * 10) - if isRookUninstalled && isNameSpaceDeleted { - logger.Infof("Rook uninstalled successfully") + logger.Infof("Rook cluster %s uninstalled successfully", clusterNamespace) return } - panic(fmt.Errorf("Rook not uninstalled")) + panic(fmt.Errorf("Rook cluster %s not uninstalled", clusterNamespace)) } -//UninstallRookFromK8sViaHelm uninstalls rook from k8s and deletes helm deploy -func (h *InstallHelper) UninstallRookFromK8sViaHelm() { - //flag used for local debugging purpose, when rook is pre-installed - skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true") - if skipRookInstall { - return - } +//CleanupCluster deletes a rook cluster for a namespace +func (h *InstallHelper) CleanupCluster(clusterName string, kv *utilversion.Version) { - serverVersion, err := h.k8shelper.Clientset.Discovery().ServerVersion() - if err != nil { - panic(err) - } - kubeVersion := utilversion.MustParseSemantic(serverVersion.GitVersion) - - logger.Infof("Uninstalling Rook") - k8sHelp, err := utils.CreateK8sHelper() + logger.Infof("Uninstalling All Rook Clusters - %s", clusterName) + _, err := h.k8shelper.DeleteResource([]string{"-n", clusterName, "cluster", clusterName}) if err != nil { + logger.Errorf("Rook Cluster %s cannot be deleted,err -> %v", clusterName, err) panic(err) } - err = h.helmHelper.DeleteLocalRookHelmChart(helmDeployName) + _, err = h.k8shelper.DeleteResource([]string{"-n", clusterName, "serviceaccount", "rook-api"}) if err != nil { + logger.Errorf("rook-api service account in namespace %s cannot be deleted,err -> %v", clusterName, err) panic(err) } - _, err = k8sHelp.DeleteResource([]string{"-n", "rook", "cluster", "rook"}) - if err != nil { - panic(err) - } - _, err = k8sHelp.DeleteResource([]string{"-n", "rook", "serviceaccount", "rook-api"}) - if err != nil { - panic(err) - } - if kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.6.0")) { - _, err = k8sHelp.DeleteResource([]string{"clusterrole", "rook-api"}) - if err != nil { - panic(err) - } - _, err = k8sHelp.DeleteResource([]string{"clusterrolebinding", "rook-api"}) - if err != nil { - panic(err) - } - } - - if kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.7.0")) { - _, err = k8sHelp.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io"}) - if err != nil { - panic(err) - } - } else { - _, err = k8sHelp.DeleteResource([]string{"thirdpartyresources", "cluster.rook.io", "pool.rook.io"}) - if err != nil { - panic(err) - } - } - _, err = k8sHelp.DeleteResource([]string{"secret", "rook-rook-user"}) - if err != nil { - panic(err) - } - _, err = k8sHelp.DeleteResource([]string{"namespace", "rook"}) + _, err = h.k8shelper.DeleteResource([]string{"secret", clusterName + "-rook-user"}) if err != nil { + logger.Errorf("rook-user secret in namespace %s cannot be deleted,err -> %v", clusterName, err) panic(err) } - isRookUninstalled := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", "rook") - isNameSpaceDeleted := k8sHelp.WaitUntilNameSpaceIsDeleted("rook") - h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) - - time.Sleep(time.Second * 10) - - if isRookUninstalled && isNameSpaceDeleted { - logger.Infof("Rook uninstalled successfully") - return + _, err = h.k8shelper.DeleteResource([]string{"namespace", clusterName}) + if err != nil { + logger.Errorf("namespace %s cannot be deleted,err -> %v", clusterName, err) + panic(err) } - panic(fmt.Errorf("Rook not uninstalled")) + + h.k8shelper.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterName) + h.k8shelper.WaitUntilNameSpaceIsDeleted(clusterName) + } //NewK8sRookhelper creates new instance of InstallHelper diff --git a/tests/framework/utils/helm_helper.go b/tests/framework/utils/helm_helper.go index c4ab373c9..ce0943372 100644 --- a/tests/framework/utils/helm_helper.go +++ b/tests/framework/utils/helm_helper.go @@ -1,3 +1,19 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package utils import ( diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index d8ec18ebb..cc24d006d 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -29,7 +29,6 @@ import ( "strconv" "github.com/coreos/pkg/capnslog" - "github.com/jmoiron/jsonq" "github.com/rook/rook/pkg/util/exec" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -207,22 +206,6 @@ type kuserContext struct { } `json:"user"` } -//GetMonIP returns IP address for a ceph mon pod -func (k8sh *K8sHelper) GetMonIP(mon string) (string, error) { - //kubectl -n rook get pod mon0 -o json|jq ".status.podIP"| - cmdArgs := []string{"-n", "rook", "get", "pod", mon, "-o", "json"} - result, err := k8sh.Kubectl(cmdArgs...) - if err == nil { - data := map[string]interface{}{} - dec := json.NewDecoder(strings.NewReader(result)) - dec.Decode(&data) - jq := jsonq.NewQuery(data) - ip, _ := jq.String("status", "podIP") - return fmt.Sprintf("%s:6790", ip), nil - } - return "", fmt.Errorf("Error Getting Monitor IP : %v", err) -} - //ResourceOperationFromTemplate performs a kubectl action from a template file after replacing its context func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinition string, config map[string]string) (string, error) { @@ -285,18 +268,18 @@ func (k8sh *K8sHelper) GetResource(args []string) (string, error) { } //GetMonitorServices returns all ceph mon pod names -func (k8sh *K8sHelper) GetMonitorServices() (map[string]string, error) { +func (k8sh *K8sHelper) GetMonitorServices(namespace string) (map[string]string, error) { + listOpts := metav1.ListOptions{LabelSelector: "app=rook-ceph-mon"} - args := []string{"-n", "rook", "get", "svc", "-l", "app=rook-ceph-mon", "--no-headers=true"} - result, err := k8sh.Kubectl(args...) + podList, err := k8sh.Clientset.Services(namespace).List(listOpts) if err != nil { - return nil, fmt.Errorf("Failed to find mon services. %v", err) + logger.Errorf("Cannot get rook monitor pods in namespace %s, err: %v", namespace, err) + return nil, fmt.Errorf("Cannot get rook monitor pods in namespace %s, err: %v", namespace, err) } + mons := []string{} + for _, svc := range podList.Items { + mons = append(mons, svc.Spec.ClusterIP) - // Get the IP address from the 2nd position in the line - mons, err := parseMonEndpoints(result) - if err != nil { - return nil, err } if len(mons) != 3 { return nil, fmt.Errorf("Unexpected monitors: %+v", mons) @@ -309,143 +292,71 @@ func (k8sh *K8sHelper) GetMonitorServices() (map[string]string, error) { }, nil } -func parseMonEndpoints(input string) ([]string, error) { - lines := strings.Split(input, "\n") - mons := []string{} - for _, line := range lines { - if len(line) == 0 { - continue - } - fields := strings.Fields(line) - if len(fields) < 2 { - return nil, fmt.Errorf("Missing ip for mon service. %s", line) - } - mons = append(mons, strings.TrimSpace(fields[1])) - } - return mons, nil -} - //IsPodRunning returns true if a Pod is running status or goes to Running status within 90s else returns false -func (k8sh *K8sHelper) IsPodRunning(name string) bool { - args := []string{"get", "pod", name, "-o", "jsonpath='{.status.phase}'"} +func (k8sh *K8sHelper) IsPodRunning(name string, namespace string) bool { + getOpts := metav1.GetOptions{} inc := 0 for inc < RetryLoop { - result, err := k8sh.Kubectl(args...) + pod, err := k8sh.Clientset.Pods(namespace).Get(name, getOpts) if err == nil { - if strings.Contains(result, "Running") { + if pod.Status.Phase == "Running" { return true } - logger.Infof("Pod %s status: %s", name, result) } - - time.Sleep(RetryInterval * time.Second) inc++ + time.Sleep(RetryInterval * time.Second) + logger.Infof("waiting for pod %s in namespace %s to be running", name, namespace) } - logger.Infof("Giving up waiting for pod %s to be running", name) - return false -} - -//IsPodRunningInNamespace returns true if a Pod in a namespace is running status or goes to Running -// status within 90s else returns false -func (k8sh *K8sHelper) IsPodRunningInNamespace(name string) bool { - args := []string{"get", "pods", "-n", "rook", name, "-o", "jsonpath='{.status.phase}'"} - inc := 0 - for inc < RetryLoop { - result, err := k8sh.Kubectl(args...) - if err == nil { - if strings.Contains(result, "Running") { - return true - } - logger.Infof("Pod %s status: %s", name, result) - } - time.Sleep(RetryInterval * time.Second) - inc++ - - } - logger.Infof("Giving up waiting for pod %s to be running", name) + logger.Infof("Giving up waiting for pod %s in namespace %s to be running", name, namespace) return false } //IsPodTerminated returns true if a Pod is terminated status or goes to Terminated status // within 90s else returns false\ -func (k8sh *K8sHelper) IsPodTerminated(name string) bool { - args := []string{"get", "pods", name} +func (k8sh *K8sHelper) IsPodTerminated(name string, namespace string) bool { + getOpts := metav1.GetOptions{} inc := 0 for inc < RetryLoop { - _, err := k8sh.Kubectl(args...) + pod, err := k8sh.Clientset.Pods(namespace).Get(name, getOpts) if err != nil { - k8slogger.Infof("Pod in default namespace terminated: " + name) + k8slogger.Infof("Pod %s in namespace %s terminated ", name, namespace) return true } + k8slogger.Infof("waiting for Pod %s in namespace %s to terminated, status : %v", name, namespace, pod.Status.Phase) time.Sleep(RetryInterval * time.Second) inc++ } - k8slogger.Infof("Pod in default namespace did not terminated: " + name) - return false -} - -//IsPodTerminatedInNamespace returns true if a Pod in a namespace is terminated status -// or goes to Terminated status within 90s else returns false\ -func (k8sh *K8sHelper) IsPodTerminatedInNamespace(name string) bool { - args := []string{"get", "-n", "rook", "pods", name} - inc := 0 - for inc < RetryLoop { - _, err := k8sh.Kubectl(args...) - if err != nil { - k8slogger.Infof("Pod in rook namespace terminated: " + name) - return true - } - time.Sleep(RetryInterval * time.Second) - inc++ - - } - k8slogger.Infof("Pod in rook namespace did not terminated: " + name) + k8slogger.Infof("Pod %s in namespace %s did not terminated", name, namespace) return false } //IsServiceUp returns true if a service is up or comes up within 40s, else returns false -func (k8sh *K8sHelper) IsServiceUp(name string) bool { - args := []string{"get", "svc", name} +func (k8sh *K8sHelper) IsServiceUp(name string, namespace string) bool { + getOpts := metav1.GetOptions{} inc := 0 for inc < RetryLoop { - _, err := k8sh.Kubectl(args...) + _, err := k8sh.Clientset.Services(namespace).Get(name, getOpts) if err == nil { - k8slogger.Infof("Service in default namespace is up: " + name) + k8slogger.Infof("Service: %s in namespace: %s is up", name, namespace) return true } + k8slogger.Infof("waiting for Service %s in namespace %s ", name, namespace) time.Sleep(RetryInterval * time.Second) inc++ } - k8slogger.Infof("Service in default namespace is not up: " + name) - return false -} - -//IsServiceUpInNameSpace returns true if a service in a namespace is up or comes up within 40s, else returns false -func (k8sh *K8sHelper) IsServiceUpInNameSpace(name string) bool { - args := []string{"get", "svc", "-n", "rook", name} - inc := 0 - for inc < RetryLoop { - _, err := k8sh.Kubectl(args...) - if err == nil { - return true - } - time.Sleep(RetryInterval * time.Second) - inc++ - - } - k8slogger.Infof("Service in rook namespace is not up: " + name) + k8slogger.Infof("Giving up waiting for service: %s in namespace %s ", name, namespace) return false } //GetService returns output from "kubectl get svc $NAME" command -func (k8sh *K8sHelper) GetService(servicename string) (string, error) { - args := []string{"get", "svc", "-n", "rook", servicename} - result, err := k8sh.Kubectl(args...) +func (k8sh *K8sHelper) GetService(servicename string, namespace string) (*v1.Service, error) { + getOpts := metav1.GetOptions{} + result, err := k8sh.Clientset.Services(namespace).Get(servicename, getOpts) if err != nil { - return "", fmt.Errorf("Cannot find service %v -- %v", servicename, err) + return nil, fmt.Errorf("Cannot find service %s in namespace %s, err-- %v", servicename, namespace, err) } return result, nil } @@ -500,28 +411,30 @@ func (k8sh *K8sHelper) GetPodDetails(podNamePattern string, namespace string) (s //GetPodHostID returns HostIP address of a pod func (k8sh *K8sHelper) GetPodHostID(podNamePattern string, namespace string) (string, error) { - output, err := k8sh.GetPodDetails(podNamePattern, namespace) + listOpts := metav1.ListOptions{LabelSelector: "app=" + podNamePattern} + podList, err := k8sh.Clientset.Pods(namespace).List(listOpts) if err != nil { - return "", err + logger.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err) + return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err) } - podNames := strings.Split(output, "\n") - if len(podNames) == 0 { - return "", fmt.Errorf("pod %s not found", podNamePattern) + if podList.Size() < 1 { + logger.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err) + return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err) } + return podList.Items[0].Status.HostIP, nil +} - //get host Ip of the pod - args := []string{"get", podNames[0], "-o", "jsonpath='{.status.hostIP}'"} - if namespace != "" { - args = append(args, []string{"-n", namespace}...) +//GetServiceNodePort returns nodeProt of service +func (k8sh *K8sHelper) GetServiceNodePort(serviceName string, namespace string) (string, error) { + getOpts := metav1.GetOptions{} + svc, err := k8sh.Clientset.Services(namespace).Get(serviceName, getOpts) + if err != nil { + logger.Errorf("Cannot get service : %v in namespace %v, err: %v", serviceName, namespace, err) + return "", fmt.Errorf("Cannot get serice : %v in namespace %v, err: %v", serviceName, namespace, err) } - result, err := k8sh.Kubectl(args...) - if err == nil { - hostIP := strings.Replace(result, "'", "", -1) - return strings.TrimSpace(hostIP), nil - } - return "", fmt.Errorf("Error Getting Monitor IP -- %v", err) - + np := svc.Spec.Ports[0].NodePort + return strconv.FormatInt(int64(np), 10), nil } //IsStorageClassPresent returns true if storageClass is present, if not false @@ -578,19 +491,30 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m return false } - if podList.Size() < minExpected { + if len(podList.Items) < minExpected { logger.Errorf("Expected at least %d pods with name %v, actual count %d", minExpected, podName, podList.Size()) return false } - for _, pod := range podList.Items { - if !(pod.Status.Phase == v1.PodPhase(expectedPhase)) { - logger.Errorf("Expected pod %v to be in %v phase, actual phase %v ", pod.Name, expectedPhase, pod.Status.Phase) - return false + inc := 0 + for inc < RetryLoop { + r := true + pl, _ := k8sh.Clientset.Pods(namespace).List(listOpts) + for _, pod := range pl.Items { + if !(pod.Status.Phase == v1.PodPhase(expectedPhase)) { + r = false + logger.Infof("waiting for pod %v to be in %s Phase, currently in %v Phase", pod.Name, expectedPhase, pod.Status.Phase) + } } - } + if r { + return true + } + inc++ + time.Sleep(RetryInterval * time.Second) - return true + } + logger.Errorf("All pods with app Name %v not in %v phase ", podName, expectedPhase) + return false } @@ -665,14 +589,18 @@ func (k8sh *K8sHelper) WaitUntilNameSpaceIsDeleted(namespace string) bool { } //GetRGWServiceURL returns URL of ceph RGW service in the cluster -func (k8sh *K8sHelper) GetRGWServiceURL() (string, error) { - hostip, err := k8sh.GetPodHostID("rook-ceph-rgw", "rook") +func (k8sh *K8sHelper) GetRGWServiceURL(servicename string, namespace string) (string, error) { + hostip, err := k8sh.GetPodHostID("rook-ceph-rgw", namespace) if err != nil { panic(fmt.Errorf("RGW pods not found/object store possibly not started")) } //TODO - Get nodePort stop hardcoding - endpoint := hostip + ":30001" + nodePort, err := k8sh.GetServiceNodePort(servicename, namespace) + if err != nil { + panic(fmt.Errorf("RGW pods not found/object store possibly not started")) + } + endpoint := hostip + ":" + nodePort return endpoint, err } @@ -720,7 +648,7 @@ func (k8sh *K8sHelper) GetRookLogs(podAppName string, namespace string, testName continue } } - fileName := testName + "_" + podName + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".log" + fileName := fmt.Sprintf("%s_%s_%s_%d.log", testName, podName, namespace, time.Now().Unix()) fpath = path.Join(fpath, fileName) file, err := os.Create(fpath) if err != nil { @@ -736,3 +664,14 @@ func (k8sh *K8sHelper) GetRookLogs(podAppName string, namespace string, testName } } } + +//CreateAnonSystemClusterBinding Creates anon-user-access clusterrolebinding for cluster-admin role - used by kubeadm env. +func (k8sh *K8sHelper) CreateAnonSystemClusterBinding() { + args := []string{"create", "clusterrolebinding", "anon-user-access", "--clusterrole", "cluster-admin", "--user", "system:anonymous"} + _, err := k8sh.Kubectl(args...) + if err != nil { + logger.Warningf("anon-user-access not created") + return + } + logger.Warningf("anon-user-access created") +} diff --git a/tests/integration/block_createBlock_api_test.go b/tests/integration/block_createBlock_api_test.go index 0edcd470f..0868859a8 100644 --- a/tests/integration/block_createBlock_api_test.go +++ b/tests/integration/block_createBlock_api_test.go @@ -55,10 +55,10 @@ func (s *BlockImageCreateSuite) SetupSuite() { s.installer = installer.NewK8sRookhelper(kh.Clientset) - err = s.installer.InstallRookOnK8s() + err = s.installer.InstallRookOnK8s("rook") require.NoError(s.T(), err) - s.testClient, err = clients.CreateTestClient(enums.Kubernetes, kh) + s.testClient, err = clients.CreateTestClient(enums.Kubernetes, kh, "rook") require.Nil(s.T(), err) s.rc = s.testClient.GetRestAPIClient() @@ -158,6 +158,6 @@ func (s *BlockImageCreateSuite) TearDownTest() { } func (s *BlockImageCreateSuite) TearDownSuite() { - s.installer.UninstallRookFromK8s() + s.installer.UninstallRookFromK8s("rook", false) } diff --git a/tests/integration/block_createBlock_k8s_test.go b/tests/integration/block_createBlock_k8s_test.go index e52d8d26f..3dc45fb2f 100644 --- a/tests/integration/block_createBlock_k8s_test.go +++ b/tests/integration/block_createBlock_k8s_test.go @@ -58,10 +58,10 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() { s.installer = installer.NewK8sRookhelper(s.kh.Clientset) - err = s.installer.InstallRookOnK8s() + err = s.installer.InstallRookOnK8s("rook") require.NoError(s.T(), err) - s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh) + s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh, "rook") require.Nil(s.T(), err) s.bc = s.testClient.GetBlockClient() @@ -238,5 +238,5 @@ func (s *K8sBlockImageCreateSuite) isPVCBound(name string) bool { } func (s *K8sBlockImageCreateSuite) TearDownSuite() { - s.installer.UninstallRookFromK8s() + s.installer.UninstallRookFromK8s("rook", false) } diff --git a/tests/integration/block_e2e_k8s_test.go b/tests/integration/block_e2e_k8s_test.go index b49bfeea0..75edb45d5 100644 --- a/tests/integration/block_e2e_k8s_test.go +++ b/tests/integration/block_e2e_k8s_test.go @@ -59,10 +59,10 @@ func (s *K8sBlockEnd2EndIntegrationSuite) SetupSuite() { s.installer = installer.NewK8sRookhelper(s.kh.Clientset) - err = s.installer.InstallRookOnK8s() + err = s.installer.InstallRookOnK8s("rook") require.NoError(s.T(), err) - s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh) + s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh, "rook") require.Nil(s.T(), err) s.bc = s.testClient.GetBlockClient() @@ -225,5 +225,5 @@ func (s *K8sBlockEnd2EndIntegrationSuite) TearDownTest() { func (s *K8sBlockEnd2EndIntegrationSuite) TearDownSuite() { - s.installer.UninstallRookFromK8s() + s.installer.UninstallRookFromK8s("rook", false) } diff --git a/tests/longhaul/block_on_k8s_test.go b/tests/longhaul/block_on_k8s_test.go index 58614eca3..b268e5ed5 100644 --- a/tests/longhaul/block_on_k8s_test.go +++ b/tests/longhaul/block_on_k8s_test.go @@ -45,11 +45,11 @@ func (s *K8sBlockLongHaulSuite) SetupSuite() { s.installer = installer.NewK8sRookhelper(s.kh.Clientset) if !s.kh.IsRookInstalled() { - err = s.installer.InstallRookOnK8s() + err = s.installer.InstallRookOnK8s("rook") require.NoError(s.T(), err) } - s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh) + s.testClient, err = clients.CreateTestClient(enums.Kubernetes, s.kh, "rook") require.Nil(s.T(), err) s.bc = s.testClient.GetBlockClient() diff --git a/tests/scripts/kubeadm-install.sh b/tests/scripts/kubeadm-install.sh index c25f3b117..ea8f2eca2 100755 --- a/tests/scripts/kubeadm-install.sh +++ b/tests/scripts/kubeadm-install.sh @@ -1,6 +1,6 @@ #!/bin/bash +e -KUBE_VERSION=${1:-"v1.7.4"} +KUBE_VERSION=${1:-"v1.7.5"} null_str= KUBE_INSTALL_VERSION="${KUBE_VERSION/v/$null_str}"-00 diff --git a/tests/scripts/kubeadm.sh b/tests/scripts/kubeadm.sh index 0abd83128..ff7adea5b 100755 --- a/tests/scripts/kubeadm.sh +++ b/tests/scripts/kubeadm.sh @@ -5,7 +5,7 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" tarname=image.tar tarfile=${WORK_DIR}/tests/${tarname} -export KUBE_VERSION=${KUBE_VERSION:-"v1.7.4"} +export KUBE_VERSION=${KUBE_VERSION:-"v1.7.5"} install() { diff --git a/tests/scripts/minikube.sh b/tests/scripts/minikube.sh index 4f307526c..b6cf2207f 100755 --- a/tests/scripts/minikube.sh +++ b/tests/scripts/minikube.sh @@ -4,7 +4,7 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "${scriptdir}/../../build/common.sh" tarname=image.tar -tarfile=${WORK_DIR}/tests/${tarname} +tarfile=$(pwd)/_output/tests/${tarname} ssh_opts=( -o StrictHostKeyChecking=no @@ -52,7 +52,7 @@ copy_image_to_cluster() { } # configure dind-cluster -KUBE_VERSION=${KUBE_VERSION:-"v1.7.2"} +KUBE_VERSION=${KUBE_VERSION:-"v1.7.5"} case "${1:-}" in up) @@ -65,6 +65,8 @@ case "${1:-}" in copy_image_to_cluster ${BUILD_REGISTRY}/rook-amd64 rook/rook:master copy_image_to_cluster ${BUILD_REGISTRY}/toolbox-amd64 rook/toolbox:master + copy_image_to_cluster ceph/base ceph/base + ;; down) minikube stop diff --git a/tests/smoke/base_block_test.go b/tests/smoke/base_block_test.go new file mode 100644 index 000000000..751ed74b1 --- /dev/null +++ b/tests/smoke/base_block_test.go @@ -0,0 +1,277 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package smoke + +import ( + "time" + + "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "strings" +) + +var ( + blockMountPath = "/tmp/rook1" + blockPodName = "block-test" +) + +// Smoke Test for Block Storage - Test check the following operations on Block Storage in order +//Create,Mount,Write,Read,Unmount and Delete. +func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) { + poolName := "replicapool" + storageClassName := "rook-block" + blockName := "block-pv-claim" + + defer blockTestDataCleanUp(helper, k8sh, namespace, poolName, storageClassName, blockName) + logger.Infof("Block Storage End to End Integration Test - create, mount, write to, read from, and unmount") + logger.Infof("Running on Rook Cluster %s", namespace) + rbc := helper.GetBlockClient() + + logger.Infof("Step 0 : Get Initial List Block") + initBlockImages, _ := rbc.BlockList() + + logger.Infof("step 1: Create block storage") + cbErr := blockStorageOperation(helper, k8sh, namespace, poolName, storageClassName, blockName, "create") + require.Nil(s.T(), cbErr) + require.True(s.T(), retryBlockImageCountCheck(helper, k8sh, len(initBlockImages), 1), "Make sure a new block is created") + logger.Infof("Block Storage created successfully") + require.True(s.T(), k8sh.WaitUntilPVCIsBound(blockName), "Make sure PVC is Bound") + + logger.Infof("step 2: Mount block storage") + _, mtErr := rbc.BlockMap(getBlockPodDefintion(blockName), blockMountPath) + require.Nil(s.T(), mtErr) + require.True(s.T(), k8sh.IsPodRunning(blockPodName, defaultNamespace), "make sure block-test pod is in running state") + logger.Infof("Block Storage Mounted successfully") + + logger.Infof("step 3: Write to block storage") + _, wtErr := rbc.BlockWrite(blockPodName, blockMountPath, "Smoke Test Data form Block storage", "bsFile1", "") + require.Nil(s.T(), wtErr) + logger.Infof("Write to Block storage successfully") + + logger.Infof("step 4: Read from block storage") + read, rErr := rbc.BlockRead(blockPodName, blockMountPath, "bsFile1", "") + require.Nil(s.T(), rErr) + require.Contains(s.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged") + logger.Infof("Read from Block storage successfully") + + logger.Infof("step 5: Unmount block storage") + _, unmtErr := rbc.BlockUnmap(getBlockPodDefintion(blockName), blockMountPath) + require.Nil(s.T(), unmtErr) + require.True(s.T(), k8sh.IsPodTerminated(blockPodName, defaultNamespace), "make sure block-test pod is terminated") + logger.Infof("Block Storage unmounted successfully") + + logger.Infof("step 6: Deleting block storage") + dbErr := blockStorageOperation(helper, k8sh, namespace, poolName, storageClassName, blockName, "delete") + require.Nil(s.T(), dbErr) + require.True(s.T(), retryBlockImageCountCheck(helper, k8sh, len(initBlockImages), 0), "Make sure a block is deleted") + logger.Infof("Block Storage deleted successfully") +} + +func runBlockE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterNamespace string) { + logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc") + logger.Infof("Running on Rook Cluster %s", clusterNamespace) + + //Check initial number of blocks + defer blockTestDataCleanUp(helper, k8sh, clusterNamespace, "rookpool", "rook-block", "test-block") + bc := helper.GetBlockClient() + initialBlocks, err := bc.BlockList() + require.Nil(s.T(), err) + initBlockCount := len(initialBlocks) + + logger.Infof("step 1: Create StorageClass and pool") + sc := map[string]string{} + storageclass := getBlockPoolAndStorageClassDefinition(k8sh, clusterNamespace, "rookpool", "rook-block") + res1, err := k8sh.ResourceOperationFromTemplate("create", storageclass, sc) + require.Contains(s.T(), res1, "pool \"rookpool\" created", "Make sure test pool is created") + require.Contains(s.T(), res1, "storageclass \"rook-block\" created", "Make sure storageclass is created") + require.NoError(s.T(), err) + // see https://github.com/rook/rook/issues/767 + time.Sleep(10 * time.Second) + + logger.Infof("step 2: Create pvc") + pvc := getPvcDefinition("test-block-claim", "rook-block") + res2, err := k8sh.ResourceOperationFromTemplate("create", pvc, sc) + require.Contains(s.T(), res2, "persistentvolumeclaim \"test-block-claim\" created", "Make sure pvc is created") + require.NoError(s.T(), err) + + require.True(s.T(), isPVCBound(k8sh, "test-block-claim")) + + //Make sure new block is created + b, _ := bc.BlockList() + require.Equal(s.T(), initBlockCount+1, len(b), "Make sure new block image is created") + + //Delete pvc and storageclass + _, err = k8sh.ResourceOperationFromTemplate("delete", storageclass, sc) + require.NoError(s.T(), err) + _, err = k8sh.ResourceOperationFromTemplate("delete", pvc, sc) + require.NoError(s.T(), err) + time.Sleep(2 * time.Second) + + b, _ = bc.BlockList() + require.Equal(s.T(), initBlockCount, len(b), "Make sure new block image is deleted") + +} + +func blockTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace string, poolname string, storageclassname string, blockname string) { + logger.Infof("Cleaning up block storage") + helper.GetBlockClient().BlockUnmap(getBlockPodDefintion(blockname), blockMountPath) + blockStorageOperation(helper, k8sh, namespace, poolname, storageclassname, blockname, "delete") + cleanUpDymanicBlockStorage(helper) +} + +func blockStorageOperation(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace string, poolname string, storageclassname string, blockname string, action string) error { + poolStorageClassDef := getBlockPoolAndStorageClassDefinition(k8sh, namespace, poolname, storageclassname) + pvcDef := getPvcDefinition(blockname, storageclassname) + k8sh.ResourceOperation(action, poolStorageClassDef) + // see https://github.com/rook/rook/issues/767 + time.Sleep(10 * time.Second) + _, err := k8sh.ResourceOperation(action, pvcDef) + + return err +} + +// periodically checking if block image count has changed to expected value +// When creating pvc in k8s platform, it may take some time for the block Image to be bounded +func retryBlockImageCountCheck(helper *clients.TestClient, k8sh *utils.K8sHelper, imageCount, expectedChange int) bool { + inc := 0 + for inc < utils.RetryLoop { + logger.Infof("Getting list of blocks (expecting %d)", (imageCount + expectedChange)) + blockImages, _ := helper.GetBlockClient().BlockList() + if imageCount+expectedChange == len(blockImages) { + return true + } + time.Sleep(time.Second * utils.RetryInterval) + inc++ + } + return false +} + +//CleanUpDymanicBlockStorage is helper method to clean up bock storage created by tests +func cleanUpDymanicBlockStorage(helper *clients.TestClient) { + // Delete storage pool, storage class and pvc + blockImagesList, _ := helper.GetBlockClient().BlockList() + for _, blockImage := range blockImagesList { + helper.GetRestAPIClient().DeleteBlockImage(blockImage) + } + +} + +func getBlockPoolAndStorageClassDefinition(k8sh *utils.K8sHelper, namespace string, poolName string, storageclassName string) string { + k8sVersion := k8sh.GetK8sServerVersion() + if strings.Contains(k8sVersion, "v1.5") { + return `apiVersion: rook.io/v1alpha1 +kind: Pool +metadata: + name: ` + poolName + ` + namespace: ` + namespace + ` +spec: + replication: + size: 1 + # For an erasure-coded pool, comment out the replication size above and uncomment the following settings. + # Make sure you have enough OSDs to support the replica size or erasure code chunks. + #erasureCode: + # codingChunks: 2 + # dataChunks: 2 +--- +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: ` + storageclassName + ` +provisioner: rook.io/block +parameters: + pool: ` + poolName + ` + clusterName: ` + namespace + ` + clusterNamespace: ` + namespace + + } + return `apiVersion: rook.io/v1alpha1 +kind: Pool +metadata: + name: ` + poolName + ` + namespace: ` + namespace + ` +spec: + replication: + size: 1 + # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. + # Make sure you have enough OSDs to support the replica count or erasure code chunks. + #erasureCode: + # codingChunks: 2 + # dataChunks: 2 +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ` + storageclassName + ` +provisioner: rook.io/block +parameters: + pool: ` + poolName + ` + clusterName: ` + namespace + ` + clusterNamespace: ` + namespace +} + +func getPvcDefinition(blockName string, storageclassName string) string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ` + blockName + ` +spec: + storageClassName: ` + storageclassName + ` + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1M` +} + +func getBlockPodDefintion(blockName string) string { + return `apiVersion: v1 +kind: Pod +metadata: + name: block-test +spec: + containers: + - image: busybox + name: block-test1 + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: block-persistent-storage + mountPath: /tmp/rook1 + volumes: + - name: block-persistent-storage + persistentVolumeClaim: + claimName: ` + blockName + ` + restartPolicy: Never` +} + +func isPVCBound(k8sh *utils.K8sHelper, name string) bool { + inc := 0 + for inc < utils.RetryLoop { + status, _ := k8sh.GetPVCStatus(name) + if strings.TrimRight(status, "\n") == "'Bound'" { + return true + } + time.Sleep(time.Second * utils.RetryInterval) + inc++ + + } + return false +} diff --git a/tests/smoke/base_deploy_test.go b/tests/smoke/base_deploy_test.go new file mode 100644 index 000000000..e165d5116 --- /dev/null +++ b/tests/smoke/base_deploy_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package smoke + +import ( + "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +var ( + logger = capnslog.NewPackageLogger("github.com/rook/rook", "integrationTest") + + defaultNamespace = "default" + defaultRookNamespace = "test-rook" + clusterNamespace1 = "test-cluster-1" + clusterNamespace2 = "test-cluster-2" +) + +//Test to make sure all rook components are installed and Running +func checkIfRookClusterIsInstalled(k8sh *utils.K8sHelper, s suite.Suite, oNamespace string, cNamespace string) { + logger.Infof("Make sure all Pods in Rook Cluster %s are running", cNamespace) + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-operator", oNamespace, 1, "Running"), + "Make sure there is 1 rook-operator present in Running state") + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-api", cNamespace, 1, "Running"), + "Make sure there is 1 rook-api present in Running state") + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", cNamespace, 1, "Running"), + "Make sure there is 1 rook-ceph-mgr present in Running state") + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-osd", cNamespace, 1, "Running"), + "Make sure there is at lest 1 rook-ceph-osd present in Running state") + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mon", cNamespace, 3, "Running"), + "Make sure there are 3 rook-ceph-mon present in Running state") +} + +func gatherAllRookLogs(k8sh *utils.K8sHelper, s suite.Suite, oNamespace string, cNamespace string) { + logger.Infof("Gathering all logs from Rook Cluster %s", cNamespace) + k8sh.GetRookLogs("rook-operator", oNamespace, s.T().Name()) + k8sh.GetRookLogs("rook-api", cNamespace, s.T().Name()) + k8sh.GetRookLogs("rook-ceph-mgr", cNamespace, s.T().Name()) + k8sh.GetRookLogs("rook-ceph-mon", cNamespace, s.T().Name()) + k8sh.GetRookLogs("rook-ceph-osd", cNamespace, s.T().Name()) + k8sh.GetRookLogs("rook-ceph-rgw", cNamespace, s.T().Name()) + k8sh.GetRookLogs("rook-ceph-mds", cNamespace, s.T().Name()) +} diff --git a/tests/smoke/base_file_test.go b/tests/smoke/base_file_test.go new file mode 100644 index 000000000..2caf15d4e --- /dev/null +++ b/tests/smoke/base_file_test.go @@ -0,0 +1,152 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package smoke + +import ( + "fmt" + + "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +var ( + fileMountPath = "/tmp/rookfs" + filePodName = "file-test" +) + +// Smoke Test for File System Storage - Test check the following operations on FileSystem Storage in order +//Create,Mount,Write,Read,Unmount and Delete. +func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, fileSystemName string) { + defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, fileSystemName, fileMountPath) + logger.Infof("Running on Rook Cluster %s", namespace) + logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount") + rfc := helper.GetFileSystemClient() + + logger.Infof("Step 1: Create file System") + _, fscErr := rfc.FSCreate(fileSystemName) + require.Nil(s.T(), fscErr) + fileSystemList, _ := rfc.FSList() + require.Equal(s.T(), 1, len(fileSystemList), "There should one shared file system present") + filesystemData := fileSystemList[0] + require.Equal(s.T(), fileSystemName, filesystemData.Name, "make sure filesystem name matches") + logger.Infof("File system created") + + logger.Infof("Step 2: Mount file System") + mtfsErr := podWithFileSystem(k8sh, s, filePodName, namespace, fileSystemName, fileMountPath, "create") + require.Nil(s.T(), mtfsErr) + require.True(s.T(), k8sh.IsPodRunning(filePodName, defaultRookNamespace), "make sure file-test pod is in running state") + logger.Infof("File system mounted successfully") + + logger.Infof("Step 3: Write to file system") + _, wfsErr := rfc.FSWrite(filePodName, fileMountPath, "Smoke Test Data for file system storage", "fsFile1", namespace) + require.Nil(s.T(), wfsErr) + logger.Infof("Write to file system successful") + + logger.Infof("Step 4: Read from file system") + read, rdErr := rfc.FSRead(filePodName, fileMountPath, "fsFile1", namespace) + require.Nil(s.T(), rdErr) + require.Contains(s.T(), read, "Smoke Test Data for file system storage", "make sure content of the files is unchanged") + logger.Infof("Read from file system successful") + + logger.Infof("Step 5: UnMount file System") + umtfsErr := podWithFileSystem(k8sh, s, filePodName, namespace, fileSystemName, fileMountPath, "delete") + require.Nil(s.T(), umtfsErr) + require.True(s.T(), k8sh.IsPodTerminated(filePodName, defaultRookNamespace), "make sure file-test pod is terminated") + logger.Infof("File system mounted successfully") + + logger.Infof("Step 6: Deleting file storage") + rfc.FSDelete(fileSystemName) + //Delete is not deleting filesystem - known issue + //require.Nil(suite.T(), fsd_err) + logger.Infof("File system deleted") +} + +//Test File System Creation on Rook that was installed on a custom namespace i.e. Namespace != "rook" +func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, fileSystemName string) { + logger.Infof("File Storage End to End Integration Test - create FileSystem and make sure mds pod is running") + logger.Infof("Running on Rook Cluster %s", namespace) + fc := helper.GetFileSystemClient() + + logger.Infof("Step 1: Create file System") + _, fscErr := fc.FSCreate(fileSystemName) + require.Nil(s.T(), fscErr) + fileSystemList, _ := fc.FSList() + require.Equal(s.T(), 1, len(fileSystemList), "There should one shared file system present") + + logger.Infof("Step 2: Make sure rook-ceph-mds pod is running") + require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-mds", namespace, "Running"), + "Make sure rook-ceph-mds is in running state") + + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mds", namespace, 1, "Running"), + "Make sure there is 1 rook-ceph-mds present in Running state") + +} + +func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, fileSystemName string, fileMountPath string) { + logger.Infof("Cleaning up file system") + podWithFileSystem(k8sh, s, podname, namespace, fileSystemName, fileMountPath, "delete") + helper.GetFileSystemClient().FSDelete(fileSystemName) + +} + +func podWithFileSystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error { + mons, err := k8sh.GetMonitorServices(namespace) + require.Nil(s.T(), err) + + logger.Infof("mountFileStorage: Mons: %+v", mons) + _, err = k8sh.ResourceOperationFromTemplate(action, getFileSystemTestPod(podname, namespace, filesystemName, fileMountPath), mons) + if err != nil { + return fmt.Errorf("failed to %s pod -- %s. %+v", action, getFileSystemTestPod(podname, namespace, filesystemName, fileMountPath), err) + } + return nil +} + +func getFileSystemTestPod(podname string, namespace string, filesystemName string, fileMountPath string) string { + return `apiVersion: v1 +kind: Pod +metadata: + name: ` + podname + ` + namespace: ` + namespace + ` +spec: + containers: + - name: ` + podname + ` + image: busybox + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + env: + volumeMounts: + - mountPath: "` + fileMountPath + `" + name: ` + filesystemName + ` + volumes: + - name: ` + filesystemName + ` + cephfs: + monitors: + - {{.mon0}} + - {{.mon1}} + - {{.mon2}} + user: admin + secretRef: + name: rook-admin + readOnly: false + restartPolicy: Never +` +} diff --git a/tests/smoke/object_test.go b/tests/smoke/base_object_test.go similarity index 55% rename from tests/smoke/object_test.go rename to tests/smoke/base_object_test.go index 63a724463..3f7c86663 100644 --- a/tests/smoke/object_test.go +++ b/tests/smoke/base_object_test.go @@ -23,8 +23,11 @@ import ( "time" "github.com/rook/rook/pkg/model" + "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) var ( @@ -39,97 +42,119 @@ var ( // Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order //Create object store, Create User, Connect to Object Store, Create Bucket, Read/Write/Delete to bucket,Delete Bucket and //Delete user -func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { - defer suite.objectTestDataCleanUp() - oc := suite.helper.GetObjectClient() +func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) { + defer objectTestDataCleanUp(helper, k8sh, namespace) + oc := helper.GetObjectClient() - logger.Infof("Object Storage Smoke Test - Create Object Store, User,Bucket and read/write to bucket") + logger.Infof("Object Storage End To End Integration Test - Create Object Store, User,Bucket and read/write to bucket") + logger.Infof("Running on Rook Cluster %s", namespace) logger.Infof("Step 0 : Create Object Store") - cobsErr := suite.createObjectStore() - require.Nil(suite.T(), cobsErr) + cobsErr := createObjectStore(helper, k8sh, namespace, "default", 1) + require.Nil(s.T(), cobsErr) logger.Infof("Object store created successfully") logger.Infof("Step 1 : Create Object Store User") initialUsers, _ := oc.ObjectListUser() _, cosuErr := oc.ObjectCreateUser(userid, userdisplayname) - require.Nil(suite.T(), cosuErr) + require.Nil(s.T(), cosuErr) usersAfterCrate, _ := oc.ObjectListUser() - require.Equal(suite.T(), len(initialUsers)+1, len(usersAfterCrate), "Make sure user list count is increaded by 1") + require.Equal(s.T(), len(initialUsers)+1, len(usersAfterCrate), "Make sure user list count is increaded by 1") getuserData, guErr := oc.ObjectGetUser(userid) - require.Nil(suite.T(), guErr) - require.Equal(suite.T(), userid, getuserData.UserID, "Check user id returned") - require.Equal(suite.T(), userdisplayname, *getuserData.DisplayName, "Check user name returned") + require.Nil(s.T(), guErr) + require.Equal(s.T(), userid, getuserData.UserID, "Check user id returned") + require.Equal(s.T(), userdisplayname, *getuserData.DisplayName, "Check user name returned") logger.Infof("Object store user created successfully") logger.Infof("Step 2 : Get connection information") conninfo, conninfoError := oc.ObjectGetUser(userid) - require.Nil(suite.T(), conninfoError) - - s3endpoint, _ := suite.k8sh.GetRGWServiceURL() + require.Nil(s.T(), conninfoError) + rgwExternalServiceName := "rgw-external-" + namespace + s3endpoint, _ := k8sh.GetRGWServiceURL(rgwExternalServiceName, defaultRookNamespace) s3client := utils.CreateNewS3Helper(s3endpoint, *conninfo.AccessKey, *conninfo.SecretKey) logger.Infof("Step 3 : Create bucket") initialBuckets, _ := oc.ObjectBucketList() s3client.CreateBucket(bucketname) BucketsAfterCreate, _ := oc.ObjectBucketList() - require.Equal(suite.T(), len(initialBuckets)+1, len(BucketsAfterCreate), "Make sure new bucket is created") + require.Equal(s.T(), len(initialBuckets)+1, len(BucketsAfterCreate), "Make sure new bucket is created") bkt, _ := getBucket(bucketname, BucketsAfterCreate) - require.Equal(suite.T(), bucketname, bkt.Name) - require.Equal(suite.T(), userid, bkt.Owner) + require.Equal(s.T(), bucketname, bkt.Name) + require.Equal(s.T(), userid, bkt.Owner) logger.Infof("Bucket created in Object store successfully") logger.Infof("Step 4 : Put Object on bucket") initObjSize, initObjNum, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterCreate) - require.Equal(suite.T(), uint64(0), initObjSize) - require.Equal(suite.T(), uint64(0), initObjNum) + require.Equal(s.T(), uint64(0), initObjSize) + require.Equal(s.T(), uint64(0), initObjNum) _, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType) - require.Nil(suite.T(), poErr) + require.Nil(s.T(), poErr) BucketsAfterPut, _ := oc.ObjectBucketList() ObjSize, ObjNum, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterPut) - require.NotEmpty(suite.T(), ObjSize) - require.Equal(suite.T(), uint64(1), ObjNum) + require.NotEmpty(s.T(), ObjSize) + require.Equal(s.T(), uint64(1), ObjNum) logger.Infof("Object Created on bucket successfully") logger.Infof("Step 5 : Put Object from bucket") read, err := s3client.GetObjectInBucket(bucketname, objectKey) - require.Nil(suite.T(), err) - require.Equal(suite.T(), objBody, read) + require.Nil(s.T(), err) + require.Equal(s.T(), objBody, read) logger.Infof("Object retrieved from bucket successfully") logger.Infof("Step 6 : Delete Object on bucket") _, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey) - require.Nil(suite.T(), delobjErr) + require.Nil(s.T(), delobjErr) BucketsAfterOjbDelete, _ := oc.ObjectBucketList() ObjSize1, ObjNum1, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterOjbDelete) - require.Equal(suite.T(), uint64(0), ObjSize1) - require.Equal(suite.T(), uint64(0), ObjNum1) + require.Equal(s.T(), uint64(0), ObjSize1) + require.Equal(s.T(), uint64(0), ObjNum1) logger.Infof("Object deleted on bucket successfully") logger.Infof("Step 6 : Delete bucket") _, bkdelErr := s3client.DeleteBucket(bucketname) - require.Nil(suite.T(), bkdelErr) + require.Nil(s.T(), bkdelErr) BucketsAfterDelete, _ := oc.ObjectBucketList() - require.Equal(suite.T(), len(initialBuckets), len(BucketsAfterDelete), "Make sure new bucket is deleted") + require.Equal(s.T(), len(initialBuckets), len(BucketsAfterDelete), "Make sure new bucket is deleted") logger.Infof("Bucket deleted successfully") logger.Infof("Step 7 : Delete User") usersBeforeDelete, _ := oc.ObjectListUser() oc.ObjectDeleteUser(userid) usersAfterDelete, _ := oc.ObjectListUser() - require.Equal(suite.T(), len(usersBeforeDelete)-1, len(usersAfterDelete), "Make sure user list count is reducd by 1") + require.Equal(s.T(), len(usersBeforeDelete)-1, len(usersAfterDelete), "Make sure user list count is reducd by 1") logger.Infof("Object store user deleted successfully") } -func (suite *SmokeSuite) objectTestDataCleanUp() { +//Test Object StoreCreation on Rook that was installed via helm +func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, name string, replicaSize int) { + logger.Infof("Object Storage End To End Integration Test - Create Object Store and check if rgw service is Running") + logger.Infof("Running on Rook Cluster %s", namespace) + + logger.Infof("Step 1 : Create Object Store") + oc := helper.GetObjectClient() + oc.ObjectCreate(name, int32(replicaSize)) + + logger.Infof("Step 2 : check rook-ceph-rgw service status and count") + require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", namespace, "Running"), + "Make sure rook-ceph-rgw is in running state") + + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"), + "Make sure all rook-ceph-rgw pods are in Running state") + + require.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace)) + +} + +func objectTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace string) { + rgwExternalServiceName := "rgw-external-" + namespace logger.Infof("Cleaning up object store") - oc := suite.helper.GetObjectClient() + oc := helper.GetObjectClient() userinfo, err := oc.ObjectGetUser(userid) if err != nil { return //when user is not found } - s3endpoint, _ := suite.k8sh.GetRGWServiceURL() + s3endpoint, _ := k8sh.GetRGWServiceURL(rgwExternalServiceName, namespace) s3client := utils.CreateNewS3Helper(s3endpoint, *userinfo.AccessKey, *userinfo.SecretKey) s3client.DeleteObjectInBucket(bucketname, objectKey) s3client.DeleteBucket(bucketname) @@ -148,14 +173,15 @@ func getBucket(bucketname string, bucketList []model.ObjectBucket) (model.Object return model.ObjectBucket{}, errors.New("Bucket not found") } -func (suite *SmokeSuite) createObjectStore() error { - suite.helper.GetObjectClient().ObjectCreate() +func createObjectStore(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace string, name string, replicaSize int32) error { + rgwExternalServiceName := "rgw-external-" + namespace + helper.GetObjectClient().ObjectCreate(name, replicaSize) time.Sleep(time.Second * 2) //wait for rgw service to to started - if suite.k8sh.IsServiceUpInNameSpace("rook-ceph-rgw-default") { - _, err := suite.k8sh.GetService("rgw-external") + if k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace) { + _, err := k8sh.GetService(rgwExternalServiceName, namespace) if err != nil { - suite.k8sh.KubectlWithStdin(getRGWExternalServiceDef(), []string{"create", "-f", "-"}...) - if !suite.k8sh.IsServiceUpInNameSpace("rgw-external") { + k8sh.KubectlWithStdin(getRGWExternalServiceDef(namespace), []string{"create", "-f", "-"}...) + if !k8sh.IsServiceUp(rgwExternalServiceName, defaultRookNamespace) { logger.Infof("Couldn't start RGW external serivce") return fmt.Errorf("Cannot expose rgw servie for external user") } @@ -177,24 +203,23 @@ func getBucketSizeAndObjectes(bucketname string, bucketList []model.ObjectBucket } //Expose RGW service -func getRGWExternalServiceDef() string { +func getRGWExternalServiceDef(namespace string) string { return `apiVersion: v1 kind: Service metadata: - name: rgw-external - namespace: rook + name: rgw-external-` + namespace + ` + namespace: ` + namespace + ` labels: app: rook-ceph-rgw - rook_cluster: rook + rook_cluster: ` + namespace + ` spec: ports: - name: rook-ceph-rgw port: 53390 protocol: TCP - nodePort: 30001 selector: app: rook-ceph-rgw - rook_cluster: rook + rook_cluster: ` + namespace + ` sessionAffinity: None type: NodePort ` diff --git a/tests/smoke/block_test.go b/tests/smoke/block_test.go deleted file mode 100644 index 4e2a5e3b2..000000000 --- a/tests/smoke/block_test.go +++ /dev/null @@ -1,210 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package smoke - -import ( - "time" - - "strings" - - "github.com/rook/rook/tests/framework/utils" - "github.com/stretchr/testify/require" -) - -var ( - pvcName = "block-pv-claim" - blockMountPath = "/tmp/rook1" - blockPodName = "block-test" -) - -// Smoke Test for Block Storage - Test check the following operations on Block Storage in order -//Create,Mount,Write,Read,Unmount and Delete. -func (suite *SmokeSuite) TestBlockStorage_SmokeTest() { - defer suite.blockTestDataCleanUp() - logger.Infof("Block Storage Smoke Test - create, mount, write to, read from, and unmount") - rbc := suite.helper.GetBlockClient() - - logger.Infof("Step 0 : Get Initial List Block") - initBlockImages, _ := rbc.BlockList() - - logger.Infof("step 1: Create block storage") - cbErr := suite.blockStorageOperation("create") - require.Nil(suite.T(), cbErr) - require.True(suite.T(), suite.retryBlockImageCountCheck(len(initBlockImages), 1), "Make sure a new block is created") - logger.Infof("Block Storage created successfully") - require.True(suite.T(), suite.k8sh.WaitUntilPVCIsBound(pvcName), "Make sure PVC is Bound") - - logger.Infof("step 2: Mount block storage") - _, mtErr := rbc.BlockMap(getBlockPodDefintion(), blockMountPath) - require.Nil(suite.T(), mtErr) - require.True(suite.T(), suite.k8sh.IsPodRunning(blockPodName), "make sure block-test pod is in running state") - logger.Infof("Block Storage Mounted successfully") - - logger.Infof("step 3: Write to block storage") - _, wtErr := rbc.BlockWrite(blockPodName, blockMountPath, "Smoke Test Data form Block storage", "bsFile1", "") - require.Nil(suite.T(), wtErr) - logger.Infof("Write to Block storage successfully") - - logger.Infof("step 4: Read from block storage") - read, rErr := rbc.BlockRead(blockPodName, blockMountPath, "bsFile1", "") - require.Nil(suite.T(), rErr) - require.Contains(suite.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged") - logger.Infof("Read from Block storage successfully") - - logger.Infof("step 5: Unmount block storage") - _, unmtErr := rbc.BlockUnmap(getBlockPodDefintion(), blockMountPath) - require.Nil(suite.T(), unmtErr) - require.True(suite.T(), suite.k8sh.IsPodTerminated(blockPodName), "make sure block-test pod is terminated") - logger.Infof("Block Storage unmounted successfully") - - logger.Infof("step 6: Deleting block storage") - dbErr := suite.blockStorageOperation("delete") - require.Nil(suite.T(), dbErr) - require.True(suite.T(), suite.retryBlockImageCountCheck(len(initBlockImages), 0), "Make sure a block is deleted") - logger.Infof("Block Storage deleted successfully") - -} - -func (suite *SmokeSuite) blockTestDataCleanUp() { - logger.Infof("Cleaning up block storage") - suite.helper.GetBlockClient().BlockUnmap(getBlockPodDefintion(), blockMountPath) - suite.blockStorageOperation("delete") - suite.cleanUpDymanicBlockStorage() -} - -func (suite *SmokeSuite) blockStorageOperation(action string) error { - poolStorageClassDef := suite.getBlockPoolAndStorageClassDefinition() - pvcDef := getPvcDefiniton() - suite.k8sh.ResourceOperation(action, poolStorageClassDef) - // see https://github.com/rook/rook/issues/767 - time.Sleep(10 * time.Second) - _, err := suite.k8sh.ResourceOperation(action, pvcDef) - - return err -} - -// periodically checking if block image count has changed to expected value -// When creating pvc in k8s platform, it may take some time for the block Image to be bounded -func (suite *SmokeSuite) retryBlockImageCountCheck(imageCount, expectedChange int) bool { - inc := 0 - for inc < utils.RetryLoop { - logger.Infof("Getting list of blocks (expecting %d)", (imageCount + expectedChange)) - blockImages, _ := suite.helper.GetBlockClient().BlockList() - if imageCount+expectedChange == len(blockImages) { - return true - } - time.Sleep(time.Second * utils.RetryInterval) - inc++ - } - return false -} - -//CleanUpDymanicBlockStorage is helper method to clean up bock storage created by tests -func (suite *SmokeSuite) cleanUpDymanicBlockStorage() { - // Delete storage pool, storage class and pvc - blockImagesList, _ := suite.helper.GetBlockClient().BlockList() - for _, blockImage := range blockImagesList { - suite.helper.GetRestAPIClient().DeleteBlockImage(blockImage) - } - -} - -func (suite *SmokeSuite) getBlockPoolAndStorageClassDefinition() string { - k8sVersion := suite.k8sh.GetK8sServerVersion() - if strings.Contains(k8sVersion, "v1.5") { - return `apiVersion: rook.io/v1alpha1 -kind: Pool -metadata: - name: replicapool - namespace: rook -spec: - replication: - size: 1 - # For an erasure-coded pool, comment out the replication size above and uncomment the following settings. - # Make sure you have enough OSDs to support the replica size or erasure code chunks. - #erasureCode: - # codingChunks: 2 - # dataChunks: 2 ---- -apiVersion: storage.k8s.io/v1beta1 -kind: StorageClass -metadata: - name: rook-block -provisioner: rook.io/block -parameters: - pool: replicapool` - - } - return `apiVersion: rook.io/v1alpha1 -kind: Pool -metadata: - name: replicapool - namespace: rook -spec: - replication: - size: 1 - # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. - # Make sure you have enough OSDs to support the replica count or erasure code chunks. - #erasureCode: - # codingChunks: 2 - # dataChunks: 2 ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: rook-block -provisioner: rook.io/block -parameters: - pool: replicapool` -} - -func getPvcDefiniton() string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: block-pv-claim - annotations: - volume.beta.kubernetes.io/storage-class: rook-block -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1M` -} -func getBlockPodDefintion() string { - return `apiVersion: v1 -kind: Pod -metadata: - name: block-test -spec: - containers: - - image: busybox - name: block-test1 - command: - - sleep - - "3600" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: block-persistent-storage - mountPath: /tmp/rook1 - volumes: - - name: block-persistent-storage - persistentVolumeClaim: - claimName: block-pv-claim - restartPolicy: Never` -} diff --git a/tests/smoke/file_test.go b/tests/smoke/file_test.go deleted file mode 100644 index 4654b9db0..000000000 --- a/tests/smoke/file_test.go +++ /dev/null @@ -1,128 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package smoke - -import ( - "fmt" - - "github.com/stretchr/testify/require" -) - -var ( - fileSystemName = "testfs" - fileMountPath = "/tmp/rookfs" - filePodName = "file-test" - filePodNamespace = "rook" -) - -// Smoke Test for File System Storage - Test check the following operations on FileSystem Storage in order -//Create,Mount,Write,Read,Unmount and Delete. -func (suite *SmokeSuite) TestFileStorage_SmokeTest() { - defer suite.fileTestDataCleanUp() - logger.Infof("File Storage Smoke Test - create, mount, write to, read from, and unmount") - rfc := suite.helper.GetFileSystemClient() - - logger.Infof("Step 1: Create file System") - _, fscErr := rfc.FSCreate(fileSystemName) - require.Nil(suite.T(), fscErr) - fileSystemList, _ := rfc.FSList() - require.Equal(suite.T(), 1, len(fileSystemList), "There should one shared file system present") - filesystemData := fileSystemList[0] - require.Equal(suite.T(), "testfs", filesystemData.Name, "make sure filesystem name matches") - logger.Infof("File system created") - - logger.Infof("Step 2: Mount file System") - mtfsErr := suite.podWithFileSystem("create") - require.Nil(suite.T(), mtfsErr) - require.True(suite.T(), suite.k8sh.IsPodRunningInNamespace(filePodName), "make sure file-test pod is in running state") - logger.Infof("File system mounted successfully") - - logger.Infof("Step 3: Write to file system") - _, wfsErr := rfc.FSWrite(filePodName, fileMountPath, "Smoke Test Data for file system storage", "fsFile1", filePodNamespace) - require.Nil(suite.T(), wfsErr) - logger.Infof("Write to file system successful") - - logger.Infof("Step 4: Read from file system") - read, rdErr := rfc.FSRead(filePodName, fileMountPath, "fsFile1", filePodNamespace) - require.Nil(suite.T(), rdErr) - require.Contains(suite.T(), read, "Smoke Test Data for file system storage", "make sure content of the files is unchanged") - logger.Infof("Read from file system successful") - - logger.Infof("Step 5: UnMount file System") - umtfsErr := suite.podWithFileSystem("delete") - require.Nil(suite.T(), umtfsErr) - require.True(suite.T(), suite.k8sh.IsPodTerminatedInNamespace(filePodName), "make sure file-test pod is terminated") - logger.Infof("File system mounted successfully") - - logger.Infof("Step 6: Deleting file storage") - rfc.FSDelete(fileSystemName) - //Delete is not deleting filesystem - known issue - //require.Nil(suite.T(), fsd_err) - logger.Infof("File system deleted") -} - -func (suite *SmokeSuite) fileTestDataCleanUp() { - logger.Infof("Cleaning up file system") - suite.podWithFileSystem("delete") - suite.helper.GetFileSystemClient().FSDelete(fileSystemName) - -} - -func (suite *SmokeSuite) podWithFileSystem(action string) error { - mons, err := suite.k8sh.GetMonitorServices() - require.Nil(suite.T(), err) - - logger.Infof("mountFileStorage: Mons: %+v", mons) - _, err = suite.k8sh.ResourceOperationFromTemplate(action, getFileSystemTestPod(), mons) - if err != nil { - return fmt.Errorf("failed to %s pod -- %s. %+v", action, getFileSystemTestPod(), err) - } - return nil -} - -func getFileSystemTestPod() string { - return `apiVersion: v1 -kind: Pod -metadata: - name: file-test - namespace: rook -spec: - containers: - - name: file-test1 - image: busybox - command: - - sleep - - "3600" - imagePullPolicy: IfNotPresent - env: - volumeMounts: - - mountPath: "/tmp/rookfs" - name: testfs - volumes: - - name: testfs - cephfs: - monitors: - - {{.mon0}} - - {{.mon1}} - - {{.mon2}} - user: admin - secretRef: - name: rook-admin - readOnly: false - restartPolicy: Never -` -} diff --git a/tests/smoke/helm_test.go b/tests/smoke/helm_test.go index 34545c419..73437cc4e 100644 --- a/tests/smoke/helm_test.go +++ b/tests/smoke/helm_test.go @@ -1,19 +1,16 @@ package smoke import ( - "strings" "testing" - "time" - "github.com/coreos/pkg/capnslog" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/enums" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "time" ) var ( @@ -26,12 +23,10 @@ func TestHelmSmokeSuite(t *testing.T) { type HelmSmokeSuite struct { suite.Suite - helper *clients.TestClient - k8sh *utils.K8sHelper - installer *installer.InstallHelper - hh *utils.HelmHelper - pvcDef string - storageclassDef string + helper *clients.TestClient + k8sh *utils.K8sHelper + installer *installer.InstallHelper + hh *utils.HelmHelper } func (hs *HelmSmokeSuite) SetupSuite() { @@ -43,181 +38,53 @@ func (hs *HelmSmokeSuite) SetupSuite() { hs.installer = installer.NewK8sRookhelper(kh.Clientset) - err = hs.installer.CreateK8sRookOperatorViaHelm() + err = hs.installer.CreateK8sRookOperatorViaHelm(defaultRookNamespace) require.NoError(hs.T(), err) - require.True(hs.T(), kh.IsPodInExpectedState("rook-operator", "rook", "Running"), + require.True(hs.T(), kh.IsPodInExpectedState("rook-operator", defaultRookNamespace, "Running"), "Make sure rook-operator is in running state") time.Sleep(10 * time.Second) - err = hs.installer.CreateK8sRookCluster() + err = hs.installer.CreateK8sRookCluster(defaultRookNamespace) require.NoError(hs.T(), err) - err = hs.installer.CreateK8sRookToolbox() + err = hs.installer.CreateK8sRookToolbox(defaultRookNamespace) require.NoError(hs.T(), err) - hs.helper, err = clients.CreateTestClient(enums.Kubernetes, kh) + hs.helper, err = clients.CreateTestClient(enums.Kubernetes, kh, defaultRookNamespace) require.Nil(hs.T(), err) - - hs.pvcDef = `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{.claimName}} - annotations: - volume.beta.kubernetes.io/storage-class: rook-block -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1M` - - hs.storageclassDef = `apiVersion: rook.io/v1alpha1 -kind: Pool -metadata: - name: {{.poolName}} - namespace: rook -spec: - replication: - size: 1 - # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. - # Make sure you have enough OSDs to support the replica count or erasure code chunks. - #erasureCode: - # codingChunks: 2 - # dataChunks: 2 ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: rook-block -provisioner: rook.io/block -parameters: - pool: {{.poolName}}` } func (hs *HelmSmokeSuite) TearDownSuite() { if hs.T().Failed() { - hs.k8sh.GetRookLogs("rook-operator", "default", hs.T().Name()) - hs.k8sh.GetRookLogs("rook-api", "rook", hs.T().Name()) - hs.k8sh.GetRookLogs("rook-ceph-mgr", "rook", hs.T().Name()) - hs.k8sh.GetRookLogs("rook-ceph-mon", "rook", hs.T().Name()) - hs.k8sh.GetRookLogs("rook-ceph-osd", "rook", hs.T().Name()) - hs.k8sh.GetRookLogs("rook-ceph-rgw", "rook", hs.T().Name()) - hs.k8sh.GetRookLogs("rook-ceph-mds", "rook", hs.T().Name()) + gatherAllRookLogs(hs.k8sh, hs.Suite, defaultRookNamespace, defaultRookNamespace) + } - hs.installer.UninstallRookFromK8sViaHelm() + hs.installer.UninstallRookFromK8s(defaultRookNamespace, true) } //Test to make sure all rook components are installed and Running func (hs *HelmSmokeSuite) TestRookInstallViaHelm() { + checkIfRookClusterIsInstalled(hs.k8sh, hs.Suite, defaultRookNamespace, defaultRookNamespace) - assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-operator", "default", 1, "Running"), - "Make sure there is 1 rook-operator present in Running state") - assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-api", "rook", 1, "Running"), - "Make sure there is 1 rook-api present in Running state") - assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-mgr", "rook", 1, "Running"), - "Make sure there is 1 rook-ceph-mgr present in Running state") - assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-osd", "rook", 1, "Running"), - "Make sure there is at lest 1 rook-ceph-osd present in Running state") - assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-mon", "rook", 3, "Running"), - "Make sure there are 3 rook-ceph-mon present in Running state") } -//Test BlockCreation on Rook that was installed via Helm charts +//Test BlockCreation on Rook that was installed via Helm func (hs *HelmSmokeSuite) TestBlockStoreOnRookInstalledViaHelm() { - - //Check initial number of blocks - bc := hs.helper.GetBlockClient() - initialBlocks, err := bc.BlockList() - require.Nil(hs.T(), err) - initBlockCount := len(initialBlocks) - - //deploy storageclass - sc := map[string]string{ - "poolName": "helm-pool", - } - - res1, err := hs.k8sh.ResourceOperationFromTemplate("create", hs.storageclassDef, sc) - require.Contains(hs.T(), res1, "pool \"helm-pool\" created", "Make sure test pool is created") - require.Contains(hs.T(), res1, "storageclass \"rook-block\" created", "Make sure storageclass is created") - require.NoError(hs.T(), err) - // see https://github.com/rook/rook/issues/767 - time.Sleep(10 * time.Second) - - // deploy pvc - vc := map[string]string{ - "claimName": "helm-claim", - } - res2, err := hs.k8sh.ResourceOperationFromTemplate("create", hs.pvcDef, vc) - require.Contains(hs.T(), res2, "persistentvolumeclaim \"helm-claim\" created", "Make sure pvc is created") - require.NoError(hs.T(), err) - - require.True(hs.T(), hs.isPVCBound("helm-claim")) - - //Make sure new block is created - b, _ := bc.BlockList() - require.Equal(hs.T(), initBlockCount+1, len(b), "Make sure new block image is created") - - //Delete pvc and storageclass - _, err = hs.k8sh.ResourceOperationFromTemplate("delete", hs.storageclassDef, sc) - require.NoError(hs.T(), err) - _, err = hs.k8sh.ResourceOperationFromTemplate("delete", hs.pvcDef, vc) - require.NoError(hs.T(), err) - time.Sleep(2 * time.Second) - - b, _ = bc.BlockList() - require.Equal(hs.T(), initBlockCount, len(b), "Make sure new block image is deleted") + runBlockE2ETestLite(hs.helper, hs.k8sh, hs.Suite, defaultRookNamespace) } -//Test File System Creation on Rook that was installed via Helm charts +//Test File System Creation on Rook that was installed via helm func (hs *HelmSmokeSuite) TestFileStoreOnRookInstalledViaHelm() { - fc := hs.helper.GetFileSystemClient() - - //Create file System - _, fscErr := fc.FSCreate(fileSystemName) - require.Nil(hs.T(), fscErr) - fileSystemList, _ := fc.FSList() - require.Equal(hs.T(), 1, len(fileSystemList), "There should one shared file system present") - - //Make sure rook-ceph-mds pods are running - require.True(hs.T(), hs.k8sh.IsPodInExpectedState("rook-ceph-mds", "rook", "Running"), - "Make sure rook-operator is in running state") - - assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-mds", "rook", 1, "Running"), - "Make sure there is 1 rook-operator present in Running state") + runFileE2ETestLite(hs.helper, hs.k8sh, hs.Suite, defaultRookNamespace, "testFs") } -//Test Object StoreCreation on Rook that was installed via Helm charts +//Test Object StoreCreation on Rook that was installed via helm func (hs *HelmSmokeSuite) TestObjectStoreOnRookInstalledViaHelm() { - //Create Object store - oc := hs.helper.GetObjectClient() - oc.ObjectCreate() - - //Make sure rook-ceph-rgw pods are running - require.True(hs.T(), hs.k8sh.IsPodInExpectedState("rook-ceph-rgw", "rook", "Running"), - "Make sure rook-operator is in running state") - - assert.True(hs.T(), hs.k8sh.CheckPodCountAndState("rook-ceph-rgw", "rook", 2, "Running"), - "Make sure there is 1 rook-operator present in Running state") - - require.True(hs.T(), hs.k8sh.IsServiceUpInNameSpace("rook-ceph-rgw-default")) + runObjectE2ETestLite(hs.helper, hs.k8sh, hs.Suite, defaultRookNamespace, "default", 3) } - -func (hs *HelmSmokeSuite) isPVCBound(name string) bool { - inc := 0 - for inc < utils.RetryLoop { - status, _ := hs.k8sh.GetPVCStatus(name) - if strings.TrimRight(status, "\n") == "'Bound'" { - return true - } - time.Sleep(time.Second * utils.RetryInterval) - inc++ - - } - return false -} diff --git a/tests/smoke/mon_test.go b/tests/smoke/mon_test.go index d05289ad0..5488bccbb 100644 --- a/tests/smoke/mon_test.go +++ b/tests/smoke/mon_test.go @@ -29,9 +29,8 @@ import ( func (suite *SmokeSuite) TestMonFailover() { logger.Infof("Mon Failover Smoke Test") - namespace := "rook" opts := metav1.ListOptions{LabelSelector: "app=rook-ceph-mon"} - monReplicas, err := suite.k8sh.Clientset.ReplicaSets(namespace).List(opts) + monReplicas, err := suite.k8sh.Clientset.ReplicaSets(defaultRookNamespace).List(opts) require.Nil(suite.T(), err) assert.Equal(suite.T(), 3, len(monReplicas.Items)) @@ -39,12 +38,12 @@ func (suite *SmokeSuite) TestMonFailover() { logger.Infof("Killing mon %s", monToKill) propagation := metav1.DeletePropagationForeground delOptions := &metav1.DeleteOptions{PropagationPolicy: &propagation} - err = suite.k8sh.Clientset.ReplicaSets(namespace).Delete(monToKill, delOptions) + err = suite.k8sh.Clientset.ReplicaSets(defaultRookNamespace).Delete(monToKill, delOptions) require.Nil(suite.T(), err) // Wait for the health check to start a new monitor for i := 0; i < 10; i++ { - monReplicas, err := suite.k8sh.Clientset.ReplicaSets(namespace).List(opts) + monReplicas, err := suite.k8sh.Clientset.ReplicaSets(defaultRookNamespace).List(opts) require.Nil(suite.T(), err) // Make sure the old mon is not still alive diff --git a/tests/smoke/multi_cluster_deploy_test.go b/tests/smoke/multi_cluster_deploy_test.go new file mode 100644 index 000000000..920156fe6 --- /dev/null +++ b/tests/smoke/multi_cluster_deploy_test.go @@ -0,0 +1,178 @@ +package smoke + +import ( + "strings" + "testing" + "time" + + "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/enums" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + utilversion "k8s.io/kubernetes/pkg/util/version" +) + +func TestInstallingMultipleRookCluster(t *testing.T) { + suite.Run(t, new(MultiRookClusterDeploySuite)) +} + +type MultiRookClusterDeploySuite struct { + suite.Suite + helper1 *clients.TestClient + helper2 *clients.TestClient + k8sh *utils.K8sHelper + installer *installer.InstallHelper + installData *installer.InstallData +} + +//Deploy Multiple Rook clusters +func (mrc *MultiRookClusterDeploySuite) SetupSuite() { + kh, err := utils.CreateK8sHelper() + require.NoError(mrc.T(), err) + + mrc.k8sh = kh + mrc.installer = installer.NewK8sRookhelper(kh.Clientset) + mrc.installData = installer.NewK8sInstallData() + + err = mrc.installer.CreateK8sRookOperator() + require.NoError(mrc.T(), err) + + require.True(mrc.T(), kh.IsPodInExpectedState("rook-operator", "default", "Running"), + "Make sure rook-operator is in running state") + + time.Sleep(10 * time.Second) + + err = mrc.installer.CreateK8sRookCluster(clusterNamespace1) + require.NoError(mrc.T(), err) + + err = mrc.installer.CreateK8sRookToolbox(clusterNamespace1) + require.NoError(mrc.T(), err) + + err = mrc.installer.CreateK8sRookCluster(clusterNamespace2) + require.NoError(mrc.T(), err) + + err = mrc.installer.CreateK8sRookToolbox(clusterNamespace2) + require.NoError(mrc.T(), err) + + mrc.helper1, err = clients.CreateTestClient(enums.Kubernetes, kh, clusterNamespace1) + require.Nil(mrc.T(), err) + + mrc.helper2, err = clients.CreateTestClient(enums.Kubernetes, kh, clusterNamespace2) + require.Nil(mrc.T(), err) + +} + +func (mrc *MultiRookClusterDeploySuite) TearDownSuite() { + if mrc.T().Failed() { + gatherAllRookLogs(mrc.k8sh, mrc.Suite, defaultNamespace, clusterNamespace1) + gatherAllRookLogs(mrc.k8sh, mrc.Suite, defaultNamespace, clusterNamespace2) + } + deleteArgs := []string{"delete", "-f", "-"} + + skipRookInstall := strings.EqualFold(mrc.installer.Env.SkipInstallRook, "true") + if skipRookInstall { + return + } + k8sVersion := mrc.k8sh.GetK8sServerVersion() + serverVersion, err := mrc.k8sh.Clientset.Discovery().ServerVersion() + if err != nil { + panic(err) + } + kubeVersion := utilversion.MustParseSemantic(serverVersion.GitVersion) + + logger.Infof("Uninstalling All Rook Clusters") + k8sHelp, err := utils.CreateK8sHelper() + if err != nil { + panic(err) + } + rookOperator := mrc.installData.GetRookOperator(k8sVersion) + + //Delete rook operator + _, err = mrc.k8sh.KubectlWithStdin(rookOperator, deleteArgs...) + if err != nil { + logger.Errorf("Rook operator cannot be deleted,err -> %v", err) + panic(err) + } + + //delete rook cluster + mrc.installer.CleanupCluster(clusterNamespace1, kubeVersion) + mrc.installer.CleanupCluster(clusterNamespace2, kubeVersion) + + //Delete clusterrol and clustterrolebindings + if kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.6.0")) { + _, err = k8sHelp.DeleteResource([]string{"clusterrole", "rook-api"}) + if err != nil { + logger.Errorf("Clusterrole rook-api cannot be deleted") + panic(err) + } + _, err = k8sHelp.DeleteResource([]string{"clusterrolebinding", "rook-api"}) + if err != nil { + logger.Errorf("clusterrolebinding rook-api cannot be deleted") + panic(err) + } + } + + // Delete crd/tpr + if kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.7.0")) { + _, err = k8sHelp.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io"}) + if err != nil { + logger.Errorf("crd cannot be deleted") + panic(err) + } + } else { + _, err = k8sHelp.DeleteResource([]string{"thirdpartyresources", "cluster.rook.io", "pool.rook.io"}) + if err != nil { + logger.Errorf("tpr cannot be deleted") + panic(err) + } + } + + isRookUninstalled1 := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterNamespace1) + isNameSpaceDeleted1 := k8sHelp.WaitUntilNameSpaceIsDeleted(clusterNamespace1) + isRookUninstalled2 := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterNamespace2) + isNameSpaceDeleted2 := k8sHelp.WaitUntilNameSpaceIsDeleted(clusterNamespace2) + mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) + + if isRookUninstalled1 && isNameSpaceDeleted1 && isRookUninstalled2 && isNameSpaceDeleted2 { + logger.Infof("Rook clusters %s and %s uninstalled successfully", clusterNamespace1, clusterNamespace2) + return + } + logger.Infof("Rook clusters %s and %s not uninstalled successfully", clusterNamespace1, clusterNamespace2) + +} + +//Test to make sure all rook components are installed and Running +func (mrc *MultiRookClusterDeploySuite) TestInstallingMultipleRookClusters() { + + //Check if Rook cluster 1 is deployed successfully + checkIfRookClusterIsInstalled(mrc.k8sh, mrc.Suite, defaultNamespace, clusterNamespace1) + + //Check if Rook cluster 1 is deployed successfully + checkIfRookClusterIsInstalled(mrc.k8sh, mrc.Suite, defaultNamespace, clusterNamespace2) + +} + +//Test Block Store Creation on multiple rook clusters +func (mrc *MultiRookClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() { + runBlockE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, clusterNamespace1) + runBlockE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, clusterNamespace2) + +} + +//Test FileSystem Creation on multiple rook clusters +func (mrc *MultiRookClusterDeploySuite) TestFileStoreOnMultiRookCluster() { + runFileE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, clusterNamespace1, "test-fs-1") + //TODO - Known Issues #https://github.com/rook/rook/issues/970 + //runFileE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, clusterNamespace2, "test-fs-2") + +} + +//Test Object Store Creation on multiple rook clusters +func (mrc *MultiRookClusterDeploySuite) TestObjectStoreOnMultiRookCluster() { + runObjectE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, clusterNamespace1, "default-c1", 2) + //TODO - Known Issues #https://github.com/rook/rook/issues/970 + //runObjectE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, clusterNamespace2, "default-c2", 1) + +} diff --git a/tests/smoke/smoke_test.go b/tests/smoke/smoke_test.go index c8c9723a5..22a668e4c 100644 --- a/tests/smoke/smoke_test.go +++ b/tests/smoke/smoke_test.go @@ -19,8 +19,6 @@ package smoke import ( "testing" - "github.com/coreos/pkg/capnslog" - "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/enums" "github.com/rook/rook/tests/framework/installer" @@ -29,8 +27,6 @@ import ( "github.com/stretchr/testify/suite" ) -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "smoketest") - func TestSmokeSuiteK8s(t *testing.T) { suite.Run(t, new(SmokeSuite)) } @@ -50,22 +46,32 @@ func (suite *SmokeSuite) SetupSuite() { suite.installer = installer.NewK8sRookhelper(kh.Clientset) - err = suite.installer.InstallRookOnK8s() + err = suite.installer.InstallRookOnK8s(defaultRookNamespace) require.NoError(suite.T(), err) - suite.helper, err = clients.CreateTestClient(enums.Kubernetes, kh) + suite.helper, err = clients.CreateTestClient(enums.Kubernetes, kh, defaultRookNamespace) require.Nil(suite.T(), err) } func (suite *SmokeSuite) TearDownSuite() { if suite.T().Failed() { - suite.k8sh.GetRookLogs("rook-operator", "default", suite.T().Name()) - suite.k8sh.GetRookLogs("rook-api", "rook", suite.T().Name()) - suite.k8sh.GetRookLogs("rook-ceph-mgr", "rook", suite.T().Name()) - suite.k8sh.GetRookLogs("rook-ceph-mon", "rook", suite.T().Name()) - suite.k8sh.GetRookLogs("rook-ceph-osd", "rook", suite.T().Name()) - suite.k8sh.GetRookLogs("rook-ceph-rgw", "rook", suite.T().Name()) - suite.k8sh.GetRookLogs("rook-ceph-mds", "rook", suite.T().Name()) + gatherAllRookLogs(suite.k8sh, suite.Suite, defaultNamespace, defaultRookNamespace) + } - suite.installer.UninstallRookFromK8s() + suite.installer.UninstallRookFromK8s(defaultRookNamespace, false) +} + +func (suite *SmokeSuite) TestBlockStorage_SmokeTest() { + runBlockE2ETest(suite.helper, suite.k8sh, suite.Suite, defaultRookNamespace) +} +func (suite *SmokeSuite) TestFileStorage_SmokeTest() { + runFileE2ETest(suite.helper, suite.k8sh, suite.Suite, defaultRookNamespace, "smoke-test-fs") +} +func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { + runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, defaultRookNamespace) +} + +//Test to make sure all rook components are installed and Running +func (suite *SmokeSuite) TestRookClusterInstallation_smokeTest() { + checkIfRookClusterIsInstalled(suite.k8sh, suite.Suite, defaultNamespace, defaultRookNamespace) }