From da8fcaa6235652deb9e4a918cc72944b8ca4fff9 Mon Sep 17 00:00:00 2001 From: Divesh Angula Date: Fri, 21 Jul 2017 12:07:15 -0700 Subject: [PATCH] refactor integration tests. -rename tests following pattern area_functionalunit_platfrom_test.go -removing unused code -not accessing static test files --- tests/data/block/pvc.tmpl | 12 - tests/data/block/storageclass_pool.tmpl | 21 - tests/data/integration/mysqlapp.yaml | 68 --- tests/data/platform.properties | 0 tests/data/smoke/block_mount.yaml | 20 - tests/data/smoke/file_mount.tmpl | 29 - tests/data/smoke/pool_pvc.yaml | 12 - tests/data/smoke/pool_sc.yaml | 21 - tests/data/smoke/pool_sc_1_5.yaml | 24 - tests/data/smoke/rgw_external.yaml | 19 - tests/data/smoke/rook_api_external.yaml | 19 - tests/framework/clients/k8s_block.go | 107 ++-- tests/framework/clients/k8s_file_system.go | 88 +-- tests/framework/clients/pool.go | 3 +- tests/framework/clients/rest_client.go | 51 +- tests/framework/clients/test_client.go | 45 +- .../contracts/interface_docker_client.go | 25 - .../contracts/interface_transport_client.go | 27 - tests/framework/enums/docker_sub_command.go | 88 --- tests/framework/installer/install_data.go | 266 +++++++++ tests/framework/installer/install_helper.go | 98 ++-- tests/framework/objects/command_arg.go | 35 -- tests/framework/objects/docker_context.go | 47 -- .../transport/docker_transport_client.go | 53 -- .../transport/k8s_transport_client.go | 93 --- tests/framework/utils/exec_utils.go | 130 +---- tests/framework/utils/k8s_helper.go | 351 ++++++------ .../block_createBlock_api_test.go} | 4 +- .../block_createBlock_k8s_test.go} | 138 ++--- .../block_e2e_k8s_test.go} | 117 +++- .../block_on_k8s_test.go} | 120 +++- tests/scripts/minikube.sh | 11 +- tests/smoke/block_test.go | 151 ++++- tests/smoke/file_test.go | 83 ++- tests/smoke/object_test.go | 90 ++- tests/smoke/smoke_test.go | 12 +- tests/smoke/test_helper.go | 531 ------------------ 37 files changed, 1245 insertions(+), 1764 deletions(-) delete mode 100644 tests/data/block/pvc.tmpl delete mode 100644 tests/data/block/storageclass_pool.tmpl delete mode 100644 tests/data/integration/mysqlapp.yaml delete mode 100644 tests/data/platform.properties delete mode 100644 tests/data/smoke/block_mount.yaml delete mode 100644 tests/data/smoke/file_mount.tmpl delete mode 100644 tests/data/smoke/pool_pvc.yaml delete mode 100644 tests/data/smoke/pool_sc.yaml delete mode 100644 tests/data/smoke/pool_sc_1_5.yaml delete mode 100644 tests/data/smoke/rgw_external.yaml delete mode 100644 tests/data/smoke/rook_api_external.yaml delete mode 100644 tests/framework/contracts/interface_docker_client.go delete mode 100644 tests/framework/contracts/interface_transport_client.go delete mode 100644 tests/framework/enums/docker_sub_command.go create mode 100644 tests/framework/installer/install_data.go delete mode 100644 tests/framework/objects/command_arg.go delete mode 100644 tests/framework/objects/docker_context.go delete mode 100644 tests/framework/transport/docker_transport_client.go delete mode 100644 tests/framework/transport/k8s_transport_client.go rename tests/{block/create_block_test.go => integration/block_createBlock_api_test.go} (98%) rename tests/{block/k8s/create_block_test.go => integration/block_createBlock_k8s_test.go} (57%) rename tests/{block/k8s/integration/k8sBlockStorageIntegration_test.go => integration/block_e2e_k8s_test.go} (64%) rename tests/{block/k8s/longhaul/basicBlockonghaul_test.go => longhaul/block_on_k8s_test.go} (59%) delete mode 100644 tests/smoke/test_helper.go diff --git a/tests/data/block/pvc.tmpl b/tests/data/block/pvc.tmpl deleted file mode 100644 index a5f015b3e..000000000 --- a/tests/data/block/pvc.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{.claimName}} - annotations: - volume.beta.kubernetes.io/storage-class: rook-block -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1M diff --git a/tests/data/block/storageclass_pool.tmpl b/tests/data/block/storageclass_pool.tmpl deleted file mode 100644 index c88632f0b..000000000 --- a/tests/data/block/storageclass_pool.tmpl +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: rook.io/v1alpha1 -kind: Pool -metadata: - name: {{.poolName}} - namespace: rook -spec: - replication: - size: 1 - # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. - # Make sure you have enough OSDs to support the replica count or erasure code chunks. - #erasureCode: - # codingChunks: 2 - # dataChunks: 2 ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: rook-block -provisioner: rook.io/block -parameters: - pool: {{.poolName}} diff --git a/tests/data/integration/mysqlapp.yaml b/tests/data/integration/mysqlapp.yaml deleted file mode 100644 index 5bd4c2744..000000000 --- a/tests/data/integration/mysqlapp.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: mysql-app - labels: - app: mysqldb -spec: - ports: - - port: 3306 - targetPort: 3306 - protocol: TCP - nodePort: 30003 - selector: - app: mysqldb - tier: mysql - type: NodePort ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mysql-pv-claim - labels: - app: mysqldb -spec: - storageClassName: rook-block - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi ---- -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: mysql-app - labels: - app: mysqldb -spec: - strategy: - type: Recreate - template: - metadata: - labels: - app: mysqldb - tier: mysql - spec: - containers: - - image: mysql:5.6 - name: mysql - env: - - name: "MYSQL_USER" - value: "mysql" - - name: "MYSQL_PASSWORD" - value: "mysql" - - name: "MYSQL_DATABASE" - value: "sample" - - name: "MYSQL_ROOT_PASSWORD" - value: "root" - ports: - - containerPort: 3306 - name: mysql - volumeMounts: - - name: mysql-persistent-storage - mountPath: /var/lib/mysql - volumes: - - name: mysql-persistent-storage - persistentVolumeClaim: - claimName: mysql-pv-claim diff --git a/tests/data/platform.properties b/tests/data/platform.properties deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/data/smoke/block_mount.yaml b/tests/data/smoke/block_mount.yaml deleted file mode 100644 index e4197ed02..000000000 --- a/tests/data/smoke/block_mount.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: block-test -spec: - containers: - - image: busybox - name: block-test1 - command: - - sleep - - "3600" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: block-persistent-storage - mountPath: /tmp/rook1 - volumes: - - name: block-persistent-storage - persistentVolumeClaim: - claimName: block-pv-claim - restartPolicy: Never diff --git a/tests/data/smoke/file_mount.tmpl b/tests/data/smoke/file_mount.tmpl deleted file mode 100644 index f42d070af..000000000 --- a/tests/data/smoke/file_mount.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: file-test - namespace: rook -spec: - containers: - - name: file-test1 - image: busybox - command: - - sleep - - "3600" - imagePullPolicy: IfNotPresent - env: - volumeMounts: - - mountPath: "/tmp/rookfs" - name: testfs - volumes: - - name: testfs - cephfs: - monitors: - - {{.mon0}} - - {{.mon1}} - - {{.mon2}} - user: admin - secretRef: - name: rook-admin - readOnly: false - restartPolicy: Never diff --git a/tests/data/smoke/pool_pvc.yaml b/tests/data/smoke/pool_pvc.yaml deleted file mode 100644 index fde95ae11..000000000 --- a/tests/data/smoke/pool_pvc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: block-pv-claim - annotations: - volume.beta.kubernetes.io/storage-class: rook-block -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1M diff --git a/tests/data/smoke/pool_sc.yaml b/tests/data/smoke/pool_sc.yaml deleted file mode 100644 index 6c49d397f..000000000 --- a/tests/data/smoke/pool_sc.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: rook.io/v1alpha1 -kind: Pool -metadata: - name: replicapool - namespace: rook -spec: - replication: - size: 1 - # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. - # Make sure you have enough OSDs to support the replica count or erasure code chunks. - #erasureCode: - # codingChunks: 2 - # dataChunks: 2 ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: rook-block -provisioner: rook.io/block -parameters: - pool: replicapool diff --git a/tests/data/smoke/pool_sc_1_5.yaml b/tests/data/smoke/pool_sc_1_5.yaml deleted file mode 100644 index 0232ec76d..000000000 --- a/tests/data/smoke/pool_sc_1_5.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: rook.io/v1alpha1 -kind: Pool -metadata: - name: replicapool - namespace: rook -spec: - replication: - size: 1 - # For an erasure-coded pool, comment out the replication size above and uncomment the following settings. - # Make sure you have enough OSDs to support the replica size or erasure code chunks. - #erasureCode: - # codingChunks: 2 - # dataChunks: 2 ---- -apiVersion: storage.k8s.io/v1beta1 -kind: StorageClass -metadata: - name: rook-block -provisioner: rook.io/block -parameters: - pool: replicapool - # Specify the Rook cluster from which to create volumes. If not specified, it will use `rook` as the namespace and name of the cluster. - # clusterName: rook - # clusterNamespace: rook diff --git a/tests/data/smoke/rgw_external.yaml b/tests/data/smoke/rgw_external.yaml deleted file mode 100644 index a3a9c0de4..000000000 --- a/tests/data/smoke/rgw_external.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: rgw-external - namespace: rook - labels: - app: rook-ceph-rgw - rook_cluster: rook -spec: - ports: - - name: rook-ceph-rgw - port: 53390 - protocol: TCP - nodePort: 30001 - selector: - app: rook-ceph-rgw - rook_cluster: rook - sessionAffinity: None - type: NodePort diff --git a/tests/data/smoke/rook_api_external.yaml b/tests/data/smoke/rook_api_external.yaml deleted file mode 100644 index 5197298d8..000000000 --- a/tests/data/smoke/rook_api_external.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: rook-api-external - namespace: rook - labels: - app: rook-api - rook_cluster: rook -spec: - ports: - - name: rook-api - port: 8124 - protocol: TCP - nodePort: 30002 - selector: - app: rook-api - rook_cluster: rook - sessionAffinity: None - type: NodePort diff --git a/tests/framework/clients/k8s_block.go b/tests/framework/clients/k8s_block.go index a3732b25a..17416b2d9 100644 --- a/tests/framework/clients/k8s_block.go +++ b/tests/framework/clients/k8s_block.go @@ -21,12 +21,13 @@ import ( "github.com/rook/rook/pkg/model" "github.com/rook/rook/tests/framework/contracts" + "github.com/rook/rook/tests/framework/utils" ) //BlockOperation is wrapper for k8s rook block operations type BlockOperation struct { - transportClient contracts.ITransportClient - restClient contracts.RestAPIOperator + k8sClient *utils.K8sHelper + restClient contracts.RestAPIOperator } var ( @@ -35,36 +36,38 @@ var ( ) // CreateK8BlockOperation - Constructor to create BlockOperation - client to perform rook Block operations on k8s -func CreateK8BlockOperation(client contracts.ITransportClient, rookRestClient contracts.RestAPIOperator) *BlockOperation { - return &BlockOperation{transportClient: client, restClient: rookRestClient} +func CreateK8BlockOperation(k8shelp *utils.K8sHelper, rookRestClient contracts.RestAPIOperator) *BlockOperation { + return &BlockOperation{k8sClient: k8shelp, restClient: rookRestClient} } // BlockCreate Function to create a Block using Rook -// Input paramaters - -//name - path to a yaml file that creates a pvc in k8s - yaml should describe name and size of pvc being created +// Input parameters - +//name - pod definition that creates a pvc in k8s - yaml should describe name and size of pvc being created //size - not user for k8s implementation since its descried on the pvc yaml definition //Output - k8s create pvc operation output and/or error func (rb *BlockOperation) BlockCreate(name string, size int) (string, error) { - cmdArgs := []string{name} - out, err, status := rb.transportClient.Create(cmdArgs, nil) - if status == 0 { - return out, nil + args := []string{"create", "-f", "-"} + result, err := rb.k8sClient.KubectlWithStdin(name, args...) + if err != nil { + return "", fmt.Errorf("Unable to create block -- : %s", err) + } - return err, fmt.Errorf("Unable to create block -- : %s", err) + return result, nil } // BlockDelete Function to delete a Block using Rook -// Input paramaters - -//name - path to a yaml file that where pvc is described - delete is run on the the yaml definition +// Input parameters - +//name - pod definition where pvc is described - delete is run on the the yaml definition //Output - k8s delete pvc operation output and/or error func (rb *BlockOperation) BlockDelete(name string) (string, error) { - cmdArgs := []string{name} - out, err, status := rb.transportClient.Delete(cmdArgs, nil) - if status == 0 { - return out, nil + args := []string{"delete", "-f", "-"} + result, err := rb.k8sClient.KubectlWithStdin(name, args...) + if err != nil { + return "", fmt.Errorf("Unable to delete block -- : %s", err) + } - return err, fmt.Errorf("Unable to delete block -- : %s", err) + return result, nil } @@ -75,22 +78,23 @@ func (rb *BlockOperation) BlockList() ([]model.BlockImage, error) { } // BlockMap Function to map a Block using Rook -// Input paramaters - -//name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier +// Input parameters - +//name - Pod definition - pod should be defined to use a pvc that was created earlier //mountpath - not used in this impl since mountpath is defined in the pod definition //Output - k8s create pod operation output and/or error func (rb *BlockOperation) BlockMap(name string, mountpath string) (string, error) { - cmdArgs := []string{name} - out, err, status := rb.transportClient.Create(cmdArgs, nil) - if status == 0 { - return out, nil + args := []string{"create", "-f", "-"} + result, err := rb.k8sClient.KubectlWithStdin(name, args...) + if err != nil { + return "", fmt.Errorf("Unable to map block -- : %s", err) + } - return err, fmt.Errorf("Unable to Map block -- : %s", err) + return result, nil } //BlockWrite Function to write data to block created by rook ,i.e. write data to a pod that is using a pvc -// Input paramaters - +// Input parameters - //name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier //mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) //data - data to be written @@ -99,21 +103,24 @@ func (rb *BlockOperation) BlockMap(name string, mountpath string) (string, error //Output - k8s exec pod operation output and/or error func (rb *BlockOperation) BlockWrite(name string, mountpath string, data string, filename string, namespace string) (string, error) { wt := "echo \"" + data + "\">" + mountpath + "/" + filename - writeDataToBlockPod[2] = wt - option := []string{name} + args := []string{"exec", name} + if namespace != "" { - option = append(option, namespace) + args = append(args, "-n", namespace) } - out, err, status := rb.transportClient.Execute(writeDataToBlockPod, option) - if status == 0 { - return out, nil + args = append(args, "--", "sh", "-c", wt) + + result, err := rb.k8sClient.Kubectl(args...) + if err != nil { + return "", fmt.Errorf("Unable to write data to pod --: %s", err) + } - return err, fmt.Errorf("Unable to write data to pod: %s --> %s", err, out) + return result, nil } // BlockRead Function to read from block created by rook ,i.e. Read data from a pod that is using a pvc -// Input paramaters - +// Input parameters - //name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier //mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) //filename - file to be read @@ -121,30 +128,34 @@ func (rb *BlockOperation) BlockWrite(name string, mountpath string, data string, //Output - k8s exec pod operation output and/or error func (rb *BlockOperation) BlockRead(name string, mountpath string, filename string, namespace string) (string, error) { rd := mountpath + "/" + filename - readDataFromBlockPod[1] = rd - option := []string{name} + args := []string{"exec", name} + if namespace != "" { - option = append(option, namespace) + args = append(args, "-n", namespace) } - out, err, status := rb.transportClient.Execute(readDataFromBlockPod, option) - if status == 0 { - return out, nil + args = append(args, "--", "cat", rd) + + result, err := rb.k8sClient.Kubectl(args...) + if err != nil { + return "", fmt.Errorf("Unable to read data to pod -- : %s", err) + } - return err, fmt.Errorf("Unable to read data to pod -- : %s", err) + return result, nil } // BlockUnmap Function to map a Block using Rook -// Input paramaters - -//name - path to a yaml file - the pod described in yam file is deleted +// Input parameters - +//name - ppod definition - the pod described in yam file is deleted //mountpath - not used in this impl since mountpath is defined in the pod definition //Output - k8s delete pod operation output and/or error func (rb *BlockOperation) BlockUnmap(name string, mountpath string) (string, error) { - cmdArgs := []string{name} - out, err, status := rb.transportClient.Delete(cmdArgs, nil) - if status == 0 { - return out, nil + args := []string{"delete", "-f", "-"} + result, err := rb.k8sClient.KubectlWithStdin(name, args...) + if err != nil { + return "", fmt.Errorf("Unable to unmap block -- : %s", err) + } - return err, fmt.Errorf("Unable to unmap block -- : %s", err) + return result, nil } diff --git a/tests/framework/clients/k8s_file_system.go b/tests/framework/clients/k8s_file_system.go index 797b9d515..3be51e2b0 100644 --- a/tests/framework/clients/k8s_file_system.go +++ b/tests/framework/clients/k8s_file_system.go @@ -21,26 +21,22 @@ import ( "github.com/rook/rook/pkg/model" "github.com/rook/rook/tests/framework/contracts" + "github.com/rook/rook/tests/framework/utils" ) //FileSystemOperation is a wrapper for k8s rook file operations type FileSystemOperation struct { - transportClient contracts.ITransportClient - restClient contracts.RestAPIOperator + k8sClient *utils.K8sHelper + restClient contracts.RestAPIOperator } -var ( - writeDataToFilePod = []string{"sh", "-c", "WRITE_DATA_CMD"} - readDataFromFilePod = []string{"cat", "READ_DATA_CMD"} -) - // CreateK8sFileSystemOperation Constructor to create FileSystemOperation - client to perform rook file system operations on k8s -func CreateK8sFileSystemOperation(client contracts.ITransportClient, rookRestClient contracts.RestAPIOperator) *FileSystemOperation { - return &FileSystemOperation{transportClient: client, restClient: rookRestClient} +func CreateK8sFileSystemOperation(k8shelp *utils.K8sHelper, rookRestClient contracts.RestAPIOperator) *FileSystemOperation { + return &FileSystemOperation{k8sClient: k8shelp, restClient: rookRestClient} } //FSCreate Function to create a fileSystem in rook -//Input paramatres - +//Input parameters - // name - name of the shared file system to be created //Output - output returned by rook rest API client func (rfs *FileSystemOperation) FSCreate(name string) (string, error) { @@ -49,7 +45,7 @@ func (rfs *FileSystemOperation) FSCreate(name string) (string, error) { } //FSDelete Function to delete a fileSystem in rook -//Input paramatres - +//Input parameters - // name - name of the shared file system to be deleted //Output - output returned by rook rest API client func (rfs *FileSystemOperation) FSDelete(name string) (string, error) { @@ -65,22 +61,23 @@ func (rfs *FileSystemOperation) FSList() ([]model.Filesystem, error) { } //FSMount Function to Mount a file system created by rook(on a pod) -//Input paramaters - -//name - path to the yaml defintion file - definition of pod to be created that mounts existing file system +//Input parameters - +//name - path to the yaml definition file - definition of pod to be created that mounts existing file system //path - ignored in this case - moount path is defined in the path definition //output - output returned by k8s create pod operaton and/or error func (rfs *FileSystemOperation) FSMount(name string, path string) (string, error) { - cmdArgs := []string{name} - out, err, status := rfs.transportClient.Create(cmdArgs, nil) - if status == 0 { - return out, nil + cmdArgs := []string{"create", "-f", "name"} + result, err := rfs.k8sClient.Kubectl(cmdArgs...) + + if err != nil { + return "", fmt.Errorf("Unable to mount FileSystem -- : %s", err) } - return err, fmt.Errorf("Unable to mount FileSystem -- : %s", err) + return result, nil } //FSWrite Function to write data to file system created by rook ,i.e. write data to a pod that has filesystem mounted -// Input paramaters - +// Input parameters - //name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier //mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) //data - data to be written @@ -89,21 +86,24 @@ func (rfs *FileSystemOperation) FSMount(name string, path string) (string, error //Output - k8s exec pod operation output and/or error func (rfs *FileSystemOperation) FSWrite(name string, mountpath string, data string, filename string, namespace string) (string, error) { wt := "echo \"" + data + "\">" + mountpath + "/" + filename - writeDataToFilePod[2] = wt - option := []string{name} + args := []string{"exec", name} + if namespace != "" { - option = append(option, namespace) + args = append(args, "-n", namespace) } - out, err, status := rfs.transportClient.Execute(writeDataToFilePod, option) - if status == 0 { - return out, nil + args = append(args, "--", "sh", "-c", wt) + + result, err := rfs.k8sClient.Kubectl(args...) + if err != nil { + return "", fmt.Errorf("Unable to write data to pod -- : %s", err) + } - return err, fmt.Errorf("Unable to write data to pod -- : %s", err) + return result, nil } //FSRead Function to write read from file system created by rook ,i.e. Read data from a pod that filesystem mounted -// Input paramaters - +// Input parameters - //name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier //mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) //filename - file to be read @@ -111,30 +111,34 @@ func (rfs *FileSystemOperation) FSWrite(name string, mountpath string, data stri //Output - k8s exec pod operation output and/or error func (rfs *FileSystemOperation) FSRead(name string, mountpath string, filename string, namespace string) (string, error) { rd := mountpath + "/" + filename - readDataFromFilePod[1] = rd - option := []string{name} + + args := []string{"exec", name} + if namespace != "" { - option = append(option, namespace) + args = append(args, "-n", namespace) } - out, err, status := rfs.transportClient.Execute(readDataFromFilePod, option) - if status == 0 { - return out, nil + args = append(args, "--", "cat", rd) + + result, err := rfs.k8sClient.Kubectl(args...) + if err != nil { + return "", fmt.Errorf("Unable to write data to pod -- : %s", err) + } - return err, fmt.Errorf("Unable to write data to pod -- : %s", err) + return result, nil } //FSUnmount Function to UnMount a file system created by rook(delete pod) -//Input paramaters - -//name - path to the yaml defintion file - definition of pod to be deleted that has a file system mounted +//Input parameters - +//name - path to the yaml definition file - definition of pod to be deleted that has a file system mounted //path - ignored in this case - moount path is defined in the path definition //output - output returned by k8s delete pod operaton and/or error func (rfs *FileSystemOperation) FSUnmount(name string) (string, error) { - cmdArgs := []string{name} - out, err, status := rfs.transportClient.Delete(cmdArgs, nil) - if status == 0 { - return out, nil - } - return err, fmt.Errorf("Unable to unmount FileSystem -- : %s", err) + args := []string{"delete", "-f", name} + result, err := rfs.k8sClient.Kubectl(args...) + if err != nil { + return "", fmt.Errorf("Unable to unmount FileSystem -- : %s", err) + } + return result, nil } diff --git a/tests/framework/clients/pool.go b/tests/framework/clients/pool.go index 726de6ad6..f6f0d939d 100644 --- a/tests/framework/clients/pool.go +++ b/tests/framework/clients/pool.go @@ -23,8 +23,7 @@ import ( //PoolClient is a wrapper for rook pool operations type PoolClient struct { - transportClient contracts.ITransportClient - restClient contracts.RestAPIOperator + restClient contracts.RestAPIOperator } //CreatePoolClient creates a new pool client diff --git a/tests/framework/clients/rest_client.go b/tests/framework/clients/rest_client.go index 4bbf44c4e..a7675aa3b 100644 --- a/tests/framework/clients/rest_client.go +++ b/tests/framework/clients/rest_client.go @@ -22,9 +22,6 @@ import ( "net/http" "time" - "os" - "path/filepath" - "github.com/rook/rook/pkg/model" rclient "github.com/rook/rook/pkg/rook/client" "github.com/rook/rook/tests/framework/enums" @@ -44,10 +41,9 @@ func CreateRestAPIClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHe //Start rook_api_external server via nodePort if not it not already running. _, err := k8sHelper.GetService("rook-api-external") if err != nil { - path := filepath.Join(os.Getenv("GOPATH"), "src/github.com/rook/rook/tests/data/smoke/rook_api_external.yaml") - _, err = k8sHelper.ResourceOperation("create", path) + _, err = k8sHelper.ResourceOperation("create", getExternalAPIServiceDefinition()) if err != nil { - panic(fmt.Errorf("failed to kubectl create %v: %+v", path, err)) + panic(fmt.Errorf("failed to kubectl create -f - %v: %+v", getExternalAPIServiceDefinition(), err)) } } apiIP, err := k8sHelper.GetPodHostID("rook-api", "rook") @@ -58,7 +54,7 @@ func CreateRestAPIClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHe case platform == enums.StandAlone: endpoint = "http://localhost:8124" default: - panic(fmt.Errorf("platfrom type %s not yet supported", platform)) + panic(fmt.Errorf("platform type %s not yet supported", platform)) } httpclient := &http.Client{ @@ -75,7 +71,21 @@ func CreateRestAPIClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHe } client := rclient.NewRookNetworkRestClient(endpoint, httpclient) - return &RestAPIClient{client} + //make sure rest client is up and available + inc := 0 + for inc < utils.RetryLoop { + _, err := client.GetStatusDetails() + if err == nil { + //hack first get block image calls is intermittently failing in test env + client.GetBlockImages() + return &RestAPIClient{client} + + } + inc++ + time.Sleep(time.Second * utils.RetryInterval) + } + + panic("Cannot access rook API - Abandoning run") } //URL returns URL for rookAPI @@ -133,7 +143,7 @@ func (a *RestAPIClient) DeleteFilesystem(fsmodel model.FilesystemRequest) (strin return a.rrc.DeleteFilesystem(fsmodel) } -//GetStatusDetails retuns rook status details +//GetStatusDetails reruns rook status details func (a *RestAPIClient) GetStatusDetails() (model.StatusDetails, error) { return a.rrc.GetStatusDetails() } @@ -179,3 +189,26 @@ func (a *RestAPIClient) DeleteObjectUser(id string) error { return a.rrc.DeleteObjectUser(id) } + +func getExternalAPIServiceDefinition() string { + return `apiVersion: v1 +kind: Service +metadata: + name: rook-api-external + namespace: rook + labels: + app: rook-api + rook_cluster: rook +spec: + ports: + - name: rook-api + port: 8124 + protocol: TCP + nodePort: 30002 + selector: + app: rook-api + rook_cluster: rook + sessionAffinity: None + type: NodePort +` +} diff --git a/tests/framework/clients/test_client.go b/tests/framework/clients/test_client.go index 563d77c48..968eb7bee 100644 --- a/tests/framework/clients/test_client.go +++ b/tests/framework/clients/test_client.go @@ -22,7 +22,6 @@ import ( "github.com/rook/rook/pkg/model" "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/enums" - "github.com/rook/rook/tests/framework/transport" "github.com/rook/rook/tests/framework/utils" ) @@ -32,13 +31,12 @@ var ( //TestClient is a wrapper for test client, containing interfaces for all rook operations type TestClient struct { - platform enums.RookPlatformType - transportClient contracts.ITransportClient - blockClient contracts.BlockOperator - fsClient contracts.FileSystemOperator - objectClient contracts.ObjectOperator - poolClient contracts.PoolOperator - restClient contracts.RestAPIOperator + platform enums.RookPlatformType + blockClient contracts.BlockOperator + fsClient contracts.FileSystemOperator + objectClient contracts.ObjectOperator + poolClient contracts.PoolOperator + restClient contracts.RestAPIOperator } const ( @@ -47,7 +45,6 @@ const ( //CreateTestClient creates new instance of test client for a platform func CreateTestClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelper) (*TestClient, error) { - var transportClient contracts.ITransportClient var blockClient contracts.BlockOperator var fsClient contracts.FileSystemOperator var objectClient contracts.ObjectOperator @@ -56,24 +53,21 @@ func CreateTestClient(platform enums.RookPlatformType, k8sHelper *utils.K8sHelpe switch { case platform == enums.Kubernetes: - transportClient = transport.CreateNewk8sTransportClient() - blockClient = CreateK8BlockOperation(transportClient, rookRestClient) - fsClient = CreateK8sFileSystemOperation(transportClient, rookRestClient) + blockClient = CreateK8BlockOperation(k8sHelper, rookRestClient) + fsClient = CreateK8sFileSystemOperation(k8sHelper, rookRestClient) objectClient = CreateObjectOperation(rookRestClient) poolClient = CreatePoolClient(rookRestClient) case platform == enums.StandAlone: - transportClient = nil //TODO- Not yet implemented - blockClient = nil //TODO- Not yet implemented - fsClient = nil //TODO- Not yet implemented - objectClient = nil //TODO- Not yet implemented - poolClient = nil //TODO- Not yet implemented + blockClient = nil //TODO- Not yet implemented + fsClient = nil //TODO- Not yet implemented + objectClient = nil //TODO- Not yet implemented + poolClient = nil //TODO- Not yet implemented default: return &TestClient{}, fmt.Errorf("Unsupported Rook Platform Type") } return &TestClient{ platform, - transportClient, blockClient, fsClient, objectClient, @@ -88,26 +82,11 @@ func (c TestClient) Status() (model.StatusDetails, error) { return c.restClient.GetStatusDetails() } -//Version returns rook version installed -func (c TestClient) Version() (string, error) { - out, err, status := c.transportClient.Execute(versionCmd, nil) - if status == 0 { - return out, nil - } - return err, fmt.Errorf(unableToCheckRookStatusMsg) - -} - //Node returns list of rook nodes func (c TestClient) Node() ([]model.Node, error) { return c.restClient.GetNodes() } -//GetTransportClient returns transport client for platform in context -func (c TestClient) GetTransportClient() contracts.ITransportClient { - return c.transportClient -} - //GetBlockClient returns Block client for platform in context func (c TestClient) GetBlockClient() contracts.BlockOperator { return c.blockClient diff --git a/tests/framework/contracts/interface_docker_client.go b/tests/framework/contracts/interface_docker_client.go deleted file mode 100644 index cd665ce53..000000000 --- a/tests/framework/contracts/interface_docker_client.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package contracts - -//IDockerClient is a interface for docker commands -type IDockerClient interface { - Execute(cmd []string) (stdout string, stderr string, err error) - Stop(cmd []string) (stdout string, stderr string, err error) - Run(cmd []string) (stdout string, stderr string, err error) - ExecuteCmd(cmdArgs []string) (stdout string, stderr string, err error) -} diff --git a/tests/framework/contracts/interface_transport_client.go b/tests/framework/contracts/interface_transport_client.go deleted file mode 100644 index 39ea7d2d1..000000000 --- a/tests/framework/contracts/interface_transport_client.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package contracts - -//ITransportClient is interface for rook commands on different platforms -type ITransportClient interface { - Execute(cmd []string, optional []string) (stdout string, stderr string, exitCode int) - Create(cmd []string, optional []string) (stdout string, stderr string, exitCode int) - Delete(cmd []string, optional []string) (stdout string, stderr string, exitCode int) - ExecuteCmd(cmd []string) (stdout string, stderr string, err error) - Apply(cmd []string) (stdout string, stderr string, err error) - CreateWithStdin(stdinText string) (stdout string, stderr string, exitCode int) -} diff --git a/tests/framework/enums/docker_sub_command.go b/tests/framework/enums/docker_sub_command.go deleted file mode 100644 index ec42d1eeb..000000000 --- a/tests/framework/enums/docker_sub_command.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package enums - -import ( - "fmt" - "strings" -) - -//DockerSubCommand is enum of docker commands -type DockerSubCommand int - -const ( - //Exec docker command - Exec DockerSubCommand = iota + 1 - //Run docker command - Run - //Copy docker command - Copy - //Ps docker command - Ps - //Pull docker command - Pull - //Save docker command - Save - //Kill docker command - Kill - //Rm docker command - Rm - //Empty docker command - Empty -) - -var dockerSubCommands = [...]string{ - "exec", - "run", - "cp", - "ps", - "pull", - "save", - "kill", - "rm", - "Empty", -} - -func (subCommand DockerSubCommand) String() string { - return dockerSubCommands[subCommand-1] -} - -//GetDockerSubCommandFromString returns docker sub command -func GetDockerSubCommandFromString(name string) (DockerSubCommand, error) { - switch { - case strings.EqualFold(name, Exec.String()): - return Exec, nil - case strings.EqualFold(name, Run.String()): - return Run, nil - case strings.EqualFold(name, Copy.String()): - return Copy, nil - case strings.EqualFold(name, Ps.String()): - return Ps, nil - case strings.EqualFold(name, Pull.String()): - return Pull, nil - case strings.EqualFold(name, Save.String()): - return Save, nil - case strings.EqualFold(name, Kill.String()): - return Kill, nil - case strings.EqualFold(name, Rm.String()): - return Rm, nil - case strings.EqualFold(name, Empty.String()): - return Empty, nil - default: - return Empty, fmt.Errorf("Unsupported Docker sub-command") - } -} diff --git a/tests/framework/installer/install_data.go b/tests/framework/installer/install_data.go new file mode 100644 index 000000000..31d1162d0 --- /dev/null +++ b/tests/framework/installer/install_data.go @@ -0,0 +1,266 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import "strings" + +//InstallData wraps rook yaml definitions +type InstallData struct { +} + +//NewK8sInstallData creates new instance of InstallData struct +func NewK8sInstallData() *InstallData { + return &InstallData{} +} + +func (i *InstallData) getRookOperator(k8sVersion string) string { + + if strings.Contains(k8sVersion, "v1.5") { + return `apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: rook-operator +spec: + replicas: 1 + template: + metadata: + labels: + name: rook-operator + spec: + containers: + - name: rook-operator + image: rook/rook:master + args: ["operator"] + env: + - name: ROOKD_REPO_PREFIX + value: roo` + } + + return `kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-operator +rules: +- apiGroups: + - "" + resources: + - namespaces + - serviceaccounts + - secrets + - pods + - services + - nodes + - configmaps + - events + - persistentvolumes + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - extensions + resources: + - thirdpartyresources + - deployments + - daemonsets + - replicasets + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - delete +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-operator + namespace: default +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-operator + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-operator +subjects: +- kind: ServiceAccount + name: rook-operator + namespace: default +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rook-operator + namespace: default +spec: + replicas: 1 + template: + metadata: + labels: + app: rook-operator + spec: + serviceAccountName: rook-operator + containers: + - name: rook-operator + image: rook/rook:master + args: ["operator"] + env: + - name: ROOKD_REPO_PREFIX + value: rook` + +} + +func (i *InstallData) getRookCluster() string { + return `apiVersion: rook.io/v1alpha1 +kind: Cluster +metadata: + name: rook + namespace: rook +spec: + versionTag: master + dataDirHostPath: +# To control where various services will be scheduled by kubernetes, use the placement configuration sections below. +# The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage' and +# tolerate taints with a key of 'storage-node'. +# placement: +# all: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: role +# operator: In +# values: +# - storage-node +# tolerations: +# - key: storage-node +# operator: Exists +# api: +# nodeAffinity: +# tolerations: +# mds: +# nodeAffinity: +# tolerations: +# mon: +# nodeAffinity: +# tolerations: +# osd: +# nodeAffinity: +# tolerations: +# rgw: +# nodeAffinity: +# tolerations: + storage: # cluster level storage configuration and selection + useAllNodes: true + useAllDevices: false + deviceFilter: + metadataDevice: + location: + storeConfig: + storeType: filestore + databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger) + journalSizeMB: 1024 # this value can be removed for environments with normal sized disks (20 GB or larger) +# Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named +# nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label. +# nodes: +# - name: "172.17.4.101" +# directories: # specific directores to use for storage can be specified for each node +# - path: "/rook/storage-dir" +# - name: "172.17.4.201" +# devices: # specific devices to use for storage can be specified for each node +# - name: "sdb" +# - name: "sdc" +# storeConfig: # configuration can be specified at the node level which overrides the cluster level config +# storeType: bluestore +# - name: "172.17.4.301" +# deviceFilter: "^sd."` +} + +func (i *InstallData) getRookToolBox() string { + return `apiVersion: v1 +kind: Pod +metadata: + name: rook-tools + namespace: rook +spec: + containers: + - name: rook-tools + image: rook/toolbox:master + imagePullPolicy: IfNotPresent + args: ["sleep", "36500d"] + env: + - name: ROOKD_ADMIN_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: admin-secret + securityContext: + privileged: true + volumeMounts: + - mountPath: /dev + name: dev + - mountPath: /sys/bus + name: sysbus + - mountPath: /lib/modules + name: libmodules + volumes: + - name: dev + hostPath: + path: /dev + - name: sysbus + hostPath: + path: /sys/bus + - name: libmodules + hostPath: + path: /lib/modules` +} diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/install_helper.go index 1209cb84c..727d56cbe 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/install_helper.go @@ -18,10 +18,6 @@ package installer import ( "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" "time" "k8s.io/client-go/kubernetes" @@ -29,55 +25,37 @@ import ( "strings" "github.com/coreos/pkg/capnslog" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/objects" - "github.com/rook/rook/tests/framework/transport" "github.com/rook/rook/tests/framework/utils" ) const ( - rookOperatorFileName = "rook-operator.yaml" - rookClusterFileName = "rook-cluster.yaml" - rookToolsFileName = "rook-tools.yaml" - podSpecPath = "src/github.com/rook/rook/demo/kubernetes" - podSpecPath1_5 = "src/github.com/rook/rook/demo/kubernetes/1.5" rookOperatorCreatedTpr = "cluster.rook.io" ) var ( - logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer") + logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer") + createArgs = []string{"create", "-f", "-"} + deleteArgs = []string{"delete", "-f", "-"} ) //InstallHelper wraps installation and uninstallaion of rook on a platfom type InstallHelper struct { - transportClient contracts.ITransportClient - Env objects.EnvironmentManifest -} - -func getPodSpecPath(k8sverion string) string { - switch k8sverion { - case "v1.5": - return filepath.Join(os.Getenv("GOPATH"), podSpecPath1_5) - default: - return filepath.Join(os.Getenv("GOPATH"), podSpecPath) - } + k8shelper *utils.K8sHelper + installData *InstallData + Env objects.EnvironmentManifest } //method for create rook-operator via kubectl -func (h *InstallHelper) createK8sRookOperator(k8sHelper *utils.K8sHelper, k8sversion string) error { +func (h *InstallHelper) createK8sRookOperator(k8sHelper *utils.K8sHelper, k8sversion string) (err error) { logger.Infof("Starting Rook Operator") - raw, err := ioutil.ReadFile(path.Join(getPodSpecPath(k8sversion), rookOperatorFileName)) + rookOperator := h.installData.getRookOperator(k8sversion) + + _, err = h.k8shelper.KubectlWithStdin(rookOperator, createArgs...) if err != nil { - return err - } - rookOperator := string(raw) - - _, _, exitCode := h.transportClient.CreateWithStdin(rookOperator) - - if exitCode != 0 { - return fmt.Errorf(string("Failed to create rook-operator pod; kubectl exit code = " + string(exitCode))) + return fmt.Errorf("Failed to create rook-operator pod : %v ", err) } if !k8sHelper.IsThirdPartyResourcePresent(rookOperatorCreatedTpr) { @@ -91,19 +69,12 @@ func (h *InstallHelper) createK8sRookOperator(k8sHelper *utils.K8sHelper, k8sver func (h *InstallHelper) createK8sRookToolbox(k8sHelper *utils.K8sHelper, k8sversion string) (err error) { logger.Infof("Starting Rook toolbox") - //Create rook toolbox - raw, err := ioutil.ReadFile(path.Join(getPodSpecPath(k8sversion), rookToolsFileName)) + rookToolbox := h.installData.getRookToolBox() + + _, err = h.k8shelper.KubectlWithStdin(rookToolbox, createArgs...) if err != nil { - panic(err) - } - - rookClient := string(raw) - - _, _, exitCode := h.transportClient.CreateWithStdin(rookClient) - - if exitCode != 0 { - return fmt.Errorf(string(exitCode)) + return fmt.Errorf("Failed to create rook-toolbox pod : %v ", err) } if !k8sHelper.IsPodRunningInNamespace("rook-tools") { @@ -114,20 +85,15 @@ func (h *InstallHelper) createK8sRookToolbox(k8sHelper *utils.K8sHelper, k8svers return nil } -func (h *InstallHelper) createk8sRookCluster(k8sHelper *utils.K8sHelper, k8sversion string) error { +func (h *InstallHelper) createk8sRookCluster(k8sHelper *utils.K8sHelper, k8sversion string) (err error) { logger.Infof("Starting Rook Cluster") - raw, err := ioutil.ReadFile(path.Join(getPodSpecPath(k8sversion), rookClusterFileName)) + rookCluster := h.installData.getRookCluster() + + _, err = h.k8shelper.KubectlWithStdin(rookCluster, createArgs...) if err != nil { - return err - } - rookCluster := string(raw) - - _, _, exitCode := h.transportClient.CreateWithStdin(rookCluster) - - if exitCode != 0 { - return fmt.Errorf("Failed to create rook-cluster pod; kubectl exit code = " + string(exitCode)) + return fmt.Errorf("Failed to create rook cluster : %v ", err) } if !k8sHelper.IsServiceUpInNameSpace("rook-api") { @@ -148,22 +114,25 @@ func (h *InstallHelper) InstallRookOnK8s() (err error) { return } - logger.Infof("Installing rook on k8s %s", h.Env.K8sVersion) + k8sversion := h.k8shelper.GetK8sServerVersion() + + logger.Infof("Installing rook on k8s %s", k8sversion) //Create rook operator k8sHelp, err := utils.CreatK8sHelper() if err != nil { panic(err) } - err = h.createK8sRookOperator(k8sHelp, h.Env.K8sVersion) + err = h.createK8sRookOperator(k8sHelp, k8sversion) if err != nil { + fmt.Println(err) panic(err) } time.Sleep(10 * time.Second) ///TODO: add real check here //Create rook cluster - err = h.createk8sRookCluster(k8sHelp, h.Env.K8sVersion) + err = h.createk8sRookCluster(k8sHelp, k8sversion) if err != nil { panic(err) } @@ -186,13 +155,16 @@ func (h *InstallHelper) UninstallRookFromK8s() { if skipRookInstall { return } + k8sversion := h.k8shelper.GetK8sServerVersion() logger.Infof("Uninstalling Rook") k8sHelp, err := utils.CreatK8sHelper() if err != nil { panic(err) } - _, err = k8sHelp.ResourceOperation("delete", path.Join(getPodSpecPath(h.Env.K8sVersion), rookOperatorFileName)) + rookOperator := h.installData.getRookOperator(k8sversion) + + _, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteArgs...) if err != nil { panic(err) } @@ -247,9 +219,13 @@ func NewK8sRookhelper(clientset *kubernetes.Clientset) *InstallHelper { env.K8sVersion = version.String() } - transportClient := transport.CreateNewk8sTransportClient() + k8shelp, err := utils.CreatK8sHelper() + if err != nil { + panic("failed to get kubectl client :" + err.Error()) + } return &InstallHelper{ - transportClient: transportClient, - Env: env, + k8shelper: k8shelp, + installData: NewK8sInstallData(), + Env: env, } } diff --git a/tests/framework/objects/command_arg.go b/tests/framework/objects/command_arg.go deleted file mode 100644 index 223da9d76..000000000 --- a/tests/framework/objects/command_arg.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package objects - -// CommandArgs is a warpper for cmd args -type CommandArgs struct { - Command string - SubCommand string - CmdArgs []string - OptionalArgs []string - PipeToStdIn string - EnvironmentVariable []string -} - -//CommandOut is a wrapper for cmd out returned after executing command args -type CommandOut struct { - StdOut string - StdErr string - ExitCode int - Err error -} diff --git a/tests/framework/objects/docker_context.go b/tests/framework/objects/docker_context.go deleted file mode 100644 index 009e097aa..000000000 --- a/tests/framework/objects/docker_context.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package objects - -import "github.com/rook/rook/tests/framework/contracts" - -//DockerContext is wrapperr for docker client -type DockerContext struct { - containerID string - dockerClient contracts.IDockerClient -} - -//SetDockerContext function sets docker context instance -func SetDockerContext(dockerClient contracts.IDockerClient) DockerContext { - return DockerContext{dockerClient: dockerClient} -} - -//GetContainerID gets container Id from dockerContext instance -func (d *DockerContext) GetContainerID() string { - return d.containerID -} - -//SetContainerID sets container Id on dockerContext instance -func (d *DockerContext) SetContainerID(containerID string) string { - d.containerID = containerID - - return containerID -} - -//GetDockerClient returns docker client instance -func (d *DockerContext) GetDockerClient() contracts.IDockerClient { - return d.dockerClient -} diff --git a/tests/framework/transport/docker_transport_client.go b/tests/framework/transport/docker_transport_client.go deleted file mode 100644 index e28479997..000000000 --- a/tests/framework/transport/docker_transport_client.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package transport - -import ( - "github.com/rook/rook/tests/framework/utils" -) - -type dockerClient struct { - env []string -} - -const dockerExecutable = "docker" - -// TODO remove this file : unused. -func (k *dockerClient) Execute(cmdArgs []string) (stdout string, stderr string, err error) { - initialArgs := []string{"exec"} - cmdArgs = append(initialArgs, cmdArgs...) - - return utils.ExecuteCmdAndLogToConsole(dockerExecutable, cmdArgs, k.env) -} - -func (k *dockerClient) Run(cmdArgs []string) (stdout string, stderr string, err error) { - initialArgs := []string{"run"} - cmdArgs = append(initialArgs, cmdArgs...) - - return utils.ExecuteCmdAndLogToConsole(dockerExecutable, cmdArgs, k.env) -} - -func (k *dockerClient) Stop(cmdArgs []string) (stdout string, stderr string, err error) { - initialArgs := []string{"stop"} - cmdArgs = append(initialArgs, cmdArgs...) - - return utils.ExecuteCmdAndLogToConsole(dockerExecutable, cmdArgs, k.env) -} - -func (k *dockerClient) ExecuteCmd(cmdArgs []string) (stdout string, stderr string, err error) { - return utils.ExecuteCmdAndLogToConsole(dockerExecutable, cmdArgs, k.env) -} diff --git a/tests/framework/transport/k8s_transport_client.go b/tests/framework/transport/k8s_transport_client.go deleted file mode 100644 index cd6bce562..000000000 --- a/tests/framework/transport/k8s_transport_client.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package transport - -import ( - "github.com/rook/rook/tests/framework/objects" - "github.com/rook/rook/tests/framework/utils" -) - -//K8sTransportClient is struct for perform kubectl operations on args -type K8sTransportClient struct { -} - -//CreateNewk8sTransportClient creates new instance of transport interface for k8s -func CreateNewk8sTransportClient() *K8sTransportClient { - return &K8sTransportClient{} -} - -const ( - kubectlExecutable = "kubectl" -) - -//ExecuteCmd executes kubectl commands -func (k *K8sTransportClient) ExecuteCmd(cmd []string) (stdout string, stderr string, err error) { - return utils.ExecuteCmdAndLogToConsole(kubectlExecutable, cmd, []string{}) - -} - -//Apply executes kubectl apply commands -func (k *K8sTransportClient) Apply(cmdArgs []string) (stdout string, stderr string, err error) { - initialArgs := []string{"replace", "--force", "-f"} - cmdArgs = append(initialArgs, cmdArgs...) - return utils.ExecuteCmdAndLogToConsole("kubectl", cmdArgs, []string{}) -} - -//Execute executes kubectl commands -func (k *K8sTransportClient) Execute(cmdArgs []string, optional []string) (stdout string, stderr string, exitCode int) { - - if optional != nil { - if len(optional) == 1 { - initialArgs := []string{"exec", optional[0], "--"} - cmdArgs = append(initialArgs, cmdArgs...) - } else if len(optional) == 2 { - initialArgs := []string{"exec", "-n", optional[1], optional[0], "--"} - cmdArgs = append(initialArgs, cmdArgs...) - } else { - return "", "invalid number of optional params used", 1 - } - } else { - initialArgs := []string{"exec", "-n", "rook", "rook-tools", "--"} - cmdArgs = append(initialArgs, cmdArgs...) - } - - return utils.ExecuteCmd(kubectlExecutable, cmdArgs) -} - -//CreateWithStdin executes kubectl create commands and logs output -func (k *K8sTransportClient) CreateWithStdin(stdinText string) (stdout string, stderr string, exitCode int) { - cmdStruct := objects.CommandArgs{Command: kubectlExecutable, PipeToStdIn: stdinText, CmdArgs: []string{"create", "-f", "-"}} - - cmdOut := utils.ExecuteCommand(cmdStruct) - - return cmdOut.StdOut, cmdOut.StdErr, cmdOut.ExitCode -} - -//Create executes kubectl create commands -func (k *K8sTransportClient) Create(cmdArgs []string, optional []string) (stdout string, stderr string, exitCode int) { - initialArgs := []string{"create", "-f"} - cmdArgs = append(initialArgs, cmdArgs...) - return utils.ExecuteCmd(kubectlExecutable, cmdArgs) -} - -//Delete executes kubectl delete commands -func (k *K8sTransportClient) Delete(cmdArgs []string, optional []string) (stdout string, stderr string, exitCode int) { - - initialArgs := []string{"delete", "-f"} - cmdArgs = append(initialArgs, cmdArgs...) - return utils.ExecuteCmd(kubectlExecutable, cmdArgs) -} diff --git a/tests/framework/utils/exec_utils.go b/tests/framework/utils/exec_utils.go index c2e1adb97..df4831b2d 100644 --- a/tests/framework/utils/exec_utils.go +++ b/tests/framework/utils/exec_utils.go @@ -26,13 +26,30 @@ import ( "syscall" "github.com/coreos/pkg/capnslog" - "github.com/rook/rook/tests/framework/objects" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "testutil") -//ExecuteCommand executes a os command and returns output -func ExecuteCommand(cmdStruct objects.CommandArgs) objects.CommandOut { +// CommandArgs is a warpper for cmd args +type CommandArgs struct { + Command string + SubCommand string + CmdArgs []string + OptionalArgs []string + PipeToStdIn string + EnvironmentVariable []string +} + +//CommandOut is a wrapper for cmd out returned after executing command args +type CommandOut struct { + StdOut string + StdErr string + ExitCode int + Err error +} + +//ExecuteCommand executes a os command with stdin and returns output +func ExecuteCommand(cmdStruct CommandArgs) CommandOut { logger.Infof("Running %s %v", cmdStruct.Command, cmdStruct.CmdArgs) var outBuffer, errBuffer bytes.Buffer @@ -44,13 +61,13 @@ func ExecuteCommand(cmdStruct objects.CommandArgs) objects.CommandOut { stdOut, err := cmd.StdoutPipe() if err != nil { - return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} + return CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} } stdin, err := cmd.StdinPipe() if err != nil { - return objects.CommandOut{Err: err} + return CommandOut{Err: err} } defer stdOut.Close() @@ -66,7 +83,7 @@ func ExecuteCommand(cmdStruct objects.CommandArgs) objects.CommandOut { stdErr, err := cmd.StderrPipe() if err != nil { - return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} + return CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} } defer stdErr.Close() @@ -86,7 +103,7 @@ func ExecuteCommand(cmdStruct objects.CommandArgs) objects.CommandOut { err = cmd.Start() if err != nil { - return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} + return CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} } if cmdStruct.PipeToStdIn != "" { @@ -99,105 +116,12 @@ func ExecuteCommand(cmdStruct objects.CommandArgs) objects.CommandOut { // The program has exited with an exit code != 0 if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { log.Printf("Exit Status: %d", status.ExitStatus()) - return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: exiterr} + return CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: exiterr} } } else { - return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: nil} + return CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: nil} } } - return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} -} - -//ExecuteCmd functions executes a command returs status,stdout and stderror -func ExecuteCmd(command string, args []string) (stdout string, stderr string, exitCode int) { - logger.Infof("Running %s %s", command, args) - var outbuf, errbuf bytes.Buffer - cmd := exec.Command(command, args...) - cmd.Stdout = &outbuf - cmd.Stderr = &errbuf - - err := cmd.Run() - stdout = outbuf.String() - stderr = errbuf.String() - - if err != nil { - if exitError, ok := err.(*exec.ExitError); ok { - ws := exitError.Sys().(syscall.WaitStatus) - exitCode = ws.ExitStatus() - } else { - exitCode = 1 - if stderr == "" { - stderr = err.Error() + stdout - } - } - } else { - ws := cmd.ProcessState.Sys().(syscall.WaitStatus) - exitCode = ws.ExitStatus() - } - return -} - -//ExecuteCmdAndLogToConsole functions executes a command returs status,stdout and stderror and logs output to console -func ExecuteCmdAndLogToConsole(command string, cmdArgs []string, cmdEnv []string) (stdout string, stderr string, err error) { - logger.Infof("Running %s %v with environment %v", command, cmdArgs, cmdEnv) - var outbuf, errbuf bytes.Buffer - - cmd := exec.Command(command, cmdArgs...) - - cmd.Env = append(cmd.Env, cmdEnv...) - - stdOut, err := cmd.StdoutPipe() - if err != nil { - - return errbuf.String(), outbuf.String(), err - } - - defer stdOut.Close() - - scanner := bufio.NewScanner(stdOut) - go func() { - for scanner.Scan() { - outbuf.WriteString(scanner.Text()) - fmt.Printf("%s\n", scanner.Text()) - } - }() - - stdErr, err := cmd.StderrPipe() - if err != nil { - return errbuf.String(), outbuf.String(), err - } - - defer stdErr.Close() - - stdErrScanner := bufio.NewScanner(stdErr) - go func() { - for stdErrScanner.Scan() { - - txt := stdErrScanner.Text() - - errbuf.WriteString(txt) - fmt.Printf("%s\n", txt) - } - }() - - err = cmd.Start() - if err != nil { - - return errbuf.String(), outbuf.String(), err - } - - if err := cmd.Wait(); err != nil { - if exiterr, ok := err.(*exec.ExitError); ok { - // The program has exited with an exit code != 0 - if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - log.Printf("Exit Status: %d", status.ExitStatus()) - return errbuf.String(), outbuf.String(), exiterr - } - } else { - return errbuf.String(), outbuf.String(), nil - } - } - - return errbuf.String(), outbuf.String(), err + return CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} } diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 384201fda..db813edf1 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -20,15 +20,13 @@ import ( "encoding/json" "fmt" "html/template" - "io/ioutil" - "os" - "path/filepath" "strings" "time" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "bytes" "github.com/coreos/pkg/capnslog" "github.com/jmoiron/jsonq" "github.com/rook/rook/pkg/util/exec" @@ -40,6 +38,13 @@ type K8sHelper struct { Clientset *kubernetes.Clientset } +const ( + //RetryLoop params for tests. + RetryLoop = 30 + //RetryInterval param for test - wait time while in RetryLoop + RetryInterval = 5 +) + //CreatK8sHelper creates a instance of k8sHelper func CreatK8sHelper() (*K8sHelper, error) { executor := &exec.CommandExecutor{} @@ -58,21 +63,47 @@ func CreatK8sHelper() (*K8sHelper, error) { var k8slogger = capnslog.NewPackageLogger("github.com/rook/rook", "k8sutil") +//GetK8sServerVersion returns k8s server version under test +func (k8sh *K8sHelper) GetK8sServerVersion() string { + versionInfo, _ := k8sh.Clientset.ServerVersion() + return versionInfo.GitVersion +} + //Kubectl is wrapper for executing kubectl commands -func (k8sh *K8sHelper) Kubectl(args ...string) string { +func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) { result, err := k8sh.executor.ExecuteCommandWithOutput("", "kubectl", args...) if err != nil { - k8slogger.Errorf("Errors Encounterd while executing kubectl command : %v", err) - panic(err) + k8slogger.Errorf("Errors Encountered while executing kubectl command : %v", err) + return "", fmt.Errorf("Failed to run kubectl commands on args %v : %v", args, err) } - return result + return result, nil + +} + +//KubectlWithStdin is wrapper for executing kubectl commands in stdin +func (k8sh *K8sHelper) KubectlWithStdin(stdin string, args ...string) (string, error) { + + cmdStruct := CommandArgs{Command: "kubectl", PipeToStdIn: stdin, CmdArgs: args} + + cmdOut := ExecuteCommand(cmdStruct) + + if cmdOut.ExitCode != 0 { + k8slogger.Errorf("Errors Encountered while executing kubectl command : %v", cmdOut.Err.Error()) + return cmdOut.StdErr, fmt.Errorf("Failed to run kubectl commands on args %v : %v", args, cmdOut.StdErr) + } + if cmdOut.StdOut == "" { + return cmdOut.StdErr, nil + } + + return cmdOut.StdOut, nil + } func getKubeConfig(executor exec.Executor) (*rest.Config, error) { context, err := executor.ExecuteCommandWithOutput("", "kubectl", "config", "view", "-o", "json") if err != nil { - k8slogger.Errorf("Errors Encounterd while executing kubectl command : %v", err) + k8slogger.Errorf("Errors Encountered while executing kubectl command : %v", err) } // Parse the kubectl context to get the settings for client connections @@ -89,7 +120,7 @@ func getKubeConfig(executor exec.Executor) (*rest.Config, error) { } } if !found { - return nil, fmt.Errorf("failed to find kube context %s in %+v.", kc.Current, kc.Clusters) + return nil, fmt.Errorf("failed to find kube context %s in %+v", kc.Current, kc.Clusters) } config := &rest.Config{Host: currentCluster.Cluster.Server} config.Insecure = true @@ -105,7 +136,7 @@ func getKubeConfig(executor exec.Executor) (*rest.Config, error) { if !currentCluster.Cluster.Insecure { if !userFound { - return nil, fmt.Errorf("failed to find kube user %s in %+v.", kc.Current, kc.Users) + return nil, fmt.Errorf("failed to find kube user %s in %+v", kc.Current, kc.Users) } config.Insecure = false config.TLSClientConfig = rest.TLSClientConfig{ @@ -145,67 +176,76 @@ type kuserContext struct { func (k8sh *K8sHelper) GetMonIP(mon string) (string, error) { //kubectl -n rook get pod mon0 -o json|jq ".status.podIP"| cmdArgs := []string{"-n", "rook", "get", "pod", mon, "-o", "json"} - out, err, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { + result, err := k8sh.Kubectl(cmdArgs...) + if err == nil { data := map[string]interface{}{} - dec := json.NewDecoder(strings.NewReader(out)) + dec := json.NewDecoder(strings.NewReader(result)) dec.Decode(&data) jq := jsonq.NewQuery(data) ip, _ := jq.String("status", "podIP") return ip + ":6790", nil } - return err, fmt.Errorf("Error Getting Monitor IP") + return "", fmt.Errorf("Error Getting Monitor IP : %v", err) } //ResourceOperationFromTemplate performs a kubectl action from a template file after replacing its context -func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, poddefPath string, config map[string]string) (string, error) { +func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinition string, config map[string]string) (string, error) { - t, _ := template.ParseFiles(poddefPath) - file, _ := ioutil.TempFile(os.TempDir(), "prefix") - t.Execute(file, config) - dir, _ := filepath.Abs(file.Name()) - - cmdArgs := []string{action, "-f", dir} - stdout, stderr, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { - return stdout, nil + t := template.New("testTemplate") + t, err := t.Parse(podDefinition) + if err != nil { + return err.Error(), err } - return stdout + " : " + stderr, fmt.Errorf("Could Not create resource in k8s. status=%d, stdout=%s, stderr=%s", status, stdout, stderr) + var tpl bytes.Buffer + + if err := t.Execute(&tpl, config); err != nil { + return err.Error(), err + } + + podDef := tpl.String() + + args := []string{action, "-f", "-"} + result, err := k8sh.KubectlWithStdin(podDef, args...) + if err == nil { + return result, nil + } + logger.Errorf("Failed to execute kubectl %v %v -- %v", args, podDef, err) + return "", fmt.Errorf("Could Not create resource in args : %v %v-- %v", args, podDef, err) } -//ResourceOperation performs a kubectl action on a yaml file -func (k8sh *K8sHelper) ResourceOperation(action string, poddefPath string) (string, error) { +//ResourceOperation performs a kubectl action on a pod definition +func (k8sh *K8sHelper) ResourceOperation(action string, podDefiniton string) (string, error) { - args := []string{action, "-f", poddefPath} - stdout, stderr, status := ExecuteCmd("kubectl", args) - if status == 0 { - return stdout, nil + args := []string{action, "-f", "-"} + result, err := k8sh.KubectlWithStdin(podDefiniton, args...) + if err == nil { + return result, nil } + logger.Errorf("Failed to execute kubectl %v -- %v", args, err) + return "", fmt.Errorf("Could Not create resource in args : %v -- %v", args, err) - logger.Errorf("Failed to execute kubectl %v (%d). stdout=%s. stderr=%s", args, status, stdout, stderr) - return "FAILURE", fmt.Errorf("Failed to execute kubectl %v (%d). stdout=%s. stderr=%s", args, status, stdout, stderr) } //DeleteResource performs a kubectl delete on give args func (k8sh *K8sHelper) DeleteResource(args []string) (string, error) { - cmdArgs := append([]string{"delete"}, args...) - out, err, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { - return out, nil + args = append([]string{"delete"}, args...) + result, err := k8sh.Kubectl(args...) + if err == nil { + return result, nil } - return out + " : " + err, fmt.Errorf("Could Not delete resource in k8s") + return "", fmt.Errorf("Could Not delete resource in k8s -- %v", err) } //GetResource performs a kubectl get on give args func (k8sh *K8sHelper) GetResource(args []string) (string, error) { - cmdArgs := append([]string{"get"}, args...) - out, err, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { - return out, nil + args = append([]string{"get"}, args...) + result, err := k8sh.Kubectl(args...) + if err == nil { + return result, nil } - return out + " : " + err, fmt.Errorf("Could Not Get resource in k8s") + return "", fmt.Errorf("Could Not get resource in k8s -- %v", err) } @@ -218,11 +258,11 @@ func (k8sh *K8sHelper) GetMonitorPods() ([]string, error) { for moncount < 3 { m := fmt.Sprintf("rook-ceph-mon%d", monIdx) selector := fmt.Sprintf("mon=%s", m) - cmdArgs := []string{"-n", "rook", "get", "pod", "-l", selector} - stdout, _, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { + args := []string{"-n", "rook", "get", "pod", "-l", selector} + result, err := k8sh.Kubectl(args...) + if err == nil { // Get the first word of the second line of the output for the mon pod - lines := strings.Split(stdout, "\n") + lines := strings.Split(result, "\n") if len(lines) > 1 { name := strings.Split(lines[1], " ")[0] mons = append(mons, name) @@ -240,60 +280,20 @@ func (k8sh *K8sHelper) GetMonitorPods() ([]string, error) { return mons, nil } -//IsPodRunning retuns true if a Pod is running status or goes to Running status within 90s else returns false +//IsPodRunning returns true if a Pod is running status or goes to Running status within 90s else returns false func (k8sh *K8sHelper) IsPodRunning(name string) bool { - cmdArgs := []string{"get", "pod", name} + args := []string{"get", "pod", name, "-o", "jsonpath='{.status.phase}'"} inc := 0 - for inc < 20 { - out, _, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { - lines := strings.Split(out, "\n") - if len(lines) == 3 { - lines = lines[1 : len(lines)-1] - bktsrawdata := strings.Split(lines[0], " ") - var r []string - for _, str := range bktsrawdata { - if str != "" { - r = append(r, strings.TrimSpace(str)) - } - } - if r[2] == "Running" { - return true - } + for inc < RetryLoop { + result, err := k8sh.Kubectl(args...) + if err == nil { + if strings.Contains(result, "Running") { + return true } + logger.Infof("Pod %s status: %s", name, result) } - time.Sleep(5 * time.Second) - inc++ - } - return false -} - -//IsPodRunningInNamespace retuns true if a Pod in a namespace is running status or goes to Running -// status within 90s else returns false -func (k8sh *K8sHelper) IsPodRunningInNamespace(name string) bool { - cmdArgs := []string{"get", "pods", "-n", "rook", name} - inc := 0 - for inc < 20 { - out, _, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { - lines := strings.Split(out, "\n") - if len(lines) == 3 { - lines = lines[1 : len(lines)-1] - bktsrawdata := strings.Split(lines[0], " ") - var r []string - for _, str := range bktsrawdata { - if str != "" { - r = append(r, strings.TrimSpace(str)) - } - } - if r[2] == "Running" { - return true - } - logger.Infof("Pod %s status: %s", name, r[2]) - } - } - time.Sleep(5 * time.Second) + time.Sleep(RetryInterval * time.Second) inc++ } @@ -301,18 +301,39 @@ func (k8sh *K8sHelper) IsPodRunningInNamespace(name string) bool { return false } -//IsPodTerminated retuns true if a Pod is terminated status or goes to Terminated status +//IsPodRunningInNamespace returns true if a Pod in a namespace is running status or goes to Running +// status within 90s else returns false +func (k8sh *K8sHelper) IsPodRunningInNamespace(name string) bool { + args := []string{"get", "pods", "-n", "rook", name, "-o", "jsonpath='{.status.phase}'"} + inc := 0 + for inc < RetryLoop { + result, err := k8sh.Kubectl(args...) + if err == nil { + if strings.Contains(result, "Running") { + return true + } + logger.Infof("Pod %s status: %s", name, result) + } + time.Sleep(RetryInterval * time.Second) + inc++ + + } + logger.Infof("Giving up waiting for pod %s to be running", name) + return false +} + +//IsPodTerminated returns true if a Pod is terminated status or goes to Terminated status // within 90s else returns false\ func (k8sh *K8sHelper) IsPodTerminated(name string) bool { - cmdArgs := []string{"get", "pods", name} + args := []string{"get", "pods", name} inc := 0 for inc < 20 { - _, _, status := ExecuteCmd("kubectl", cmdArgs) - if status != 0 { + _, err := k8sh.Kubectl(args...) + if err != nil { k8slogger.Infof("Pod in default namespace terminated: " + name) return true } - time.Sleep(5 * time.Second) + time.Sleep(RetryInterval * time.Second) inc++ } @@ -320,18 +341,18 @@ func (k8sh *K8sHelper) IsPodTerminated(name string) bool { return false } -//IsPodTerminatedInNamespace retuns true if a Pod in a namespace is terminated status +//IsPodTerminatedInNamespace returns true if a Pod in a namespace is terminated status // or goes to Terminated status within 90s else returns false\ func (k8sh *K8sHelper) IsPodTerminatedInNamespace(name string) bool { - cmdArgs := []string{"get", "-n", "rook", "pods", name} + args := []string{"get", "-n", "rook", "pods", name} inc := 0 for inc < 20 { - _, _, status := ExecuteCmd("kubectl", cmdArgs) - if status != 0 { + _, err := k8sh.Kubectl(args...) + if err != nil { k8slogger.Infof("Pod in rook namespace terminated: " + name) return true } - time.Sleep(5 * time.Second) + time.Sleep(RetryInterval * time.Second) inc++ } @@ -341,15 +362,15 @@ func (k8sh *K8sHelper) IsPodTerminatedInNamespace(name string) bool { //IsServiceUp returns true if a service is up or comes up within 40s, else returns false func (k8sh *K8sHelper) IsServiceUp(name string) bool { - cmdArgs := []string{"get", "svc", name} + args := []string{"get", "svc", name} inc := 0 for inc < 20 { - _, _, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { + _, err := k8sh.Kubectl(args...) + if err == nil { k8slogger.Infof("Service in default namespace is up: " + name) return true } - time.Sleep(2 * time.Second) + time.Sleep(RetryInterval * time.Second) inc++ } @@ -359,15 +380,14 @@ func (k8sh *K8sHelper) IsServiceUp(name string) bool { //IsServiceUpInNameSpace returns true if a service in a namespace is up or comes up within 40s, else returns false func (k8sh *K8sHelper) IsServiceUpInNameSpace(name string) bool { - - cmdArgs := []string{"get", "svc", "-n", "rook", name} + args := []string{"get", "svc", "-n", "rook", name} inc := 0 for inc < 20 { - _, _, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { + _, err := k8sh.Kubectl(args...) + if err == nil { return true } - time.Sleep(5 * time.Second) + time.Sleep(RetryInterval * time.Second) inc++ } @@ -377,26 +397,25 @@ func (k8sh *K8sHelper) IsServiceUpInNameSpace(name string) bool { //GetService returns output from "kubectl get svc $NAME" command func (k8sh *K8sHelper) GetService(servicename string) (string, error) { - cmdArgs := []string{"get", "svc", "-n", "rook", servicename} - sout, serr, status := ExecuteCmd("kubectl", cmdArgs) - if status != 0 { - return serr, fmt.Errorf("Cannot find service") + args := []string{"get", "svc", "-n", "rook", servicename} + result, err := k8sh.Kubectl(args...) + if err != nil { + return "", fmt.Errorf("Cannot find service %v -- %v", servicename, err) } - return sout, nil + return result, nil } //IsThirdPartyResourcePresent returns true if Third party resource is present func (k8sh *K8sHelper) IsThirdPartyResourcePresent(tprname string) bool { - - cmdArgs := []string{"get", "thirdpartyresources", tprname} + args := []string{"get", "thirdpartyresources", tprname} inc := 0 for inc < 20 { - _, _, exitCode := ExecuteCmd("kubectl", cmdArgs) - if exitCode == 0 { + _, err := k8sh.Kubectl(args...) + if err == nil { k8slogger.Infof("Found the thirdparty resource: " + tprname) return true } - time.Sleep(5 * time.Second) + time.Sleep(RetryInterval * time.Second) inc++ } @@ -405,15 +424,15 @@ func (k8sh *K8sHelper) IsThirdPartyResourcePresent(tprname string) bool { //GetPodDetails returns details about a pod func (k8sh *K8sHelper) GetPodDetails(podNamePattern string, namespace string) (string, error) { - cmdArgs := []string{"get", "pods", "-l", "app=" + podNamePattern, "-o", "wide", "--no-headers=true", "-o", "name"} + args := []string{"get", "pods", "-l", "app=" + podNamePattern, "-o", "wide", "--no-headers=true", "-o", "name"} if namespace != "" { - cmdArgs = append(cmdArgs, []string{"-n", namespace}...) + args = append(args, []string{"-n", namespace}...) } - sout, serr, status := ExecuteCmd("kubectl", cmdArgs) - if status != 0 || strings.Contains(sout, "No resources found") { - return serr, fmt.Errorf("Cannot find pod in with name like %s in namespace : %s", podNamePattern, namespace) + result, err := k8sh.Kubectl(args...) + if err != nil || strings.Contains(result, "No resources found") { + return "", fmt.Errorf("Cannot find pod in with name like %s in namespace : %s -- %v", podNamePattern, namespace, err) } - return strings.TrimSpace(sout), nil + return strings.TrimSpace(result), nil } //GetPodHostID returns HostIP address of a pod @@ -429,59 +448,59 @@ func (k8sh *K8sHelper) GetPodHostID(podNamePattern string, namespace string) (st } //get host Ip of the pod - cmdArgs := []string{"get", podNames[0], "-o", "jsonpath='{.status.hostIP}'"} + args := []string{"get", podNames[0], "-o", "jsonpath='{.status.hostIP}'"} if namespace != "" { - cmdArgs = append(cmdArgs, []string{"-n", namespace}...) + args = append(args, []string{"-n", namespace}...) } - sout, serr, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { - hostIP := strings.Replace(sout, "'", "", -1) + result, err := k8sh.Kubectl(args...) + if err == nil { + hostIP := strings.Replace(result, "'", "", -1) return strings.TrimSpace(hostIP), nil } - return serr, fmt.Errorf("Error Getting Monitor IP") + return "", fmt.Errorf("Error Getting Monitor IP -- %v", err) } //IsStorageClassPresent returns true if storageClass is present, if not false func (k8sh *K8sHelper) IsStorageClassPresent(name string) (bool, error) { - cmdArgs := []string{"get", "storageclass", "-o", "jsonpath='{.items[*].metadata.name}'"} - sout, serr, _ := ExecuteCmd("kubectl", cmdArgs) - if strings.Contains(sout, name) { + args := []string{"get", "storageclass", "-o", "jsonpath='{.items[*].metadata.name}'"} + result, err := k8sh.Kubectl(args...) + if strings.Contains(result, name) { return true, nil } - return false, fmt.Errorf("Storageclass %s not found, err ->%s", name, serr) + return false, fmt.Errorf("Storageclass %s not found, err ->%v", name, err) } //GetPVCStatus returns status of PVC func (k8sh *K8sHelper) GetPVCStatus(name string) (string, error) { - cmdArgs := []string{"get", "pvc", "-o", "jsonpath='{.items[*].metadata.name}'"} - sout, serr, _ := ExecuteCmd("kubectl", cmdArgs) - if strings.Contains(sout, name) { - cmdArgs := []string{"get", "pvc", name, "-o", "jsonpath='{.status.phase}'"} - res, _, _ := ExecuteCmd("kubectl", cmdArgs) + args := []string{"get", "pvc", "-o", "jsonpath='{.items[*].metadata.name}'"} + result, err := k8sh.Kubectl(args...) + if strings.Contains(result, name) { + args := []string{"get", "pvc", name, "-o", "jsonpath='{.status.phase}'"} + res, _ := k8sh.Kubectl(args...) return res, nil } - return "PVC NOT FOUND", fmt.Errorf("PVC %s not found,err->%s", name, serr) + return "PVC NOT FOUND", fmt.Errorf("PVC %s not found,err->%v", name, err) } //IsPodInExpectedState waits for 90s for a pod to be an expected state //If the pod is in expected state within 90s true is returned, if not false func (k8sh *K8sHelper) IsPodInExpectedState(podNamePattern string, namespace string, state string) bool { - cmdArgs := []string{"get", "pods", "-l", "app=" + podNamePattern, "-o", "jsonpath={.items[0].status.phase}", "--no-headers=true"} + args := []string{"get", "pods", "-l", "app=" + podNamePattern, "-o", "jsonpath={.items[0].status.phase}", "--no-headers=true"} if namespace != "" { - cmdArgs = append(cmdArgs, []string{"-n", namespace}...) + args = append(args, []string{"-n", namespace}...) } inc := 0 - for inc < 30 { - res, _, status := ExecuteCmd("kubectl", cmdArgs) - if status == 0 { - if res == state { + for inc < RetryLoop { + result, err := k8sh.Kubectl(args...) + if err == nil { + if result == state { return true } } inc++ - time.Sleep(3 * time.Second) + time.Sleep(RetryInterval * time.Second) } return false @@ -492,14 +511,14 @@ func (k8sh *K8sHelper) IsPodInExpectedState(podNamePattern string, namespace str func (k8sh *K8sHelper) WaitUntilPodInNamespaceIsDeleted(podNamePattern string, namespace string) bool { args := []string{"-n", namespace, "pods", "-l", "app=" + podNamePattern} inc := 0 - for inc < 30 { + for inc < RetryLoop { out, _ := k8sh.GetResource(args) if !strings.Contains(out, podNamePattern) { return true } inc++ - time.Sleep(3 * time.Second) + time.Sleep(RetryInterval * time.Second) } panic(fmt.Errorf("Rook not uninstalled")) } @@ -509,14 +528,14 @@ func (k8sh *K8sHelper) WaitUntilPodInNamespaceIsDeleted(podNamePattern string, n func (k8sh *K8sHelper) WaitUntilPodIsDeleted(podNamePattern string) bool { args := []string{"pods", "-l", "app=" + podNamePattern} inc := 0 - for inc < 30 { + for inc < RetryLoop { out, _ := k8sh.GetResource(args) if !strings.Contains(out, podNamePattern) { return true } inc++ - time.Sleep(3 * time.Second) + time.Sleep(RetryInterval * time.Second) } return false } @@ -526,7 +545,7 @@ func (k8sh *K8sHelper) WaitUntilPodIsDeleted(podNamePattern string) bool { func (k8sh *K8sHelper) WaitUntilPVCIsBound(pvcname string) bool { inc := 0 - for inc < 30 { + for inc < RetryLoop { out, err := k8sh.GetPVCStatus(pvcname) if strings.Contains(out, "Bound") { return true @@ -534,7 +553,19 @@ func (k8sh *K8sHelper) WaitUntilPVCIsBound(pvcname string) bool { logger.Infof("waiting for PVC to be bound. current=%s. err=%+v", out, err) inc++ - time.Sleep(3 * time.Second) + time.Sleep(RetryInterval * time.Second) } return false } + +//GetRGWServiceURL returns URL of ceph RGW service in the cluster +func (k8sh *K8sHelper) GetRGWServiceURL() (string, error) { + hostip, err := k8sh.GetPodHostID("rook-ceph-rgw", "rook") + if err != nil { + panic(fmt.Errorf("RGW pods not found/object store possibly not started")) + } + + //TODO - Get nodePort stop hardcoding + endpoint := hostip + ":30001" + return endpoint, err +} diff --git a/tests/block/create_block_test.go b/tests/integration/block_createBlock_api_test.go similarity index 98% rename from tests/block/create_block_test.go rename to tests/integration/block_createBlock_api_test.go index 8e0fa928f..dc1e2ca4f 100644 --- a/tests/block/create_block_test.go +++ b/tests/integration/block_createBlock_api_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package block +package integration import ( "strings" @@ -36,7 +36,7 @@ var ( blockImageName = "testImage" ) -func TestBlockCreate(t *testing.T) { +func TestBlockCreateAPI(t *testing.T) { suite.Run(t, new(BlockImageCreateSuite)) } diff --git a/tests/block/k8s/create_block_test.go b/tests/integration/block_createBlock_k8s_test.go similarity index 57% rename from tests/block/k8s/create_block_test.go rename to tests/integration/block_createBlock_k8s_test.go index c333196ce..d59e58007 100644 --- a/tests/block/k8s/create_block_test.go +++ b/tests/integration/block_createBlock_k8s_test.go @@ -14,13 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -package k8s +package integration import ( - "bytes" - "html/template" - "os" - "path/filepath" "strings" "testing" "time" @@ -29,7 +25,6 @@ import ( "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/enums" "github.com/rook/rook/tests/framework/installer" - "github.com/rook/rook/tests/framework/objects" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -46,13 +41,13 @@ func TestK8sBlockCreate(t *testing.T) { type K8sBlockImageCreateSuite struct { suite.Suite - testClient *clients.TestClient - bc contracts.BlockOperator - kh *utils.K8sHelper - initBlockCount int - pvPath string - storageclassPath string - installer *installer.InstallHelper + testClient *clients.TestClient + bc contracts.BlockOperator + kh *utils.K8sHelper + initBlockCount int + pvcDef string + storageclassDef string + installer *installer.InstallHelper } func (s *K8sBlockImageCreateSuite) SetupSuite() { @@ -73,8 +68,41 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() { initialBlocks, err := s.bc.BlockList() require.Nil(s.T(), err) s.initBlockCount = len(initialBlocks) - s.pvPath = filepath.Join(os.Getenv("GOPATH"), "src/github.com/rook/rook/tests/data/block/pvc.tmpl") - s.storageclassPath = filepath.Join(os.Getenv("GOPATH"), "src/github.com/rook/rook/tests/data/block/storageclass_pool.tmpl") + + s.pvcDef = `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{.claimName}} + annotations: + volume.beta.kubernetes.io/storage-class: rook-block +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1M` + + s.storageclassDef = `apiVersion: rook.io/v1alpha1 +kind: Pool +metadata: + name: {{.poolName}} + namespace: rook +spec: + replication: + size: 1 + # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. + # Make sure you have enough OSDs to support the replica count or erasure code chunks. + #erasureCode: + # codingChunks: 2 + # dataChunks: 2 +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-block +provisioner: rook.io/block +parameters: + pool: {{.poolName}}` } // Test case when persistentvolumeclaim is created for a storage class that doesn't exist @@ -82,10 +110,9 @@ func (s *K8sBlockImageCreateSuite) TestCreatePVCWhenNoStorageClassExists() { s.T().Log("Test creating PVC(block images) when storage class is not created") //Create PVC - sout1, serr1, status1 := s.pvcOperation(claimName, "create") - require.Contains(s.T(), sout1, "persistentvolumeclaim \"test-claim1\" created", "Make sure pvc is created") - require.Empty(s.T(), serr1, "Make sure there are no errors") - require.Equal(s.T(), 0, status1) + result, err := s.pvcOperation(claimName, "create") + require.Contains(s.T(), result, "persistentvolumeclaim \"test-claim1\" created", "Make sure pvc is created") + require.NoError(s.T(), err) //check status of PVC pvcStatus, err := s.kh.GetPVCStatus(claimName) @@ -103,11 +130,10 @@ func (s *K8sBlockImageCreateSuite) TestCreatePVCWhenStorageClassExists() { s.T().Log("Test creating PVC(block images) when storage class is created") //create pool and storageclass - sout, serr, status := s.storageClassOperation("test-pool-1", "create") - require.Contains(s.T(), sout, "pool \"test-pool-1\" created", "Make sure test pool is created") - require.Contains(s.T(), sout, "storageclass \"rook-block\" created", "Make sure storageclass is created") - require.Empty(s.T(), serr, "Make sure there are no errors") - require.Equal(s.T(), 0, status) + result1, err1 := s.storageClassOperation("test-pool-1", "create") + require.Contains(s.T(), result1, "pool \"test-pool-1\" created", "Make sure test pool is created") + require.Contains(s.T(), result1, "storageclass \"rook-block\" created", "Make sure storageclass is created") + require.NoError(s.T(), err1) //make sure storageclass is created present, err := s.kh.IsStorageClassPresent("rook-block") @@ -115,10 +141,9 @@ func (s *K8sBlockImageCreateSuite) TestCreatePVCWhenStorageClassExists() { require.True(s.T(), present, "Make sure storageclass is present") //create pvc - sout1, serr1, status1 := s.pvcOperation(claimName, "create") - require.Contains(s.T(), sout1, "persistentvolumeclaim \"test-claim1\" created", "Make sure pvc is created") - require.Empty(s.T(), serr1, "Make sure there are no errors") - require.Equal(s.T(), 0, status1) + result2, err2 := s.pvcOperation(claimName, "create") + require.Contains(s.T(), result2, "persistentvolumeclaim \"test-claim1\" created", "Make sure pvc is created") + require.NoError(s.T(), err2) //check status of PVC require.True(s.T(), s.isPVCBound(claimName)) @@ -132,14 +157,12 @@ func (s *K8sBlockImageCreateSuite) TestCreatePVCWhenStorageClassExists() { // Test case when persistentvolumeclaim is created for a valid storage class twice func (s *K8sBlockImageCreateSuite) TestCreateSamePVCTwice() { - s.T().Log("Test PVC(create block images) when storage class is not created") - s.T().Log("Test creating PVC(block images) when storage class is created") + s.T().Log("Test creating PVC(create block images) twice") //create pool and storageclass - sout, serr, status := s.storageClassOperation("test-pool-1", "create") - require.Contains(s.T(), sout, "pool \"test-pool-1\" created", "Make sure test pool is created") - require.Contains(s.T(), sout, "storageclass \"rook-block\" created", "Make sure storageclass is created") - require.Empty(s.T(), serr, "Make sure there are no errors") - require.Equal(s.T(), 0, status) + result1, err1 := s.storageClassOperation("test-pool-1", "create") + require.Contains(s.T(), result1, "pool \"test-pool-1\" created", "Make sure test pool is created") + require.Contains(s.T(), result1, "storageclass \"rook-block\" created", "Make sure storageclass is created") + require.NoError(s.T(), err1) //make sure storageclass is created present, err := s.kh.IsStorageClassPresent("rook-block") @@ -147,10 +170,9 @@ func (s *K8sBlockImageCreateSuite) TestCreateSamePVCTwice() { require.True(s.T(), present, "Make sure storageclass is present") //create pvc - sout1, serr1, status1 := s.pvcOperation(claimName, "create") - require.Contains(s.T(), sout1, "persistentvolumeclaim \"test-claim1\" created", "Make sure pvc is created") - require.Empty(s.T(), serr1, "Make sure there are no errors") - require.Equal(s.T(), 0, status1) + result2, err2 := s.pvcOperation(claimName, "create") + require.Contains(s.T(), result2, "persistentvolumeclaim \"test-claim1\" created", "Make sure pvc is created") + require.NoError(s.T(), err2) //check status of PVC require.True(s.T(), s.isPVCBound(claimName)) @@ -159,10 +181,9 @@ func (s *K8sBlockImageCreateSuite) TestCreateSamePVCTwice() { require.Equal(s.T(), s.initBlockCount+1, len(b1), "Make sure new block image is created") //Create same pvc again - sout2, serr2, status2 := s.pvcOperation(claimName, "create") - require.Empty(s.T(), sout2, "Make sure there are no output") - require.Contains(s.T(), serr2, "persistentvolumeclaims \"test-claim1\" already exists", "Check Error") - require.Equal(s.T(), 0, status2) + result3, err3 := s.pvcOperation(claimName, "create") + require.Contains(s.T(), result3, "persistentvolumeclaims \"test-claim1\" already exists", "make sure PVC is not created again") + require.NoError(s.T(), err3) //check status of PVC require.True(s.T(), s.isPVCBound(claimName)) @@ -180,51 +201,36 @@ func (s *K8sBlockImageCreateSuite) TearDownTest() { } -func (s *K8sBlockImageCreateSuite) storageClassOperation(poolName string, action string) (string, string, int) { - - t, _ := template.ParseFiles(s.storageclassPath) - - var tpl bytes.Buffer +func (s *K8sBlockImageCreateSuite) storageClassOperation(poolName string, action string) (string, error) { config := map[string]string{ "poolName": poolName, } - t.Execute(&tpl, config) + result, err := s.kh.ResourceOperationFromTemplate(action, s.storageclassDef, config) - cmdStruct := objects.CommandArgs{Command: "kubectl", PipeToStdIn: tpl.String(), CmdArgs: []string{action, "-f", "-"}} - - cmdOut := utils.ExecuteCommand(cmdStruct) - - return cmdOut.StdOut, cmdOut.StdErr, cmdOut.ExitCode + return result, err } -func (s *K8sBlockImageCreateSuite) pvcOperation(claimName string, action string) (string, string, int) { - t, _ := template.ParseFiles(s.pvPath) - - var tpl bytes.Buffer +func (s *K8sBlockImageCreateSuite) pvcOperation(claimName string, action string) (string, error) { config := map[string]string{ "claimName": claimName, } - t.Execute(&tpl, config) + result, err := s.kh.ResourceOperationFromTemplate(action, s.pvcDef, config) - cmdStruct := objects.CommandArgs{Command: "kubectl", PipeToStdIn: tpl.String(), CmdArgs: []string{action, "-f", "-"}} - - cmdOut := utils.ExecuteCommand(cmdStruct) - - return cmdOut.StdOut, cmdOut.StdErr, cmdOut.ExitCode + return result, err } func (s *K8sBlockImageCreateSuite) isPVCBound(name string) bool { inc := 0 - for inc < 10 { + for inc < utils.RetryLoop { status, _ := s.kh.GetPVCStatus(claimName) if strings.TrimRight(status, "\n") == "'Bound'" { return true } - time.Sleep(time.Second * 1) + time.Sleep(time.Second * utils.RetryInterval) inc++ } diff --git a/tests/block/k8s/integration/k8sBlockStorageIntegration_test.go b/tests/integration/block_e2e_k8s_test.go similarity index 64% rename from tests/block/k8s/integration/k8sBlockStorageIntegration_test.go rename to tests/integration/block_e2e_k8s_test.go index fd35a7844..cb4a6ef21 100644 --- a/tests/block/k8s/integration/k8sBlockStorageIntegration_test.go +++ b/tests/integration/block_e2e_k8s_test.go @@ -17,17 +17,12 @@ limitations under the License. package integration import ( - "bytes" - "html/template" - "os" - "path/filepath" "testing" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/enums" "github.com/rook/rook/tests/framework/installer" - "github.com/rook/rook/tests/framework/objects" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -74,12 +69,101 @@ func (s *K8sBlockEnd2EndIntegrationSuite) SetupSuite() { initialBlocks, err := s.bc.BlockList() require.Nil(s.T(), err) s.initBlockCount = len(initialBlocks) - s.storageclassPath = filepath.Join(os.Getenv("GOPATH"), "src/github.com/rook/rook/tests/data/block/storageclass_pool.tmpl") - s.mysqlappPath = filepath.Join(os.Getenv("GOPATH"), "src/github.com/rook/rook/tests/data/integration/mysqlapp.yaml") + + s.storageclassPath = `apiVersion: rook.io/v1alpha1 +kind: Pool +metadata: + name: {{.poolName}} + namespace: rook +spec: + replication: + size: 1 + # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. + # Make sure you have enough OSDs to support the replica count or erasure code chunks. + #erasureCode: + # codingChunks: 2 + # dataChunks: 2 +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-block +provisioner: rook.io/block +parameters: + pool: {{.poolName}}` + + s.mysqlappPath = `apiVersion: v1 +kind: Service +metadata: + name: mysql-app + labels: + app: mysqldb +spec: + ports: + - port: 3306 + targetPort: 3306 + protocol: TCP + nodePort: 30003 + selector: + app: mysqldb + tier: mysql + type: NodePort +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-pv-claim + labels: + app: mysqldb +spec: + storageClassName: rook-block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: mysql-app + labels: + app: mysqldb +spec: + strategy: + type: Recreate + template: + metadata: + labels: + app: mysqldb + tier: mysql + spec: + containers: + - image: mysql:5.6 + name: mysql + env: + - name: "MYSQL_USER" + value: "mysql" + - name: "MYSQL_PASSWORD" + value: "mysql" + - name: "MYSQL_DATABASE" + value: "sample" + - name: "MYSQL_ROOT_PASSWORD" + value: "root" + ports: + - containerPort: 3306 + name: mysql + volumeMounts: + - name: mysql-persistent-storage + mountPath: /var/lib/mysql + volumes: + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pv-claim` //create storage class - _, _, scs := s.storageClassOperation("mysql-pool", "create") - require.Equal(s.T(), 0, scs) + _, err = s.storageClassOperation("mysql-pool", "create") + require.NoError(s.T(), err) //make sure storageclass is created present, err := s.kh.IsStorageClassPresent("rook-block") @@ -122,23 +206,14 @@ func (s *K8sBlockEnd2EndIntegrationSuite) TestBlockE2EIntegrationWithMySqlDataba } -func (s *K8sBlockEnd2EndIntegrationSuite) storageClassOperation(poolName string, action string) (string, string, int) { - - t, _ := template.ParseFiles(s.storageclassPath) - - var tpl bytes.Buffer +func (s *K8sBlockEnd2EndIntegrationSuite) storageClassOperation(poolName string, action string) (string, error) { config := map[string]string{ "poolName": poolName, } - t.Execute(&tpl, config) - - cmdStruct := objects.CommandArgs{Command: "kubectl", PipeToStdIn: tpl.String(), CmdArgs: []string{action, "-f", "-"}} - - cmdOut := utils.ExecuteCommand(cmdStruct) - - return cmdOut.StdOut, cmdOut.StdErr, cmdOut.ExitCode + result, err := s.kh.ResourceOperationFromTemplate(action, s.storageclassPath, config) + return result, err } func (s *K8sBlockEnd2EndIntegrationSuite) TearDownTest() { diff --git a/tests/block/k8s/longhaul/basicBlockonghaul_test.go b/tests/longhaul/block_on_k8s_test.go similarity index 59% rename from tests/block/k8s/longhaul/basicBlockonghaul_test.go rename to tests/longhaul/block_on_k8s_test.go index 7b675d165..cd97e8615 100644 --- a/tests/block/k8s/longhaul/basicBlockonghaul_test.go +++ b/tests/longhaul/block_on_k8s_test.go @@ -1,8 +1,6 @@ package longhaul import ( - "bytes" - "html/template" "sync" "testing" @@ -10,7 +8,6 @@ import ( "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/enums" "github.com/rook/rook/tests/framework/installer" - "github.com/rook/rook/tests/framework/objects" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -58,17 +55,105 @@ func (s *K8sBlockLongHaulSuite) SetupSuite() { initialBlocks, err := s.bc.BlockList() require.Nil(s.T(), err) s.initBlockCount = len(initialBlocks) - s.storageClassPath = "../../../../data/block/storageclass_pool.tmpl" - s.mysqlAppPath = "../../../../data/integration/mysqlapp.yaml" + s.storageClassPath = `apiVersion: rook.io/v1alpha1 +kind: Pool +metadata: + name: {{.poolName}} + namespace: rook +spec: + replication: + size: 1 + # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. + # Make sure you have enough OSDs to support the replica count or erasure code chunks. + #erasureCode: + # codingChunks: 2 + # dataChunks: 2 +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-block +provisioner: rook.io/block +parameters: + pool: {{.poolName}}` + + s.mysqlAppPath = `apiVersion: v1 +kind: Service +metadata: + name: mysql-app + labels: + app: mysqldb +spec: + ports: + - port: 3306 + targetPort: 3306 + protocol: TCP + nodePort: 30003 + selector: + app: mysqldb + tier: mysql + type: NodePort +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-pv-claim + labels: + app: mysqldb +spec: + storageClassName: rook-block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: mysql-app + labels: + app: mysqldb +spec: + strategy: + type: Recreate + template: + metadata: + labels: + app: mysqldb + tier: mysql + spec: + containers: + - image: mysql:5.6 + name: mysql + env: + - name: "MYSQL_USER" + value: "mysql" + - name: "MYSQL_PASSWORD" + value: "mysql" + - name: "MYSQL_DATABASE" + value: "sample" + - name: "MYSQL_ROOT_PASSWORD" + value: "root" + ports: + - containerPort: 3306 + name: mysql + volumeMounts: + - name: mysql-persistent-storage + mountPath: /var/lib/mysql + volumes: + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pv-claim` //create storage class if scp, _ := s.kh.IsStorageClassPresent("rook-block"); !scp { - _, _, scs := s.storageClassOperation("mysql-pool", "create") - require.Equal(s.T(), 0, scs) + _, err = s.storageClassOperation("mysql-pool", "create") + require.NoError(s.T(), err) //make sure storageclass is created present, err := s.kh.IsStorageClassPresent("rook-block") - require.Nil(s.T(), err) + require.NoError(s.T(), err) require.True(s.T(), present, "Make sure storageclass is present") } //create mysql pod @@ -120,30 +205,23 @@ func (s *K8sBlockLongHaulSuite) dbOperation(i int) { } func (s *K8sBlockLongHaulSuite) TearDownSuite() { s.db.CloseConnection() + s.kh.ResourceOperation("delete", s.mysqlAppPath) + s.storageClassOperation("mysql-pool", "delete") + s.installer.UninstallRookFromK8s() s.testClient = nil s.bc = nil s.kh = nil s.db = nil s = nil - s.installer.UninstallRookFromK8s() - } -func (s *K8sBlockLongHaulSuite) storageClassOperation(poolName string, action string) (string, string, int) { - - t, _ := template.ParseFiles(s.storageClassPath) - - var tpl bytes.Buffer +func (s *K8sBlockLongHaulSuite) storageClassOperation(poolName string, action string) (string, error) { config := map[string]string{ "poolName": poolName, } - t.Execute(&tpl, config) + result, err := s.kh.ResourceOperationFromTemplate(action, s.storageClassPath, config) - cmdStruct := objects.CommandArgs{Command: "kubectl", PipeToStdIn: tpl.String(), CmdArgs: []string{action, "-f", "-"}} - - cmdOut := utils.ExecuteCommand(cmdStruct) - - return cmdOut.StdOut, cmdOut.StdErr, cmdOut.ExitCode + return result, err } diff --git a/tests/scripts/minikube.sh b/tests/scripts/minikube.sh index 602f9ee1f..5d679063f 100755 --- a/tests/scripts/minikube.sh +++ b/tests/scripts/minikube.sh @@ -38,13 +38,16 @@ wait_for_ssh() { } copy_image_to_cluster() { - local final_image=$1 + local build_image=$1 + local final_image=$2 - echo copying ${final_image} to minikube + echo copying ${build_image} to minikube mkdir -p ${WORK_DIR}/tests docker save -o ${tarfile} ${build_image} minikube_scp ${tarfile} /home/docker minikube_ssh "docker load -i /home/docker/${tarname}" + minikube_ssh "docker tag ${build_image} ${final_image}" + } # configure dind-cluster @@ -54,8 +57,8 @@ case "${1:-}" in up) minikube start --memory=2600 --iso-url=https://s3-us-west-2.amazonaws.com/minikube-cephfs/minikube.iso --kubernetes-version ${KUBE_VERSION} wait_for_ssh - copy_image_to_cluster rook/rook:master - copy_image_to_cluster rook/toolbox:master + copy_image_to_cluster ${BUILD_REGISTRY}/rook-amd64 rook/rook:master + copy_image_to_cluster ${BUILD_REGISTRY}/toolbox-amd64 rook/toolbox:master echo setting up rbd minikube_scp ${scriptdir}/minikube-rbd /home/docker/minikube-rbd diff --git a/tests/smoke/block_test.go b/tests/smoke/block_test.go index 6e13164e8..914c62288 100644 --- a/tests/smoke/block_test.go +++ b/tests/smoke/block_test.go @@ -19,7 +19,15 @@ package smoke import ( "time" + "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/require" + "strings" +) + +var ( + pvcName = "block-pv-claim" + blockMountPath = "/tmp/rook1" + blockPodName = "block-test" ) // Smoke Test for Block Storage - Test check the following operations on Block Storage in order @@ -33,60 +41,169 @@ func (suite *SmokeSuite) TestBlockStorage_SmokeTest() { initBlockImages, _ := rbc.BlockList() logger.Infof("step 1: Create block storage") - cbErr := suite.helper.CreateBlockStorage() + cbErr := suite.blockStorageOperation("create") require.Nil(suite.T(), cbErr) - require.True(suite.T(), retryBlockImageCountCheck(suite.helper, len(initBlockImages), 1), "Make sure a new block is created") + require.True(suite.T(), suite.retryBlockImageCountCheck(len(initBlockImages), 1), "Make sure a new block is created") logger.Infof("Block Storage created successfully") - require.True(suite.T(), suite.helper.WaitUntilPVCIsBound(), "Make sure PVC is Bound") + require.True(suite.T(), suite.k8sh.WaitUntilPVCIsBound(pvcName), "Make sure PVC is Bound") logger.Infof("step 2: Mount block storage") - mtErr := suite.helper.MountBlockStorage() + _, mtErr := rbc.BlockMap(getBlockPodDefintion(), blockMountPath) require.Nil(suite.T(), mtErr) + require.True(suite.T(), suite.k8sh.IsPodRunning(blockPodName), "make sure block-test pod is in running state") logger.Infof("Block Storage Mounted successfully") logger.Infof("step 3: Write to block storage") - wtErr := suite.helper.WriteToBlockStorage("Test Data", "testFile1") + _, wtErr := rbc.BlockWrite(blockPodName, blockMountPath, "Smoke Test Data form Block storage", "bsFile1", "") require.Nil(suite.T(), wtErr) logger.Infof("Write to Block storage successfully") logger.Infof("step 4: Read from block storage") - read, rErr := suite.helper.ReadFromBlockStorage("testFile1") + read, rErr := rbc.BlockRead(blockPodName, blockMountPath, "bsFile1", "") require.Nil(suite.T(), rErr) - require.Contains(suite.T(), read, "Test Data", "make sure content of the files is unchanged") + require.Contains(suite.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged") logger.Infof("Read from Block storage successfully") logger.Infof("step 5: Unmount block storage") - unmtErr := suite.helper.UnMountBlockStorage() + _, unmtErr := rbc.BlockUnmap(getBlockPodDefintion(), blockMountPath) require.Nil(suite.T(), unmtErr) + require.True(suite.T(), suite.k8sh.IsPodTerminated(blockPodName), "make sure block-test pod is terminated") logger.Infof("Block Storage unmounted successfully") logger.Infof("step 6: Deleting block storage") - dbErr := suite.helper.DeleteBlockStorage() + dbErr := suite.blockStorageOperation("delete") require.Nil(suite.T(), dbErr) - require.True(suite.T(), retryBlockImageCountCheck(suite.helper, len(initBlockImages), 0), "Make sure a block is deleted") + require.True(suite.T(), suite.retryBlockImageCountCheck(len(initBlockImages), 0), "Make sure a block is deleted") logger.Infof("Block Storage deleted successfully") } func (suite *SmokeSuite) blockTestDataCleanUp() { logger.Infof("Cleaning up block storage") - suite.helper.UnMountBlockStorage() - suite.helper.DeleteBlockStorage() - suite.helper.CleanUpDymanicBlockStorage() + suite.helper.GetBlockClient().BlockUnmap(getBlockPodDefintion(), blockMountPath) + suite.blockStorageOperation("delete") + suite.cleanUpDymanicBlockStorage() +} + +func (suite *SmokeSuite) blockStorageOperation(action string) error { + poolStorageClassDef := suite.getBlockPoolAndStorageClassDefinition() + pvcDef := getPvcDefiniton() + suite.k8sh.ResourceOperation(action, poolStorageClassDef) + // see https://github.com/rook/rook/issues/767 + time.Sleep(10 * time.Second) + _, err := suite.k8sh.ResourceOperation(action, pvcDef) + + return err } // periodically checking if block image count has changed to expected value // When creating pvc in k8s platform, it may take some time for the block Image to be bounded -func retryBlockImageCountCheck(h *TestHelper, imageCount, expectedChange int) bool { +func (suite *SmokeSuite) retryBlockImageCountCheck(imageCount, expectedChange int) bool { inc := 0 - for inc < 30 { + for inc < utils.RetryLoop { logger.Infof("Getting list of blocks (expecting %d)", (imageCount + expectedChange)) - blockImages, _ := h.GetBlockClient().BlockList() + blockImages, _ := suite.helper.GetBlockClient().BlockList() if imageCount+expectedChange == len(blockImages) { return true } - time.Sleep(time.Second * 5) + time.Sleep(time.Second * utils.RetryInterval) inc++ } return false } + +//CleanUpDymanicBlockStorage is helper method to clean up bock storage created by tests +func (suite *SmokeSuite) cleanUpDymanicBlockStorage() { + // Delete storage pool, storage class and pvc + blockImagesList, _ := suite.helper.GetBlockClient().BlockList() + for _, blockImage := range blockImagesList { + suite.helper.GetRestAPIClient().DeleteBlockImage(blockImage) + } + +} + +func (suite *SmokeSuite) getBlockPoolAndStorageClassDefinition() string { + k8sVersion := suite.k8sh.GetK8sServerVersion() + if strings.Contains(k8sVersion, "v1.5") { + return `apiVersion: rook.io/v1alpha1 +kind: Pool +metadata: + name: replicapool + namespace: rook +spec: + replication: + size: 1 + # For an erasure-coded pool, comment out the replication size above and uncomment the following settings. + # Make sure you have enough OSDs to support the replica size or erasure code chunks. + #erasureCode: + # codingChunks: 2 + # dataChunks: 2 +--- +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: rook-block +provisioner: rook.io/block +parameters: + pool: replicapool` + + } + return `apiVersion: rook.io/v1alpha1 +kind: Pool +metadata: + name: replicapool + namespace: rook +spec: + replication: + size: 1 + # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. + # Make sure you have enough OSDs to support the replica count or erasure code chunks. + #erasureCode: + # codingChunks: 2 + # dataChunks: 2 +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-block +provisioner: rook.io/block +parameters: + pool: replicapool` +} + +func getPvcDefiniton() string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: block-pv-claim + annotations: + volume.beta.kubernetes.io/storage-class: rook-block +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1M` +} +func getBlockPodDefintion() string { + return `apiVersion: v1 +kind: Pod +metadata: + name: block-test +spec: + containers: + - image: busybox + name: block-test1 + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: block-persistent-storage + mountPath: /tmp/rook1 + volumes: + - name: block-persistent-storage + persistentVolumeClaim: + claimName: block-pv-claim + restartPolicy: Never` +} diff --git a/tests/smoke/file_test.go b/tests/smoke/file_test.go index ed30473fa..aabd4114b 100644 --- a/tests/smoke/file_test.go +++ b/tests/smoke/file_test.go @@ -17,9 +17,17 @@ limitations under the License. package smoke import ( + "fmt" "github.com/stretchr/testify/require" ) +var ( + fileSystemName = "testfs" + fileMountPath = "/tmp/rookfs" + filePodName = "file-test" + filePodNamespace = "rook" +) + // Smoke Test for File System Storage - Test check the following operations on FileSystem Storage in order //Create,Mount,Write,Read,Unmount and Delete. func (suite *SmokeSuite) TestFileStorage_SmokeTest() { @@ -28,7 +36,7 @@ func (suite *SmokeSuite) TestFileStorage_SmokeTest() { rfc := suite.helper.GetFileSystemClient() logger.Infof("Step 1: Create file System") - fscErr := suite.helper.CreateFileStorage() + _, fscErr := rfc.FSCreate(fileSystemName) require.Nil(suite.T(), fscErr) fileSystemList, _ := rfc.FSList() require.Equal(suite.T(), 1, len(fileSystemList), "There should one shared file system present") @@ -37,28 +45,30 @@ func (suite *SmokeSuite) TestFileStorage_SmokeTest() { logger.Infof("File system created") logger.Infof("Step 2: Mount file System") - mtfsErr := suite.helper.MountFileStorage() + mtfsErr := suite.podWithFileSystem("create") require.Nil(suite.T(), mtfsErr) + require.True(suite.T(), suite.k8sh.IsPodRunningInNamespace(filePodName), "make sure file-test pod is in running state") logger.Infof("File system mounted successfully") logger.Infof("Step 3: Write to file system") - wfsErr := suite.helper.WriteToFileStorage("Test data for file", "fsFile1") + _, wfsErr := rfc.FSWrite(filePodName, fileMountPath, "Smoke Test Data for file system storage", "fsFile1", filePodNamespace) require.Nil(suite.T(), wfsErr) logger.Infof("Write to file system successful") logger.Infof("Step 4: Read from file system") - read, rdErr := suite.helper.ReadFromFileStorage("fsFile1") + read, rdErr := rfc.FSRead(filePodName, fileMountPath, "fsFile1", filePodNamespace) require.Nil(suite.T(), rdErr) - require.Contains(suite.T(), read, "Test data for file", "make sure content of the files is unchanged") + require.Contains(suite.T(), read, "Smoke Test Data for file system storage", "make sure content of the files is unchanged") logger.Infof("Read from file system successful") logger.Infof("Step 5: UnMount file System") - umtfsErr := suite.helper.UnmountFileStorage() + umtfsErr := suite.podWithFileSystem("delete") require.Nil(suite.T(), umtfsErr) + require.True(suite.T(), suite.k8sh.IsPodTerminatedInNamespace(filePodName), "make sure file-test pod is terminated") logger.Infof("File system mounted successfully") logger.Infof("Step 6: Deleting file storage") - suite.helper.DeleteFileStorage() + rfc.FSDelete(fileSystemName) //Delete is not deleting filesystem - known issue //require.Nil(suite.T(), fsd_err) logger.Infof("File system deleted") @@ -66,7 +76,62 @@ func (suite *SmokeSuite) TestFileStorage_SmokeTest() { func (suite *SmokeSuite) fileTestDataCleanUp() { logger.Infof("Cleaning up file system") - suite.helper.UnmountFileStorage() - suite.helper.DeleteFileStorage() + suite.podWithFileSystem("delete") + suite.helper.GetFileSystemClient().FSDelete(fileSystemName) } + +func (suite *SmokeSuite) podWithFileSystem(action string) error { + mons, err := suite.k8sh.GetMonitorPods() + if err != nil { + return err + } + ip1, _ := suite.k8sh.GetMonIP(mons[0]) + ip2, _ := suite.k8sh.GetMonIP(mons[1]) + ip3, _ := suite.k8sh.GetMonIP(mons[2]) + + config := map[string]string{ + "mon0": ip1, + "mon1": ip2, + "mon2": ip3, + } + logger.Infof("mountFileStorage: Mons: %+v", mons) + _, err = suite.k8sh.ResourceOperationFromTemplate(action, getFileSystemTestPod(), config) + if err != nil { + return fmt.Errorf("failed to %s pod -- %s. %+v", action, getFileSystemTestPod(), err) + } + return nil +} + +func getFileSystemTestPod() string { + return `apiVersion: v1 +kind: Pod +metadata: + name: file-test + namespace: rook +spec: + containers: + - name: file-test1 + image: busybox + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + env: + volumeMounts: + - mountPath: "/tmp/rookfs" + name: testfs + volumes: + - name: testfs + cephfs: + monitors: + - {{.mon0}} + - {{.mon1}} + - {{.mon2}} + user: admin + secretRef: + name: rook-admin + readOnly: false + restartPolicy: Never +` +} diff --git a/tests/smoke/object_test.go b/tests/smoke/object_test.go index 95072cdfb..e33136f0c 100644 --- a/tests/smoke/object_test.go +++ b/tests/smoke/object_test.go @@ -19,9 +19,11 @@ package smoke import ( "errors" + "fmt" "github.com/rook/rook/pkg/model" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/require" + "time" ) var ( @@ -37,37 +39,39 @@ var ( //Create object store, Create User, Connect to Object Store, Create Bucket, Read/Write/Delete to bucket,Delete Bucket and //Delete user func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { - defer suite.blockTestDataCleanUp() + defer suite.objectTestDataCleanUp() + oc := suite.helper.GetObjectClient() + logger.Infof("Object Storage Smoke Test - Create Object Store, User,Bucket and read/write to bucket") logger.Infof("Step 0 : Create Object Store") - _, cobsErr := suite.helper.CreateObjectStore() + cobsErr := suite.createObjectStore() require.Nil(suite.T(), cobsErr) logger.Infof("Object store created successfully") logger.Infof("Step 1 : Create Object Store User") - initialUsers, _ := suite.helper.GetObjectStoreUsers() - _, cosuErr := suite.helper.CreateObjectStoreUser() + initialUsers, _ := oc.ObjectListUser() + _, cosuErr := oc.ObjectCreateUser(userid, userdisplayname) require.Nil(suite.T(), cosuErr) - usersAfterCrate, _ := suite.helper.GetObjectStoreUsers() + usersAfterCrate, _ := oc.ObjectListUser() require.Equal(suite.T(), len(initialUsers)+1, len(usersAfterCrate), "Make sure user list count is increaded by 1") - getuserData, guErr := suite.helper.GetObjectStoreUser(userid) + getuserData, guErr := oc.ObjectGetUser(userid) require.Nil(suite.T(), guErr) require.Equal(suite.T(), userid, getuserData.UserID, "Check user id returned") require.Equal(suite.T(), userdisplayname, *getuserData.DisplayName, "Check user name returned") logger.Infof("Object store user created successfully") logger.Infof("Step 2 : Get connection information") - conninfo, conninfoError := suite.helper.GetObjectStoreUser(userid) + conninfo, conninfoError := oc.ObjectGetUser(userid) require.Nil(suite.T(), conninfoError) - s3endpoint, _ := suite.helper.GetRGWServiceURL() + s3endpoint, _ := suite.k8sh.GetRGWServiceURL() s3client := utils.CreateNewS3Helper(s3endpoint, *conninfo.AccessKey, *conninfo.SecretKey) logger.Infof("Step 3 : Create bucket") - initialBuckets, _ := suite.helper.GetObjectStoreBucketList() + initialBuckets, _ := oc.ObjectBucketList() s3client.CreateBucket(bucketname) - BucketsAfterCreate, _ := suite.helper.GetObjectStoreBucketList() + BucketsAfterCreate, _ := oc.ObjectBucketList() require.Equal(suite.T(), len(initialBuckets)+1, len(BucketsAfterCreate), "Make sure new bucket is created") bkt, _ := getBucket(bucketname, BucketsAfterCreate) require.Equal(suite.T(), bucketname, bkt.Name) @@ -80,7 +84,7 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { require.Equal(suite.T(), uint64(0), initObjNum) _, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType) require.Nil(suite.T(), poErr) - BucketsAfterPut, _ := suite.helper.GetObjectStoreBucketList() + BucketsAfterPut, _ := oc.ObjectBucketList() ObjSize, ObjNum, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterPut) require.NotEmpty(suite.T(), ObjSize) require.Equal(suite.T(), uint64(1), ObjNum) @@ -90,12 +94,12 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { read, err := s3client.GetObjectInBucket(bucketname, objectKey) require.Nil(suite.T(), err) require.Equal(suite.T(), objBody, read) - logger.Infof("Object retrived from bucket successfully") + logger.Infof("Object retrieved from bucket successfully") logger.Infof("Step 6 : Delete Object on bucket") _, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey) require.Nil(suite.T(), delobjErr) - BucketsAfterOjbDelete, _ := suite.helper.GetObjectStoreBucketList() + BucketsAfterOjbDelete, _ := oc.ObjectBucketList() ObjSize1, ObjNum1, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterOjbDelete) require.Equal(suite.T(), uint64(0), ObjSize1) require.Equal(suite.T(), uint64(0), ObjNum1) @@ -104,14 +108,14 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { logger.Infof("Step 6 : Delete bucket") _, bkdelErr := s3client.DeleteBucket(bucketname) require.Nil(suite.T(), bkdelErr) - BucketsAfterDelete, _ := suite.helper.GetObjectStoreBucketList() + BucketsAfterDelete, _ := oc.ObjectBucketList() require.Equal(suite.T(), len(initialBuckets), len(BucketsAfterDelete), "Make sure new bucket is deleted") logger.Infof("Bucket deleted successfully") logger.Infof("Step 7 : Delete User") - usersBeforeDelete, _ := suite.helper.GetObjectStoreUsers() - suite.helper.DeleteObjectStoreUser() - usersAfterDelete, _ := suite.helper.GetObjectStoreUsers() + usersBeforeDelete, _ := oc.ObjectListUser() + oc.ObjectDeleteUser(userid) + usersAfterDelete, _ := oc.ObjectListUser() require.Equal(suite.T(), len(usersBeforeDelete)-1, len(usersAfterDelete), "Make sure user list count is reducd by 1") logger.Infof("Object store user deleted successfully") @@ -119,15 +123,16 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { func (suite *SmokeSuite) objectTestDataCleanUp() { logger.Infof("Cleaning up object store") - userinfo, err := suite.helper.GetObjectStoreUser(userid) + oc := suite.helper.GetObjectClient() + userinfo, err := oc.ObjectGetUser(userid) if err != nil { return //when user is not found } - s3endpoint, _ := suite.helper.GetRGWServiceURL() + s3endpoint, _ := suite.k8sh.GetRGWServiceURL() s3client := utils.CreateNewS3Helper(s3endpoint, *userinfo.AccessKey, *userinfo.SecretKey) s3client.DeleteObjectInBucket(bucketname, objectKey) s3client.DeleteBucket(bucketname) - suite.helper.DeleteObjectStoreUser() + oc.ObjectDeleteUser(userid) } @@ -142,6 +147,26 @@ func getBucket(bucketname string, bucketList []model.ObjectBucket) (model.Object return model.ObjectBucket{}, errors.New("Bucket not found") } +func (suite *SmokeSuite) createObjectStore() error { + suite.helper.GetObjectClient().ObjectCreate() + time.Sleep(time.Second * 2) //wait for rgw service to to started + if suite.k8sh.IsServiceUpInNameSpace("rook-ceph-rgw") { + _, err := suite.k8sh.GetService("rgw-external") + if err != nil { + suite.k8sh.KubectlWithStdin(getRGWExternalServiceDef(), []string{"create", "-f", "-"}...) + if !suite.k8sh.IsServiceUpInNameSpace("rgw-external") { + logger.Infof("Couldn't start RGW external serivce") + return fmt.Errorf("Cannot expose rgw servie for external user") + } + + } + + return nil + } + return fmt.Errorf("RGW service not started") + +} + func getBucketSizeAndObjectes(bucketname string, bucketList []model.ObjectBucket) (uint64, uint64, error) { bkt, err := getBucket(bucketname, bucketList) if err != nil { @@ -149,3 +174,28 @@ func getBucketSizeAndObjectes(bucketname string, bucketList []model.ObjectBucket } return bkt.Size, bkt.NumberOfObjects, nil } + +//Expose RGW service +func getRGWExternalServiceDef() string { + return `apiVersion: v1 +kind: Service +metadata: + name: rgw-external + namespace: rook + labels: + app: rook-ceph-rgw + rook_cluster: rook +spec: + ports: + - name: rook-ceph-rgw + port: 53390 + protocol: TCP + nodePort: 30001 + selector: + app: rook-ceph-rgw + rook_cluster: rook + sessionAffinity: None + type: NodePort +` + +} diff --git a/tests/smoke/smoke_test.go b/tests/smoke/smoke_test.go index 6088a7bdd..938f98085 100644 --- a/tests/smoke/smoke_test.go +++ b/tests/smoke/smoke_test.go @@ -19,6 +19,9 @@ package smoke import ( "testing" + "github.com/coreos/pkg/capnslog" + + "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/enums" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" @@ -26,13 +29,16 @@ import ( "github.com/stretchr/testify/suite" ) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "smoketest") + func TestSmokeSuiteK8s(t *testing.T) { suite.Run(t, new(SmokeSuite)) } type SmokeSuite struct { suite.Suite - helper *TestHelper + helper *clients.TestClient + k8sh *utils.K8sHelper installer *installer.InstallHelper } @@ -40,12 +46,14 @@ func (suite *SmokeSuite) SetupSuite() { kh, err := utils.CreatK8sHelper() require.NoError(suite.T(), err) + suite.k8sh = kh + suite.installer = installer.NewK8sRookhelper(kh.Clientset) err = suite.installer.InstallRookOnK8s() require.NoError(suite.T(), err) - suite.helper, err = CreateSmokeTestClient(enums.Kubernetes, suite.installer.Env.K8sVersion, kh) + suite.helper, err = clients.CreateTestClient(enums.Kubernetes, kh) require.Nil(suite.T(), err) } diff --git a/tests/smoke/test_helper.go b/tests/smoke/test_helper.go deleted file mode 100644 index 89ab61f3b..000000000 --- a/tests/smoke/test_helper.go +++ /dev/null @@ -1,531 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package smoke - -import ( - "fmt" - "os" - "path/filepath" - "time" - - "strings" - - "github.com/coreos/pkg/capnslog" - "github.com/rook/rook/pkg/model" - "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" - "github.com/rook/rook/tests/framework/enums" - "github.com/rook/rook/tests/framework/utils" -) - -const ( - dataPath = "src/github.com/rook/rook/tests/data" -) - -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "smoketest") - -//TestHelper struct for smoke test -type TestHelper struct { - platform enums.RookPlatformType - rookclient *clients.TestClient - utils.K8sHelper - k8sVersion string - namespace string -} - -type blockTestData struct { - name string - size int - mountpath string - pvcDefPath string - podDefPath string -} - -type fileTestData struct { - name string - mountpath string - podname string - podDefPath string -} - -type objectUserData struct { - userID string - displayname string - emailID string -} - -type objectConnectionData struct { - awsEndpoint string - awsHost string - awsSecretKeyID string - awsSecretAccessKey string -} - -func createObjectUserData(userid string, displayname string, emailid string) objectUserData { - return objectUserData{userid, displayname, emailid} -} - -func createObjectConnectionData(endpoint string, host string, secretid string, secretkey string) objectConnectionData { - return objectConnectionData{endpoint, host, secretid, secretkey} -} - -//CreateSmokeTestClient constructor for creating instance of smoke test client -func CreateSmokeTestClient(platform enums.RookPlatformType, k8sversion string, kh *utils.K8sHelper) (*TestHelper, error) { - - rc, err := getRookClient(platform, kh) - if err != nil { - return nil, fmt.Errorf("failed to get rook client. %+v", err) - } - - return &TestHelper{platform: platform, - rookclient: rc, - K8sHelper: *kh, - namespace: "rook", - k8sVersion: k8sversion}, err -} - -func getRookClient(platform enums.RookPlatformType, kh *utils.K8sHelper) (*clients.TestClient, error) { - return clients.CreateTestClient(platform, kh) -} - -//GetBlockClient returns a pointer to block client to perform block operations -func (h *TestHelper) GetBlockClient() contracts.BlockOperator { - return h.rookclient.GetBlockClient() -} - -//GetFileSystemClient returns a pointer to fileSystem client to perform filesSystem operations -func (h *TestHelper) GetFileSystemClient() contracts.FileSystemOperator { - return h.rookclient.GetFileSystemClient() -} - -//GetObjectClient returns a pointer to object store client to perform object store operations -func (h *TestHelper) GetObjectClient() contracts.ObjectOperator { - return h.rookclient.GetObjectClient() -} - -func getDataPath(file string) string { - return filepath.Join(os.Getenv("GOPATH"), dataPath, file) -} - -func (h *TestHelper) getBlockTestData() (blockTestData, error) { - switch h.platform { - case enums.Kubernetes: - return blockTestData{"block-test", - 1048576, - "/tmp/rook1", - getDataPath("smoke/pool_pvc.yaml"), - getDataPath("smoke/block_mount.yaml")}, nil - case enums.StandAlone: - return blockTestData{"block-test", - 1048576, - "/tmp/rook1", - "", - ""}, nil - default: - return blockTestData{}, fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -func (h *TestHelper) getFileTestData() (fileTestData, error) { - switch h.platform { - case enums.Kubernetes: - return fileTestData{"testfs", - "/tmp/rookfs", - "file-test", - getDataPath("smoke/file_mount.tmpl")}, nil - case enums.StandAlone: - return fileTestData{"testfs", - "/tmp/rookfs", - "", ""}, nil - default: - return fileTestData{}, fmt.Errorf("Unsupported Rook Platform Type") - } -} - -func (h *TestHelper) getObjectStoreUserData() objectUserData { - return objectUserData{"rook-user", "A rook RGW user", ""} -} -func (h *TestHelper) getRGWExtenalSevDef() string { - return getDataPath("smoke/rgw_external.yaml") -} - -//CreateBlockStorage function creates a block store -func (h *TestHelper) CreateBlockStorage() error { - switch h.platform { - case enums.Kubernetes: - blockData, err := h.getBlockTestData() - if err != nil { - return fmt.Errorf("Failed to get block test data. %+v", err) - } - - if strings.EqualFold(h.k8sVersion, "v1.5") { - _, err := h.ResourceOperation("create", getDataPath("smoke/pool_sc_1_5.yaml")) - if err != nil { - return fmt.Errorf("failed to create 1.5 pool. %+v", err) - } - } else { - _, err := h.ResourceOperation("create", getDataPath("smoke/pool_sc.yaml")) - if err != nil { - return fmt.Errorf("failed to create 1.5 pool. %+v", err) - } - } - // see https://github.com/rook/rook/issues/767 - time.Sleep(10 * time.Second) - _, err = h.ResourceOperation("create", blockData.pvcDefPath) - return err - - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//WaitUntilPVCIsBound waits untill PVC is in bound state and returs true if the state is transitisioned to -//Bound state, or returns false -func (h *TestHelper) WaitUntilPVCIsBound() bool { - - return h.K8sHelper.WaitUntilPVCIsBound("block-pv-claim") -} - -//MountBlockStorage function mounts a rook created block storage -func (h *TestHelper) MountBlockStorage() error { - switch h.platform { - case enums.Kubernetes: - blockData, _ := h.getBlockTestData() - _, err := h.GetBlockClient().BlockMap(blockData.podDefPath, blockData.mountpath) - if err != nil { - return err - } - if h.IsPodRunning(blockData.name) { - - return nil - } - return fmt.Errorf("Cannot mount block storage") - - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//WriteToBlockStorage function writes a block to rook provisioned block store -func (h *TestHelper) WriteToBlockStorage(data string, filename string) error { - switch h.platform { - case enums.Kubernetes: - blockData, _ := h.getBlockTestData() - _, err := h.GetBlockClient().BlockWrite(blockData.name, blockData.mountpath, data, filename, "") - return err - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//ReadFromBlockStorage function Reads a block from rook provisioned block store -func (h *TestHelper) ReadFromBlockStorage(filename string) (string, error) { - switch h.platform { - case enums.Kubernetes: - blockData, _ := h.getBlockTestData() - return h.GetBlockClient().BlockRead(blockData.name, blockData.mountpath, filename, "") - case enums.StandAlone: - return "", fmt.Errorf("NOT YET IMPLEMENTED") - default: - return "", fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//UnMountBlockStorage function unmounts Block storage -func (h *TestHelper) UnMountBlockStorage() error { - switch h.platform { - case enums.Kubernetes: - blockData, _ := h.getBlockTestData() - _, err := h.GetBlockClient().BlockUnmap(blockData.podDefPath, blockData.mountpath) - if err != nil { - return err - } - if h.IsPodTerminated(blockData.name) { - return nil - } - return fmt.Errorf("Cannot unmount block storage") - - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//DeleteBlockStorage function deletes block storage -func (h *TestHelper) DeleteBlockStorage() error { - switch h.platform { - case enums.Kubernetes: - blockData, _ := h.getBlockTestData() - // Delete storage pool, storage class and pvc - if strings.EqualFold(h.k8sVersion, "v1.5") { - h.ResourceOperation("delete", getDataPath("smoke/pool_sc_1_5.yaml")) - } else { - h.ResourceOperation("delete", getDataPath("smoke/pool_sc.yaml")) - } - _, err := h.ResourceOperation("delete", blockData.pvcDefPath) - return err - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//CleanUpDymanicBlockStorage is helper method to clean up bock storage created by tests -func (h *TestHelper) CleanUpDymanicBlockStorage() { - switch h.platform { - case enums.Kubernetes: - // Delete storage pool, storage class and pvc - blockImagesList, _ := h.GetBlockClient().BlockList() - for _, blockImage := range blockImagesList { - h.rookclient.GetRestAPIClient().DeleteBlockImage(blockImage) - } - } -} - -//CreateFileStorage function creates a file system -func (h *TestHelper) CreateFileStorage() error { - switch h.platform { - case enums.Kubernetes: - fileTestData, _ := h.getFileTestData() - // Create storage pool, storage class and pvc - _, err := h.GetFileSystemClient().FSCreate(fileTestData.name) - return err - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//MountFileStorage function mounts a file system -func (h *TestHelper) MountFileStorage() error { - switch h.platform { - case enums.Kubernetes: - fileTestData, _ := h.getFileTestData() - mons, err := h.GetMonitorPods() - if err != nil { - return err - } - ip1, _ := h.GetMonIP(mons[0]) - ip2, _ := h.GetMonIP(mons[1]) - ip3, _ := h.GetMonIP(mons[2]) - - config := map[string]string{ - "mon0": ip1, - "mon1": ip2, - "mon2": ip3, - } - - // Create pod that has has file sytem mounted - logger.Infof("mountFileStorage: Mons: %+v", mons) - _, err = h.ResourceOperationFromTemplate("create", fileTestData.podDefPath, config) - if err != nil { - return fmt.Errorf("failed to create podDefPath %s. %+v", fileTestData.podDefPath, err) - } - - logger.Infof("mountFileStorage: checking for running pod") - if h.IsPodRunningInNamespace(fileTestData.podname) { - return nil - } - return fmt.Errorf("File test pod did not start") - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - } -} - -//WriteToFileStorage functions creates a new file on rook file system -func (h *TestHelper) WriteToFileStorage(data string, filename string) error { - switch h.platform { - case enums.Kubernetes: - fileTestData, _ := h.getFileTestData() - _, err := h.GetFileSystemClient().FSWrite(fileTestData.podname, - fileTestData.mountpath, data, filename, h.namespace) - return err - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - } -} - -//ReadFromFileStorage function get file content from a rook file storage -func (h *TestHelper) ReadFromFileStorage(filename string) (string, error) { - switch h.platform { - case enums.Kubernetes: - fileTestData, _ := h.getFileTestData() - return h.GetFileSystemClient().FSRead(fileTestData.podname, - fileTestData.mountpath, filename, h.namespace) - case enums.StandAlone: - return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") - default: - return "", fmt.Errorf("Unsupported Rook Platform Type") - } -} - -//UnmountFileStorage funxtion unmounts a file system -func (h *TestHelper) UnmountFileStorage() error { - switch h.platform { - case enums.Kubernetes: - fileTestData, _ := h.getFileTestData() - mons, err := h.GetMonitorPods() - if err != nil { - return err - } - ip1, _ := h.GetMonIP(mons[0]) - ip2, _ := h.GetMonIP(mons[1]) - ip3, _ := h.GetMonIP(mons[2]) - - config := map[string]string{ - "mon0": ip1, - "mon1": ip2, - "mon2": ip3, - } - - // Create pod that has has file sytem mounted - _, err = h.ResourceOperationFromTemplate("delete", fileTestData.podDefPath, config) - if err != nil { - return err - } - if h.IsPodTerminatedInNamespace(fileTestData.podname) { - return nil - } - return fmt.Errorf("Cannot unmount File storage") - case enums.StandAlone: - return fmt.Errorf("NOT YET IMPLEMENTED") - default: - return fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//DeleteFileStorage deletes File system -func (h *TestHelper) DeleteFileStorage() (string, error) { - switch h.platform { - case enums.Kubernetes: - fileTestData, _ := h.getFileTestData() - // Create storage pool, storage class and pvc - return h.GetFileSystemClient().FSDelete(fileTestData.name) - case enums.StandAlone: - return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") - default: - return "", fmt.Errorf("Unsupported Rook Platform Type") - - } -} - -//CreateObjectStore function creates a object store -func (h *TestHelper) CreateObjectStore() (string, error) { - switch h.platform { - case enums.Kubernetes: - h.GetObjectClient().ObjectCreate() - time.Sleep(time.Second * 2) //wait for rgw service to to started - if h.IsServiceUpInNameSpace("rook-ceph-rgw") { - _, err := h.GetService("rgw-external") - if err != nil { - h.ResourceOperation("create", h.getRGWExtenalSevDef()) - if !h.IsServiceUpInNameSpace("rgw-external") { - return "Couldn't create object store ", fmt.Errorf("Cannot expose rgw servie for external user") - } - - } - - return "OJECT STORE CREATED", nil - } - return "Couldn't create object store ", fmt.Errorf("Cannot create object store") - case enums.StandAlone: - return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") - default: - return "", fmt.Errorf("Unsupported Rook Platform Type") - } -} - -//CreateObjectStoreUser function creates a new object store user -func (h *TestHelper) CreateObjectStoreUser() (string, error) { - objectUserdata := h.getObjectStoreUserData() - _, err := h.GetObjectClient().ObjectCreateUser(objectUserdata.userID, objectUserdata.displayname) - if err != nil { - return "USER NOT CREATED", fmt.Errorf("User not created for object store %+v", err) - } - return "USER CREATED ", nil -} - -//GetObjectStoreUsers returns all useres -func (h *TestHelper) GetObjectStoreUsers() ([]model.ObjectUser, error) { - return h.GetObjectClient().ObjectListUser() - -} - -//GetObjectStoreUser function returns a user object with matching userId -func (h *TestHelper) GetObjectStoreUser(userid string) (*model.ObjectUser, error) { - return h.GetObjectClient().ObjectGetUser(userid) - -} - -//GetObjectStoreConnection function returns connection information about object store -func (h *TestHelper) GetObjectStoreConnection() (*model.ObjectStoreConnectInfo, error) { - return h.GetObjectClient().ObjectConnection() - -} - -//GetObjectStoreBucketList function returns list of buckets for a user -func (h *TestHelper) GetObjectStoreBucketList() ([]model.ObjectBucket, error) { - return h.GetObjectClient().ObjectBucketList() - -} - -//DeleteObjectStoreUser function deletes rgw object store user -func (h *TestHelper) DeleteObjectStoreUser() error { - objectUserdata := h.getObjectStoreUserData() - return h.GetObjectClient().ObjectDeleteUser(objectUserdata.userID) - -} - -//GetRGWServiceURL returns URL of ceph RGW service in the cluster -func (h *TestHelper) GetRGWServiceURL() (string, error) { - switch h.platform { - case enums.Kubernetes: - hostip, err := h.GetPodHostID("rook-ceph-rgw", h.namespace) - if err != nil { - panic(fmt.Errorf("RGW pods not found/object store possibly not started")) - } - endpoint := hostip + ":30001" - return endpoint, err - case enums.StandAlone: - return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") - default: - return "", fmt.Errorf("Unsupported Rook Platform Type") - - } -}