diff --git a/.gitignore b/.gitignore index 647e38655..d43bd4be6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.vscode/ /.work /.download +/.idea /bin /vendor @@ -16,3 +17,7 @@ demo/standalone/cloud-config.yml .virtualbox/ .discovery-token +e2e/results +e2e/vendor +e2e/.glide +tests/smoke/temp_image.tar diff --git a/Jenkinsfile b/Jenkinsfile index 1cbb0be6b..cf1266493 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,26 @@ try { sh 'build/run make -j\$(nproc) check' } + stage('E2E') { + def exists = fileExists 'release/version' + + if (!exists) { + error('The file release/version does not exist') + } + + def rook_tag = readFile (file: 'release/version', encoding : 'utf-8').trim() + + if (rook_tag == '') { + error('Failed to get rook_tag from version file') + } + + echo 'Rook Tag is ' + rook_tag + + sh "e2e/scripts/smoke_test.sh ${rook_tag} Kubernetes v1.6" + + junit 'e2e/results/*.xml' + } + stage('Publish') { withCredentials([ [$class: 'UsernamePasswordMultiBinding', credentialsId: 'rook-quay-io', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASSWORD'], diff --git a/build/makelib/release.mk b/build/makelib/release.mk index 1b8893a13..14063a2da 100644 --- a/build/makelib/release.mk +++ b/build/makelib/release.mk @@ -99,9 +99,13 @@ endef $(foreach f,$(RELEASE_FLAVORS),$(foreach p,$(RELEASE_PLATFORMS), $(eval $(call release-target,$(f),$(p))))) -release.build: release.build.all +release.build: + @build/release/release.sh build.init + @$(MAKE) release.build.all -release.publish: release.publish.all +release.publish: + @build/release/release.sh publish.init + @$(MAKE) release.publish.all release.promote: @build/release/release.sh promote.init diff --git a/build/release/common.sh b/build/release/common.sh index d6bdeb658..282d76969 100644 --- a/build/release/common.sh +++ b/build/release/common.sh @@ -133,7 +133,7 @@ github_release_complete() { write_version_file() { # upload a file with the version number cat < ${RELEASE_DIR}/version -${version} +${RELEASE_VERSION} EOF } diff --git a/build/release/release.sh b/build/release/release.sh index c5a535123..9d772a9de 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -9,6 +9,13 @@ shift mkdir -p ${RELEASE_DIR} case ${action} in + build.init) + write_version_file + ;; + + publish.init) + ;; + build|publish|promote|cleanup) platform=$1 shift @@ -26,7 +33,6 @@ case ${action} in fi echo promoting release ${RELEASE_VERSION} to channel ${RELEASE_CHANNEL} - write_version_file publish_version_file github_create_or_replace_release ;; diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 000000000..3e5c15e22 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,30 @@ +# Rook Test Framework + +The Rook Test Framework is a Go project that that uses a Docker container with Kubernetes +tools and Docker pre-installed to quickly create a 3 node Kubernetes cluster using Docker in +Docker (DinD) techniques, installs Rook and then runs automation tests to verify its functionality. + +##Requirements +1. Docker version => 1.2 && < 17.0 +2. Ubuntu Host 16 (currently the framework has only been tested on this version) + +##Instructions +On your Ubuntu box do the following; +1. Be sure to allocate a minimum of 4 processors to your vm +2. Navigate to the root directory of the repo and run the following:; + ```e2e/scripts/smoke_test.sh``` + +At this point a Docker container with the base image of "quay.io/quantum/rook-test" is created, a +Kubernetes 3 node cluster is created using Kubeadm, that is visible from the Docker host. The +nodes can be identified by the following container names (kube-master, kube-node1 and kube-node2). +All tests with the name 'SmokeSuite' in the method will execute and the results will be of a junit +type output to the e2e/results directory. + + +### Cleanup +The Rook Test Framework normally cleans up all the containers it creates to setup the environment +and to run the tests. If for some reason the cleanup of the Rook Test framework should fail, the easiest way to manually +cleanup the environment is by doing the following; + +1. Delete the docker container that uses an image named "quay.io/quantum/rook-test" +2. Run the script rook/e2e/framework/manager/scripts/rook-dind-cluster-v1.6.sh clean \ No newline at end of file diff --git a/e2e/data/platform.properties b/e2e/data/platform.properties new file mode 100644 index 000000000..e69de29bb diff --git a/e2e/data/smoke/block_mount.yaml b/e2e/data/smoke/block_mount.yaml new file mode 100644 index 000000000..e4197ed02 --- /dev/null +++ b/e2e/data/smoke/block_mount.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: block-test +spec: + containers: + - image: busybox + name: block-test1 + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: block-persistent-storage + mountPath: /tmp/rook1 + volumes: + - name: block-persistent-storage + persistentVolumeClaim: + claimName: block-pv-claim + restartPolicy: Never diff --git a/e2e/data/smoke/file_mount.tmpl b/e2e/data/smoke/file_mount.tmpl new file mode 100644 index 000000000..f42d070af --- /dev/null +++ b/e2e/data/smoke/file_mount.tmpl @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +metadata: + name: file-test + namespace: rook +spec: + containers: + - name: file-test1 + image: busybox + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + env: + volumeMounts: + - mountPath: "/tmp/rookfs" + name: testfs + volumes: + - name: testfs + cephfs: + monitors: + - {{.mon0}} + - {{.mon1}} + - {{.mon2}} + user: admin + secretRef: + name: rook-admin + readOnly: false + restartPolicy: Never diff --git a/e2e/data/smoke/pool_sc_pvc.yaml b/e2e/data/smoke/pool_sc_pvc.yaml new file mode 100644 index 000000000..914e541e8 --- /dev/null +++ b/e2e/data/smoke/pool_sc_pvc.yaml @@ -0,0 +1,34 @@ +apiVersion: rook.io/v1alpha1 +kind: Pool +metadata: + name: replicapool + namespace: rook +spec: + replication: + size: 1 + # For an erasure-coded pool, comment out the replication count above and uncomment the following settings. + # Make sure you have enough OSDs to support the replica count or erasure code chunks. + #erasureCode: + # codingChunks: 2 + # dataChunks: 2 +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-block +provisioner: rook.io/block +parameters: + pool: replicapool +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: block-pv-claim + annotations: + volume.beta.kubernetes.io/storage-class: rook-block +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1M diff --git a/e2e/data/smoke/rgw_external.yaml b/e2e/data/smoke/rgw_external.yaml new file mode 100644 index 000000000..1458da8f2 --- /dev/null +++ b/e2e/data/smoke/rgw_external.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: rgw-external + namespace: rook + labels: + app: rgw + rook_cluster: rook +spec: + ports: + - name: rgw + port: 53390 + protocol: TCP + targetPort: 53390 + selector: + app: rgw + rook_cluster: rook + sessionAffinity: None + type: NodePort diff --git a/e2e/framework/clients/k8s_rook_block.go b/e2e/framework/clients/k8s_rook_block.go new file mode 100644 index 000000000..7b8d2b59d --- /dev/null +++ b/e2e/framework/clients/k8s_rook_block.go @@ -0,0 +1,138 @@ +package clients + +import ( + "fmt" + "github.com/rook/rook/e2e/framework/contracts" +) + +type k8sRookBlock struct { + transportClient contracts.ITransportClient +} + +var ( + listCmd = []string{"rook", "block", "ls"} + writeDataToBlockPod = []string{"sh", "-c", "WRITE_DATA_CMD"} + readDataFromBlockPod = []string{"cat", "READ_DATA_CMD"} +) + +// Constructor to create k8sRookBlock - client to perform rook Block operations on k8s +func CreateK8SRookBlock(client contracts.ITransportClient) *k8sRookBlock { + return &k8sRookBlock{transportClient: client} +} + +// Function to create a Block using Rook +// Input paramaters - +//name - path to a yaml file that creates a pvc in k8s - yaml should describe name and size of pvc being created +//size - not user for k8s implementation since its descried on the pvc yaml definition +//Output - k8s create pvc operation output and/or error +func (rb *k8sRookBlock) BlockCreate(name string, size int) (string, error) { + cmdArgs := []string{name} + out, err, status := rb.transportClient.Create(cmdArgs, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to create block -- : %s", err) + } +} + +// Function to delete a Block using Rook +// Input paramaters - +//name - path to a yaml file that where pvc is desirbed - delete is run on the the yaml definition +//Output - k8s delete pvc operation output and/or error +func (rb *k8sRookBlock) BlockDelete(name string) (string, error) { + cmdArgs := []string{name} + out, err, status := rb.transportClient.Delete(cmdArgs, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to delete block -- : %s", err) + } +} + +// Function to list all the blocks created/being managed by rook +//Returns a ouput for rook cli for block list +func (rb *k8sRookBlock) BlockList() (string, error) { + out, err, status := rb.transportClient.Execute(listCmd, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to list all blocks -- : %s", err) + } +} + +// Function to map a Block using Rook +// Input paramaters - +//name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier +//mountpath - not used in this impl since mountpath is defined in the pod definition +//Output - k8s create pod operation output and/or error +func (rb *k8sRookBlock) BlockMap(name string, mountpath string) (string, error) { + cmdArgs := []string{name} + out, err, status := rb.transportClient.Create(cmdArgs, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to Map block -- : %s", err) + } + +} + +// Function to write data to block created by rook ,i.e. write data to a pod that is using a pvc +// Input paramaters - +//name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier +//mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) +//data - data to be written +//filename - file where data is written to +//namespace - optional param - namespace of the pod +//Output - k8s exec pod operation output and/or error +func (rb *k8sRookBlock) BlockWrite(name string, mountpath string, data string, filename string, namespace string) (string, error) { + wt := "echo \"" + data + "\">" + mountpath + "/" + filename + writeDataToBlockPod[2] = wt + option := []string{name} + if namespace != "" { + option = append(option, namespace) + } + out, err, status := rb.transportClient.Execute(writeDataToBlockPod, option) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to write data to pod: %s --> %s", err, out) + } +} + +// Function to read from block created by rook ,i.e. Read data from a pod that is using a pvc +// Input paramaters - +//name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier +//mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) +//filename - file to be read +//namespace - optional param - namespace of the pod +//Output - k8s exec pod operation output and/or error +func (rb *k8sRookBlock) BlockRead(name string, mountpath string, filename string, namespace string) (string, error) { + rd := mountpath + "/" + filename + readDataFromBlockPod[1] = rd + option := []string{name} + if namespace != "" { + option = append(option, namespace) + } + out, err, status := rb.transportClient.Execute(readDataFromBlockPod, option) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to read data to pod -- : %s", err) + } +} + +// Function to map a Block using Rook +// Input paramaters - +//name - path to a yaml file - the pod described in yam file is deleted +//mountpath - not used in this impl since mountpath is defined in the pod definition +//Output - k8s delete pod operation output and/or error +func (rb *k8sRookBlock) BlockUnmap(name string, mountpath string) (string, error) { + cmdArgs := []string{name} + out, err, status := rb.transportClient.Delete(cmdArgs, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to unmap block -- : %s", err) + } + +} diff --git a/e2e/framework/clients/k8s_rook_file_system.go b/e2e/framework/clients/k8s_rook_file_system.go new file mode 100644 index 000000000..d63ead289 --- /dev/null +++ b/e2e/framework/clients/k8s_rook_file_system.go @@ -0,0 +1,140 @@ +package clients + +import ( + "fmt" + "github.com/rook/rook/e2e/framework/contracts" +) + +type k8sRookFileSystem struct { + transportClient contracts.ITransportClient +} + +var ( + fileSystemCreate = []string{"rook", "filesystem", "create", "--name", "NAME"} + fileSystemDelete = []string{"rook", "filesystem", "delete", "--name", "NAME"} + fileSystemList = []string{"rook", "filesystem", "ls"} + writeDataToFilePod = []string{"sh", "-c", "WRITE_DATA_CMD"} + readDataFromFilePod = []string{"cat", "READ_DATA_CMD"} +) + +// Constructor to create k8sRookFileSystem - client to perform rook file system operations on k8s +func CreateK8sRookFileSystem(client contracts.ITransportClient) *k8sRookFileSystem { + return &k8sRookFileSystem{transportClient: client} +} + +//Function to create a fileSystem in rook +//Input paramatres - +// name - name of the shared file system to be created +//Output - output returned by rook cli and/or error +func (rfs *k8sRookFileSystem) FSCreate(name string) (string, error) { + fileSystemCreate[4] = name + out, err, status := rfs.transportClient.Execute(fileSystemCreate, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to create FileSystem -- : %s",err) + } +} + +//Function to delete a fileSystem in rook +//Input paramatres - +// name - name of the shared file system to be deleted +//Output - output returned by rook cli and/or error +func (rfs *k8sRookFileSystem) FSDelete(name string) (string, error) { + fileSystemDelete[4] = name + out, err, status := rfs.transportClient.Execute(fileSystemDelete, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to delete FileSystem -- : %s",err) + } +} + +//Function to list a fileSystem in rook +//Output - output returned by rook cli and/or error +func (rfs *k8sRookFileSystem) FSList() (string, error) { + out, err, status := rfs.transportClient.Execute(fileSystemList, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to list FileSystem -- : %s",err) + } + +} + +//Function to Mount a file system created by rook(on a pod) +//Input paramaters - +//name - path to the yaml defintion file - definition of pod to be created that mounts existing file system +//path - ignored in this case - moount path is defined in the path definition +//output - output returned by k8s create pod operaton and/or error +func (rfs *k8sRookFileSystem) FSMount(name string, path string) (string, error) { + cmdArgs := []string{name} + out, err, status := rfs.transportClient.Create(cmdArgs, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to mount FileSystem -- : %s",err) + } + +} + +// Function to write data to file system created by rook ,i.e. write data to a pod that has filesystem mounted +// Input paramaters - +//name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier +//mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) +//data - data to be written +//filename - file where data is written to +//namespace - optional param - namespace of the pod +//Output - k8s exec pod operation output and/or error +func (rfs *k8sRookFileSystem) FSWrite(name string, mountpath string, data string, filename string, namespace string) (string, error) { + wt := "echo \"" + data + "\">" + mountpath + "/" + filename + writeDataToFilePod[2] = wt + option := []string{name} + if namespace != "" { + option = append(option, namespace) + } + out, err, status := rfs.transportClient.Execute(writeDataToFilePod, option) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to write data to pod -- : %s",err) + } +} + +// Function to write read from file system created by rook ,i.e. Read data from a pod that filesystem mounted +// Input paramaters - +//name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier +//mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) +//filename - file to be read +//namespace - optional param - namespace of the pod +//Output - k8s exec pod operation output and/or error +func (rfs *k8sRookFileSystem) FSRead(name string, mountpath string, filename string, namespace string) (string, error) { + rd := mountpath + "/" + filename + readDataFromFilePod[1] = rd + option := []string{name} + if namespace != "" { + option = append(option, namespace) + } + out, err, status := rfs.transportClient.Execute(readDataFromFilePod, option) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to write data to pod -- : %s",err) + } +} + +//Function to UnMount a file system created by rook(delete pod) +//Input paramaters - +//name - path to the yaml defintion file - definition of pod to be deleted that has a file system mounted +//path - ignored in this case - moount path is defined in the path definition +//output - output returned by k8s delete pod operaton and/or error +func (rfs *k8sRookFileSystem) FSUnmount(name string) (string, error) { + cmdArgs := []string{name} + out, err, status := rfs.transportClient.Delete(cmdArgs, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Unable to unmount FileSystem -- : %s",err) + } + +} diff --git a/e2e/framework/clients/k8s_rook_object.go b/e2e/framework/clients/k8s_rook_object.go new file mode 100644 index 000000000..6d5f9c152 --- /dev/null +++ b/e2e/framework/clients/k8s_rook_object.go @@ -0,0 +1,136 @@ +package clients + +import ( + "fmt" + "github.com/rook/rook/e2e/framework/contracts" +) + +type k8sRookObject struct { + transportClient contracts.ITransportClient +} + +var ( + objectStoreCreate = []string{"rook", "object", "create"} + objectStoreGetBuckets = []string{"rook", "object", "bucket", "list"} + objectStoreGetConnection = []string{"rook", "object", "connection", "REPLACE"} + objectStoreCreateUser = []string{"rook", "object", "user", "create", "USERID", "DISPALYNAME"} + objectStoreDeleteUser = []string{"rook", "object", "user", "delete", "USERID"} + objectStoreGetUser = []string{"rook", "object", "user", "get", "USERID"} + objectStoreGetUsers = []string{"rook", "object", "user", "list"} + objectStoreUpdateUser = []string{"rook", "object", "user", "update", "USERID", "--display-name", "DISPLAYNAME", + "--email", "EMAILID"} +) + +func CreateK8sRookObject(client contracts.ITransportClient) *k8sRookObject { + return &k8sRookObject{transportClient: client} +} + +//Function to create a object store in rook +//Input paramatres -None +//Output - output returned by rook cli and/or error +func (ro *k8sRookObject) ObjectCreate() (string, error) { + out, err, status := ro.transportClient.Execute(objectStoreCreate, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Failed to create Object store -- : %s",err) + } +} + +//Function to get Buckets present in rook object store +//Input paramatres - None +//Output - output returned by rook cli and/or error +func (ro *k8sRookObject) ObjectBucketList() (string, error) { + out, err, status := ro.transportClient.Execute(objectStoreGetBuckets, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Failed to list buckets -- : %s",err) + } +} + +//Function to get connection information for a user +//Input paramatres - userId +//Output - output returned by rook cli and/or error +func (ro *k8sRookObject) ObjectConnection(userid string) (string, error) { + objectStoreGetConnection[3] = userid + out, err, status := ro.transportClient.Execute(objectStoreGetConnection, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Failed to connect to object store -- : %s",err) + } +} + +//Function to create user on rook object store +//Input paramatres - userId and display Name +//Output - output returned by rook cli and/or error +func (ro *k8sRookObject) ObjectCreateUser(userid string, displayname string) (string, error) { + objectStoreCreateUser[4] = userid + objectStoreCreateUser[5] = displayname + + out, err, status := ro.transportClient.Execute(objectStoreCreateUser, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Failed to create user -- : %s",err) + } +} + +//Function to delete user on rook object store +//Input paramatres - userId +//Output - output returned by rook cli and/or error +func (ro *k8sRookObject) ObjectDeleteUser(userid string) (string, error) { + objectStoreDeleteUser[4] = userid + + out, err, status := ro.transportClient.Execute(objectStoreDeleteUser, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Failed to delete user -- : %s",err) + } +} + +//Function to get a user on rook object store +//Input paramatres - userId +//Output - output returned by rook cli and/or error +func (ro *k8sRookObject) ObjectGetUser(userid string) (string, error) { + + objectStoreGetUser[4] = userid + + out, err, status := ro.transportClient.Execute(objectStoreGetUser, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Failed to get user -- : %s",err) + } +} + +//Function to get all users on rook object store +//Input paramatres - none +//Output - output returned by rook cli and/or error +func (ro *k8sRookObject) ObjectListUser() (string, error) { + + out, err, status := ro.transportClient.Execute(objectStoreGetUsers, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Failed to list users -- : %s",err) + } +} + +//Function to update a user on rook object store +//Input paramatres - userId,display Name and email address +//Output - output returned by rook cli and/or error +func (ro *k8sRookObject) ObjectUpdateUser(userid string, displayname string, emailid string) (string, error) { + objectStoreUpdateUser[4] = userid + objectStoreUpdateUser[6] = displayname + objectStoreUpdateUser[8] = emailid + + out, err, status := ro.transportClient.Execute(objectStoreUpdateUser, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf("Failed to update object user -- : %s",err) + } +} diff --git a/e2e/framework/clients/k8s_rook_pool.go b/e2e/framework/clients/k8s_rook_pool.go new file mode 100644 index 000000000..da190a1d2 --- /dev/null +++ b/e2e/framework/clients/k8s_rook_pool.go @@ -0,0 +1,24 @@ +package clients + +import ( + "fmt" + "github.com/rook/rook/e2e/framework/contracts" +) + +type k8sRookPool struct { + transportClient contracts.ITransportClient +} + +func CreateK8sPool(client contracts.ITransportClient) *k8sRookPool { + return &k8sRookPool{transportClient: client} +} + +func (rp *k8sRookPool) PoolList() (string, error) { + //TODO - implement + return "Not YET IMPLEMENTED", fmt.Errorf("NOT YET IMPLEMENTED") +} + +func (rp *k8sRookPool) PoolCreate() (string, error) { + //TODO - implement + return "Not YET IMPLEMENTED", fmt.Errorf("NOT YET IMPLEMENTED") +} diff --git a/e2e/framework/clients/rook_client.go b/e2e/framework/clients/rook_client.go new file mode 100644 index 000000000..2289fdcce --- /dev/null +++ b/e2e/framework/clients/rook_client.go @@ -0,0 +1,105 @@ +package clients + +import ( + "fmt" + "github.com/rook/rook/e2e/framework/contracts" + "github.com/rook/rook/e2e/framework/enums" + "github.com/rook/rook/e2e/framework/transport" +) + +var ( + STATUS_CMD = []string{"rook", "status"} + VERSION_CMD = []string{"rook", "version"} + NODE_CMD = []string{"rook", "node"} +) + +type RookClient struct { + platform enums.RookPlatformType + transportClient contracts.ITransportClient + blockClient contracts.IRookBlock + fsClient contracts.IRookFilesystem + objectClient contracts.IRookObject + poolClient contracts.IRookPool +} + +const ( + unable_to_check_rook_status_msg = "Unable to check rook status - please check of rook is up and running" +) + +func CreateRook_Client(platform enums.RookPlatformType) (*RookClient, error) { + var transportClient contracts.ITransportClient + var block_client contracts.IRookBlock + var fs_client contracts.IRookFilesystem + var object_client contracts.IRookObject + var pool_client contracts.IRookPool + + switch { + case platform == enums.Kubernetes: + transportClient = transport.CreateNewk8sTransportClient() + block_client = CreateK8SRookBlock(transportClient) + fs_client = CreateK8sRookFileSystem(transportClient) + object_client = CreateK8sRookObject(transportClient) + pool_client = CreateK8sPool(transportClient) + case platform == enums.StandAlone: + transportClient = nil //TODO- Not yet implemented + block_client = nil //TODO- Not yet implemented + fs_client = nil //TODO- Not yet implemented + object_client = nil //TODO- Not yet implemented + pool_client = nil //TODO- Not yet implemented + default: + return &RookClient{}, fmt.Errorf("Unsupported Rook Platform Type") + } + + return &RookClient{ + platform, + transportClient, + block_client, + fs_client, + object_client, + pool_client, + }, nil + +} + +func (Client RookClient) Status() (string, error) { + out, err, status := Client.transportClient.Execute(STATUS_CMD, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf(unable_to_check_rook_status_msg) + } +} + +func (Client RookClient) Version() (string, error) { + out, err, status := Client.transportClient.Execute(VERSION_CMD, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf(unable_to_check_rook_status_msg) + } +} + +func (Client RookClient) Node() (string, error) { + out, err, status := Client.transportClient.Execute(NODE_CMD, nil) + if status == 0 { + return out, nil + } else { + return err, fmt.Errorf(unable_to_check_rook_status_msg) + } +} + +func (Client RookClient) GetBlockClient() contracts.IRookBlock { + return Client.blockClient +} + +func (Client RookClient) GetFileSystemClient() contracts.IRookFilesystem { + return Client.fsClient +} + +func (Client RookClient) GetObjectClient() contracts.IRookObject { + return Client.objectClient +} + +func (Client RookClient) GetPoolClient() contracts.IRookPool { + return Client.poolClient +} diff --git a/e2e/framework/contracts/interface_docker_client.go b/e2e/framework/contracts/interface_docker_client.go new file mode 100644 index 000000000..d27f29bfd --- /dev/null +++ b/e2e/framework/contracts/interface_docker_client.go @@ -0,0 +1,8 @@ +package contracts + +type IDockerClient interface { + Execute(cmd []string) (stdout string, stderr string, err error) + Stop(cmd []string) (stdout string, stderr string, err error) + Run(cmd []string) (stdout string, stderr string, err error) + ExecuteCmd(cmdArgs []string) (stdout string, stderr string, err error) +} diff --git a/e2e/framework/contracts/interface_rook_block_client.go b/e2e/framework/contracts/interface_rook_block_client.go new file mode 100644 index 000000000..2c3828dd7 --- /dev/null +++ b/e2e/framework/contracts/interface_rook_block_client.go @@ -0,0 +1,11 @@ +package contracts + +type IRookBlock interface { + BlockCreate(name string, size int) (string, error) + BlockDelete(name string) (string, error) + BlockList() (string, error) + BlockMap(name string, mountpath string) (string, error) + BlockWrite(name string, mountpath string, data string, filename string, namespace string) (string, error) + BlockRead(name string, mountpath string, filename string, namespace string) (string, error) + BlockUnmap(name string, mountpath string) (string, error) +} diff --git a/e2e/framework/contracts/interface_rook_client.go b/e2e/framework/contracts/interface_rook_client.go new file mode 100644 index 000000000..091bfb183 --- /dev/null +++ b/e2e/framework/contracts/interface_rook_client.go @@ -0,0 +1,7 @@ +package contracts + +type IRookClient interface { + Status() (string, error) + Version() (string, error) + Node() (string, error) +} diff --git a/e2e/framework/contracts/interface_rook_filesystem_client.go b/e2e/framework/contracts/interface_rook_filesystem_client.go new file mode 100644 index 000000000..b79814fef --- /dev/null +++ b/e2e/framework/contracts/interface_rook_filesystem_client.go @@ -0,0 +1,11 @@ +package contracts + +type IRookFilesystem interface { + FSCreate(name string) (string, error) + FSDelete(name string) (string, error) + FSList() (string, error) + FSMount(name string, mountpath string) (string, error) + FSWrite(name string, mountpath string, data string, filename string, namespace string) (string, error) + FSRead(name string, mountpath string, filename string, namespace string) (string, error) + FSUnmount(mountpath string) (string, error) +} diff --git a/e2e/framework/contracts/interface_rook_object_client.go b/e2e/framework/contracts/interface_rook_object_client.go new file mode 100644 index 000000000..84ee7d249 --- /dev/null +++ b/e2e/framework/contracts/interface_rook_object_client.go @@ -0,0 +1,12 @@ +package contracts + +type IRookObject interface { + ObjectCreate() (string, error) + ObjectBucketList() (string, error) + ObjectConnection(uerid string) (string, error) + ObjectCreateUser(userid string, displayname string) (string, error) + ObjectDeleteUser(userid string) (string, error) + ObjectGetUser(userid string) (string, error) + ObjectListUser() (string, error) + ObjectUpdateUser(userid string, displayname string, emailid string) (string, error) +} diff --git a/e2e/framework/contracts/interface_rook_pool.go b/e2e/framework/contracts/interface_rook_pool.go new file mode 100644 index 000000000..c8d382ecb --- /dev/null +++ b/e2e/framework/contracts/interface_rook_pool.go @@ -0,0 +1,6 @@ +package contracts + +type IRookPool interface { + PoolList() (string, error) + PoolCreate() (string, error) +} diff --git a/e2e/framework/contracts/interface_transport_client.go b/e2e/framework/contracts/interface_transport_client.go new file mode 100644 index 000000000..11b75adeb --- /dev/null +++ b/e2e/framework/contracts/interface_transport_client.go @@ -0,0 +1,10 @@ +package contracts + +type ITransportClient interface { + Execute(cmd []string, optional []string) (stdout string, stderr string, exitCode int) + Create(cmd []string, optional []string) (stdout string, stderr string, exitCode int) + Delete(cmd []string, optional []string) (stdout string, stderr string, exitCode int) + ExecuteCmd(cmd []string) (stdout string, stderr string, err error) + Apply(cmd []string) (stdout string, stderr string, err error) + CreateWithStdin(stdinText string) (stdout string, stderr string, exitCode int) +} diff --git a/e2e/framework/enums/docker_sub_command.go b/e2e/framework/enums/docker_sub_command.go new file mode 100644 index 000000000..7372dc59c --- /dev/null +++ b/e2e/framework/enums/docker_sub_command.go @@ -0,0 +1,61 @@ +package enums + +import ( + "fmt" + "strings" +) + +type DockerSubCommand int + +const ( + Exec DockerSubCommand = iota + 1 + Run + Copy + Ps + Pull + Save + Kill + Rm + Empty +) + +var dockerSubCommands = [...]string{ + "exec", + "run", + "cp", + "ps", + "pull", + "save", + "kill", + "rm", + "Empty", +} + +func (subCommand DockerSubCommand) String() string { + return dockerSubCommands[subCommand-1] +} + +func GetDockerSubCommandFromString(name string) (DockerSubCommand, error) { + switch { + case strings.EqualFold(name, Exec.String()): + return Exec, nil + case strings.EqualFold(name, Run.String()): + return Run, nil + case strings.EqualFold(name, Copy.String()): + return Copy, nil + case strings.EqualFold(name, Ps.String()): + return Ps, nil + case strings.EqualFold(name, Pull.String()): + return Pull, nil + case strings.EqualFold(name, Save.String()): + return Save, nil + case strings.EqualFold(name, Kill.String()): + return Kill, nil + case strings.EqualFold(name, Rm.String()): + return Rm, nil + case strings.EqualFold(name, Empty.String()): + return Empty, nil + default: + return Empty, fmt.Errorf("Unsupported Docker sub-command") + } +} diff --git a/e2e/framework/enums/k8s_versions.go b/e2e/framework/enums/k8s_versions.go new file mode 100644 index 000000000..1d184f07b --- /dev/null +++ b/e2e/framework/enums/k8s_versions.go @@ -0,0 +1,32 @@ +package enums + +import ( + "fmt" + "strings" +) + +type K8sVersion int + +const ( + V1dot5 K8sVersion = iota + 1 + V1dot6 +) + +var versions = [...]string{ + "v1.5", + "v1.6", + "None", +} + +func (version K8sVersion) String() string { + return versions[version-1] +} + +func GetK8sVersionFromString(name string) (K8sVersion, error) { + switch { + case strings.EqualFold(name, V1dot6.String()): + return V1dot6, nil + default: + return V1dot5, fmt.Errorf("Unsupported Kubernetes version: " + name) + } +} diff --git a/e2e/framework/enums/rook_platform_types.go b/e2e/framework/enums/rook_platform_types.go new file mode 100644 index 000000000..c83f5d791 --- /dev/null +++ b/e2e/framework/enums/rook_platform_types.go @@ -0,0 +1,41 @@ +package enums + +import ( + "fmt" + "strings" +) + +type RookPlatformType int + +const ( + Kubernetes RookPlatformType = iota + 1 + BareMetal + StandAlone + None +) + +var platforms = [...]string{ + "Kubernetes", + "BareMetal", + "StandAlone", + "None", +} + +func (platform RookPlatformType) String() string { + return platforms[platform-1] +} + +func GetRookPlatFormTypeFromString(name string) (RookPlatformType, error) { + switch { + case strings.EqualFold(name, Kubernetes.String()): + return Kubernetes, nil + case strings.EqualFold(name, BareMetal.String()): + return BareMetal, nil + case strings.EqualFold(name, StandAlone.String()): + return StandAlone, nil + case strings.EqualFold(name, None.String()): + return None, nil + default: + return None, fmt.Errorf("Unsupported Rook Platform Type: " + name) + } +} diff --git a/e2e/framework/manager/rook_test_infra.go b/e2e/framework/manager/rook_test_infra.go new file mode 100644 index 000000000..f3afd32de --- /dev/null +++ b/e2e/framework/manager/rook_test_infra.go @@ -0,0 +1,407 @@ +package rook_test_infra + +import ( + "bytes" + "fmt" + "github.com/rook/rook/e2e/framework/contracts" + "github.com/rook/rook/e2e/framework/enums" + "github.com/rook/rook/e2e/framework/objects" + "github.com/rook/rook/e2e/framework/transport" + "github.com/rook/rook/e2e/framework/utils" + "io/ioutil" + "path" + "runtime" + "strings" + "sync" + "time" +) + +type rookTestInfraManager struct { + transportClient contracts.ITransportClient + platformType enums.RookPlatformType + dockerized bool + dockerContext *objects.DockerContext + k8sVersion enums.K8sVersion +} + +var ( + r *rookTestInfraManager + once sync.Once + curdir string +) + +const ( + temp_image_file_name = "temp_image.tar" + rookOperatorFileName = "rook-operator.yaml" + rookClusterFileName = "rook-cluster.yaml" + rookClientFileName = "rook-client.yml" + podSpecPath = "../../../demo/kubernetes" + scriptsPath = "scripts" + k8sFalsePostiveSuccessErrorMsg = "exit status 1" //When kubectl drain is executed, exit status 1 is always returned in stdout + rookDindK8sClusterScriptv1_5 = "rook-dind-cluster-v1.5.sh" + rookDindK8sClusterScriptv1_6 = "rook-dind-cluster-v1.6.sh" + rookOperatorImagePodSpecTag = "quay.io/rook/rookd:master-latest" + rookClientImagePodSpecTag = "quay.io/rook/rook-client:latest" + rookOperatorPrefix = "quay.io/rook/rookd" + rookClientPrefix = "quay.io/rook/rook-client" + rookOperatorCreatedTpr = "cluster.rook.io" + k8s_master_container_name = "kube-master" + k8s_node1_container_name = "kube-node-1" + k8s_node2_container_name = "kube-node-2" + cephBaseImageName = "ceph/base" +) + +func GetRookTestInfraManager(platformType enums.RookPlatformType, isDockerized bool, version enums.K8sVersion) (error, *rookTestInfraManager) { + var transportClient contracts.ITransportClient + var dockerContext objects.DockerContext + var dockerized bool = isDockerized + + //dont recreate singleton manager if it already exists + if r != nil { + return nil, r + } + + curdir = getCurrentDirectory() + + switch platformType { + case enums.Kubernetes: + transportClient = transport.CreateNewk8sTransportClient() + case enums.StandAlone: + return fmt.Errorf("Unsupported Rook Platform Type: " + platformType.String()), r + default: + return fmt.Errorf("Unsupported Rook Platform Type: " + platformType.String()), r + } + + //init singleton manager in a thread safe manner + once.Do(func() { + dockerEnv := []string{} + + if isDockerized { + dockerContext = objects.SetDockerContext(transport.CreateDockerClient(dockerEnv)) + } + + r = &rookTestInfraManager{ + platformType: platformType, + transportClient: transportClient, + dockerized: dockerized, + dockerContext: &dockerContext, + k8sVersion: version, + } + }) + + return nil, r +} + +func getCurrentDirectory() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + panic("No caller information") + } + + return path.Dir(filename) +} + +//Gets the platform the manager was configured with when created +func (r *rookTestInfraManager) GetRookPlatform() enums.RookPlatformType { + return r.platformType +} + +//Wrapper method for executing docker commands +//if method fails it will panic +func (r *rookTestInfraManager) executeDockerCommand(containerId string, subCommand enums.DockerSubCommand, args ...string) (stdOut, stdErr string) { + var cmdPrefix string = "" + var cmdArgs []string + + if subCommand != enums.Empty { + cmdPrefix = subCommand.String() + } + + if strings.EqualFold(containerId, "") { + containerId = r.dockerContext.Get_ContainerId() + } + + if subCommand == enums.Copy || subCommand == enums.Ps { + cmdArgs = append([]string{cmdPrefix}, args...) + } else { + cmdArgs = append([]string{cmdPrefix, containerId}, args...) + } + + fmt.Println("Executing the command 'docker " + strings.Join(cmdArgs, " ") + "'") + + stdOut, stdErr, err := r.dockerContext.Get_DockerClient().ExecuteCmd(cmdArgs) + + if err != nil { + panic(err) + } + + fmt.Println("Command succeeded...") + + return stdOut, stdErr +} + +//Get the k8s cluster creation script based on k8s version desired +func getDindScriptName(k8sVersion enums.K8sVersion) (dindScriptName string) { + switch k8sVersion { + case enums.V1dot5: + dindScriptName = rookDindK8sClusterScriptv1_5 + case enums.V1dot6: + dindScriptName = rookDindK8sClusterScriptv1_6 + default: + panic(fmt.Errorf("Unsupported Kubernetes version: " + k8sVersion.String())) + } + + return dindScriptName +} + +func (r *rookTestInfraManager) copyImageToNode(containerId string, imageName string) error { + fmt.Println("looking in local repository for the docker image: " + imageName) + cmdOut := utils.ExecuteCommand(objects.CommandArgs{Command: "docker", CmdArgs: []string{"images", "-q", imageName}}) + + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + fmt.Println("Image does not exist locally") + } + + if strings.EqualFold(cmdOut.StdOut, "") { + fmt.Println("searching remotely for the docker image: " + imageName) + cmdOut := utils.ExecuteCommand(objects.CommandArgs{Command: "docker", CmdArgs: []string{"pull", imageName}}) + + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + return fmt.Errorf("Failed to pull image") + } + fmt.Println("success...") + } + + fmt.Println("archiving the docker image: " + temp_image_file_name) + cmdOut = utils.ExecuteCommand(objects.CommandArgs{Command: "docker", CmdArgs: []string{"save", "-o", temp_image_file_name, imageName}}) + + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + return fmt.Errorf("Failed to archive image") + } + fmt.Println("success...") + + fmt.Println("copying archived image to node container") + _, _ = r.executeDockerCommand("", enums.Copy, temp_image_file_name, containerId+":/"+temp_image_file_name) + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + return fmt.Errorf("Failed to copy image to container") + } + fmt.Println("success...") + + fmt.Println("importing archived image into container registry") + _, _ = r.executeDockerCommand(containerId, enums.Exec, "/bin/bash", "-c", "docker load -i /"+temp_image_file_name) + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + return fmt.Errorf("Failed load image into containers repository") + } + fmt.Println("success...") + + return nil +} + +//This method stands up a k8s cluster and pre-configures it for running tests +func (r *rookTestInfraManager) ValidateAndSetupTestPlatform() { + fmt.Println("pulling the docker image: " + cephBaseImageName) + cmdOut := utils.ExecuteCommand(objects.CommandArgs{Command: "docker", CmdArgs: []string{"pull", cephBaseImageName}}) + + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + panic(fmt.Errorf("Failed to pull image")) + } + fmt.Println("success...") + + dindScriptName := getDindScriptName(r.k8sVersion) + + //make the k8s creation script executable + cmdOut = utils.ExecuteCommand(objects.CommandArgs{Command: "chmod", CmdArgs: []string{"+x", curdir + "/" + scriptsPath + "/" + dindScriptName}}) + + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + panic(fmt.Errorf("Failed to chmod cluster creation script")) + } + + //launch the k8s creation script + cmdOut = utils.ExecuteCommand(objects.CommandArgs{Command: curdir + "/" + scriptsPath + "/" + dindScriptName, CmdArgs: []string{"up"}}) + + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + panic(fmt.Errorf("Failed to execute cluster creation execute script")) + } + + //Untaint all nodes making them schedulable + k8sClient := transport.CreateNewk8sTransportClient() + + _, _, err := k8sClient.ExecuteCmd([]string{"taint", "nodes", "--all", "node-role.kubernetes.io/master--"}) + + if err != nil && !strings.EqualFold(err.Error(), k8sFalsePostiveSuccessErrorMsg) { + panic(err) + } + + //if any of these methods fail, the executeDockerCommand will panic + k8sMasterContainerId := r.getContainerIdByName(k8s_master_container_name) + k8sNode1ContainerId := r.getContainerIdByName(k8s_node1_container_name) + k8sNode2ContainerId := r.getContainerIdByName(k8s_node2_container_name) + + //install rbd shim + _, _ = r.executeDockerCommand("", enums.Copy, curdir+"/"+scriptsPath+"/rbd", k8sMasterContainerId+":/bin/rbd") + _, _ = r.executeDockerCommand(k8sMasterContainerId, enums.Exec, "chmod", "+x", "/bin/rbd") + + _, _ = r.executeDockerCommand("", enums.Copy, curdir+"/"+scriptsPath+"/rbd", k8sNode1ContainerId+":/bin/rbd") + _, _ = r.executeDockerCommand(k8sNode1ContainerId, enums.Exec, "chmod", "+x", "/bin/rbd") + + _, _ = r.executeDockerCommand("", enums.Copy, curdir+"/"+scriptsPath+"/rbd", k8sNode2ContainerId+":/bin/rbd") + _, _ = r.executeDockerCommand(k8sNode2ContainerId, enums.Exec, "chmod", "+x", "/bin/rbd") +} + +func (r *rookTestInfraManager) getContainerIdByName(containerName string) (containerId string) { + _, containerId = r.executeDockerCommand("", enums.Ps, "--filter", "name="+containerName, "--format", "{{.ID}}") + + return containerId +} + +//method for create rook-operator via kubectl +func createK8sRookOperator(k8sHelper *utils.K8sHelper, tag string) error { + raw, err := ioutil.ReadFile(podSpecPath + "/" + rookOperatorFileName) + + if err != nil { + return err + } + + rawUpdated := bytes.Replace(raw, []byte(rookOperatorImagePodSpecTag), []byte(tag), 1) + rookOperator := string(rawUpdated) + + fmt.Println("Creating rook-operator with tag of: " + tag) + + _, _, exitCode := r.transportClient.CreateWithStdin(rookOperator) + + if exitCode != 0 { + return fmt.Errorf(string("Failed to create rook-operator pod; kubectl exit code = " + string(exitCode))) + } + + fmt.Println() + if !k8sHelper.IsThirdPartyResourcePresent(rookOperatorCreatedTpr) { + return fmt.Errorf("Failed to start Rook Operator; k8s thirdpartyresource did not appear") + } else { + fmt.Println("Rook Operator started") + } + + return nil +} + +func createK8sRookClient(k8sHelper *utils.K8sHelper, tag string) (err error) { + + //Create rook client + raw, err := ioutil.ReadFile(podSpecPath + "/" + rookClientFileName) + + if err != nil { + panic(err) + } + + rookClient := string(raw) + + _, _, exitCode := r.transportClient.CreateWithStdin(rookClient) + + if exitCode != 0 { + return fmt.Errorf(string(exitCode)) + } + + if !k8sHelper.IsPodRunningInNamespace("rook-client") { + return fmt.Errorf("Rook Client couldn't start") + } else { + fmt.Println("Rook Client started") + } + + return nil +} + +func createk8sRookCluster(k8sHelper *utils.K8sHelper) error { + raw, err := ioutil.ReadFile(podSpecPath + "/" + rookClusterFileName) + + if err != nil { + return err + } + + rookCluster := string(raw) + + _, _, exitCode := r.transportClient.CreateWithStdin(rookCluster) + + if exitCode != 0 { + return fmt.Errorf("Failed to create rook-cluster pod; kubectl exit code = " + string(exitCode)) + } + + if !k8sHelper.IsServiceUpInNameSpace("rook-api") { + fmt.Println("Rook Cluster couldn't start") + } else { + fmt.Println("Rook Cluster started") + } + + return nil +} + +func (r *rookTestInfraManager) InstallRook(tag string) (err error, client contracts.IRookClient) { + rookOperatorTag := rookOperatorPrefix + ":" + tag + + err = r.copyImageToNode(r.getContainerIdByName(k8s_master_container_name), rookOperatorTag) + if err != nil { + panic(err) + } + + err = r.copyImageToNode(r.getContainerIdByName(k8s_node1_container_name), rookOperatorTag) + if err != nil { + panic(err) + } + + err = r.copyImageToNode(r.getContainerIdByName(k8s_node2_container_name), rookOperatorTag) + if err != nil { + panic(err) + } + + //Create rook operator + k8sHelp := utils.CreatK8sHelper() + + err = createK8sRookOperator(k8sHelp, rookOperatorTag) + + if err != nil { + panic(err) + } + + time.Sleep(10 * time.Second) ///TODO: add real check here + + //Create rook cluster + err = createk8sRookCluster(k8sHelp) + + if err != nil { + panic(err) + } + + time.Sleep(5 * time.Second) + + //Create rook client + err = createK8sRookClient(k8sHelp, rookClientImagePodSpecTag) + + if err != nil { + panic(err) + } + + return nil, nil +} + +func (r *rookTestInfraManager) isContainerRunning(containerId string) bool { + dockerClient := r.dockerContext.Get_DockerClient() + + _, stdErr, _ := dockerClient.ExecuteCmd([]string{"ps", "--filter", "status=running", "--filter", "id=" + containerId, "--format", "\"{{.ID}}\""}) + + return strings.EqualFold(stdErr, containerId) +} + +func (r rookTestInfraManager) TearDownInfrastructureCreatedEnvironment() error { + dindScriptName := getDindScriptName(r.k8sVersion) + + cmdOut := utils.ExecuteCommand(objects.CommandArgs{Command: "chmod", CmdArgs: []string{"+x", curdir + "/" + scriptsPath + "/" + dindScriptName}}) + + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + panic(fmt.Errorf("Failed to chmod script")) + } + + cmdOut = utils.ExecuteCommand(objects.CommandArgs{Command: curdir + "/" + scriptsPath + "/" + dindScriptName, CmdArgs: []string{"clean"}}) + + if cmdOut.Err != nil || cmdOut.ExitCode != 0 { + panic(fmt.Errorf("Failed to execute clean up script")) + } + + return nil +} diff --git a/e2e/framework/manager/scripts/rbd b/e2e/framework/manager/scripts/rbd new file mode 100644 index 000000000..576ed9e68 --- /dev/null +++ b/e2e/framework/manager/scripts/rbd @@ -0,0 +1,3 @@ +#!/bin/sh + +DOCKER_HOST=unix:///opt/outer-docker.sock /usr/bin/docker run --rm -v /sys:/sys --net=host --privileged=true ceph/base rbd "$@" diff --git a/e2e/framework/manager/scripts/rook-dind-cluster-v1.6.sh b/e2e/framework/manager/scripts/rook-dind-cluster-v1.6.sh new file mode 100755 index 000000000..6121833d5 --- /dev/null +++ b/e2e/framework/manager/scripts/rook-dind-cluster-v1.6.sh @@ -0,0 +1,1047 @@ +#!/bin/bash +# Copyright 2017 Mirantis +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +if [ $(uname) = Darwin ]; then + readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} +else + readlinkf(){ readlink -f "$1"; } +fi +DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" + +RUN_ON_BTRFS_ANYWAY="${RUN_ON_BTRFS_ANYWAY:-}" +if [[ ! ${RUN_ON_BTRFS_ANYWAY} ]] && docker info| grep -q '^Storage Driver: btrfs'; then + echo "ERROR: Docker is using btrfs storage driver which is unsupported by kubeadm-dind-cluster" >&2 + echo "Please refer to the documentation for more info." >&2 + echo "Set RUN_ON_BTRFS_ANYWAY to non-empty string to continue anyway." >&2 + exit 1 +fi + +# In case of moby linux, -v will not work so we can't +# mount /lib/modules and /boot +is_moby_linux= +if docker info|grep -q '^Kernel Version: .*-moby$'; then + is_moby_linux=1 +fi + +EMBEDDED_CONFIG=y;DIND_IMAGE=mirantis/kubeadm-dind-cluster:v1.6 + +if [[ ! ${EMBEDDED_CONFIG:-} ]]; then + source "${DIND_ROOT}/config.sh" +fi + +CNI_PLUGIN="${CNI_PLUGIN:-bridge}" +DIND_SUBNET="${DIND_SUBNET:-10.192.0.0}" +dind_ip_base="$(echo "${DIND_SUBNET}" | sed 's/\.0$//')" +DIND_IMAGE="${DIND_IMAGE:-}" +BUILD_KUBEADM="${BUILD_KUBEADM:-}" +BUILD_HYPERKUBE="${BUILD_HYPERKUBE:-}" +APISERVER_PORT=${APISERVER_PORT:-8080} +NUM_NODES=${NUM_NODES:-2} +LOCAL_KUBECTL_VERSION=${LOCAL_KUBECTL_VERSION:-} +KUBECTL_DIR="${KUBECTL_DIR:-${HOME}/.kubeadm-dind-cluster}" +DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml}" +E2E_REPORT_DIR="${E2E_REPORT_DIR:-}" + +if [[ ! ${LOCAL_KUBECTL_VERSION:-} && ${DIND_IMAGE:-} =~ :(v[0-9]+\.[0-9]+)$ ]]; then + LOCAL_KUBECTL_VERSION="${BASH_REMATCH[1]}" +fi + +function dind::need-source { + if [[ ! -f cluster/kubectl.sh ]]; then + echo "$0 must be called from the Kubernetes repository root directory" 1>&2 + exit 1 + fi +} + +build_tools_dir="build" +use_k8s_source=y +if [[ ! ${BUILD_KUBEADM} && ! ${BUILD_HYPERKUBE} ]]; then + use_k8s_source= +fi +if [[ ${use_k8s_source} ]]; then + dind::need-source + kubectl=cluster/kubectl.sh + if [[ ! -f ${build_tools_dir}/common.sh ]]; then + build_tools_dir="build-tools" + fi +else + if [[ ! ${LOCAL_KUBECTL_VERSION:-} ]] && ! hash kubectl 2>/dev/null; then + echo "You need kubectl binary in your PATH to use prebuilt DIND image" 1>&2 + exit 1 + fi + kubectl=kubectl +fi + +busybox_image="busybox:1.26.2" +e2e_base_image="golang:1.7.1" +sys_volume_args=() +build_volume_args=() + +function dind::set-build-volume-args { + if [ ${#build_volume_args[@]} -gt 0 ]; then + return 0 + fi + build_container_name= + if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then + build_volume_args=(-v "$PWD:/go/src/k8s.io/kubernetes") + else + build_container_name="$(KUBE_ROOT=$PWD && + . ${build_tools_dir}/common.sh && + kube::build::verify_prereqs >&2 && + echo "${KUBE_DATA_CONTAINER_NAME:-${KUBE_BUILD_DATA_CONTAINER_NAME}}")" + build_volume_args=(--volumes-from "${build_container_name}") + fi +} + +function dind::volume-exists { + local name="$1" + if docker volume inspect "${name}" >& /dev/null; then + return 0 + fi + return 1 +} + +function dind::create-volume { + local name="$1" + docker volume create --label mirantis.kubeadm_dind_cluster --name "${name}" >/dev/null +} + +# We mount /boot and /lib/modules into the container +# below to in case some of the workloads need them. +# This includes virtlet, for instance. Also this may be +# useful in future if we want DIND nodes to pass +# preflight checks. +# Unfortunately we can't do this when using Mac Docker +# (unless a remote docker daemon on Linux is used) +# NB: there's no /boot on recent Mac dockers +function dind::prepare-sys-mounts { + if [[ ! ${is_moby_linux} ]]; then + sys_volume_args=() + if [[ -d /boot ]]; then + sys_volume_args+=(-v /boot:/boot) + fi + if [[ -d /lib/modules ]]; then + sys_volume_args+=(-v /lib/modules:/lib/modules) + fi + if [[ -d /dev ]]; then + sys_volume_args+=(-v /dev:/dev) + fi + if [[ -d /sys ]]; then + sys_volume_args+=(-v /sys/bus:/sys/bus) + fi + if [[ -d /var/run/docker.sock ]]; then + sys_volume_args+=(-v /var/run/docker.sock:/var/run/docker.sock) + fi + return 0 + fi + if ! dind::volume-exists kubeadm-dind-sys; then + dind::step "Saving a copy of docker host's /lib/modules" + dind::create-volume kubeadm-dind-sys + # Use a dirty nsenter trick to fool Docker on Mac and grab system + # /lib/modules into sys.tar file on kubeadm-dind-sys volume. + local nsenter="nsenter --mount=/proc/1/ns/mnt --" + docker run \ + --rm \ + --privileged \ + -v kubeadm-dind-sys:/dest \ + --pid=host \ + "${busybox_image}" \ + /bin/sh -c \ + "if ${nsenter} test -d /lib/modules; then ${nsenter} tar -C / -c lib/modules >/dest/sys.tar; fi" + fi + sys_volume_args=(-v kubeadm-dind-sys:/dind-sys) +} + +tmp_containers=() + +function dind::cleanup { + if [ ${#tmp_containers[@]} -gt 0 ]; then + for name in "${tmp_containers[@]}"; do + docker rm -vf "${name}" 2>/dev/null + done + fi +} + +trap dind::cleanup EXIT + +function dind::check-image { + local name="$1" + if docker inspect --format 'x' "${name}" >&/dev/null; then + return 0 + else + return 1 + fi +} + +function dind::filter-make-output { + # these messages make output too long and make Travis CI choke + egrep -v --line-buffered 'I[0-9][0-9][0-9][0-9] .*(parse|conversion|defaulter|deepcopy)\.go:[0-9]+\]' +} + +function dind::make-for-linux { + local copy="$1" + shift + dind::step "Building binaries:" "$*" + if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then + dind::step "+ make WHAT=\"$*\"" + make WHAT="$*" 2>&1 | dind::filter-make-output + elif [ "${copy}" = "y" ]; then + dind::step "+ ${build_tools_dir}/run.sh make WHAT=\"$*\"" + "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output + else + dind::step "+ KUBE_RUN_COPY_OUTPUT=n ${build_tools_dir}/run.sh make WHAT=\"$*\"" + KUBE_RUN_COPY_OUTPUT=n "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output + fi +} + +function dind::check-binary { + local filename="$1" + local dockerized="_output/dockerized/bin/linux/amd64/${filename}" + local plain="_output/local/bin/linux/amd64/${filename}" + dind::set-build-volume-args + # FIXME: don't hardcode amd64 arch + if [ -n "${KUBEADM_DIND_LOCAL:-${force_local:-}}" ]; then + if [ -f "${dockerized}" -o -f "${plain}" ]; then + return 0 + fi + elif docker run --rm "${build_volume_args[@]}" \ + "${busybox_image}" \ + test -f "/go/src/k8s.io/kubernetes/${dockerized}" >&/dev/null; then + return 0 + fi + return 1 +} + +function dind::ensure-downloaded-kubectl { + local kubectl_url + local kubectl_sha1 + local kubectl_sha1_linux + local kubectl_sha1_darwin + local kubectl_link + local kubectl_os + local full_kubectl_version + + case "${LOCAL_KUBECTL_VERSION}" in + v1.4) + full_kubectl_version=v1.4.9 + kubectl_sha1_linux=5726e8f17d56a5efeb2a644d8e7e2fdd8da8b8fd + kubectl_sha1_darwin=630fb3e0f4f442178cde0264d4b399291226eb2b + ;; + v1.5) + full_kubectl_version=v1.5.4 + kubectl_sha1_linux=15d8430dc52b1f3772b88bc6a236c8fa58e07c0d + kubectl_sha1_darwin=5e671ba792567574eea48be4eddd844ba2f07c27 + ;; + v1.6) + full_kubectl_version=v1.6.1 + kubectl_sha1_linux=65d27d731fbd25aa6beb6049691d58c5f09848c0 + kubectl_sha1_darwin=f12a74544a19d3480466cc6046e18cb7c04591e1 + ;; + "") + return 0 + ;; + *) + echo "Invalid kubectl version" >&2 + exit 1 + esac + + export PATH="${KUBECTL_DIR}:$PATH" + + if [ $(uname) = Darwin ]; then + kubectl_sha1="${kubectl_sha1_darwin}" + kubectl_os=darwin + else + kubectl_sha1="${kubectl_sha1_linux}" + kubectl_os=linux + fi + local link_target="kubectl-${full_kubectl_version}" + local link_name="${KUBECTL_DIR}"/kubectl + if [[ -h "${link_name}" && "$(readlink "${link_name}")" = "${link_target}" ]]; then + return 0 + fi + + local path="${KUBECTL_DIR}/${link_target}" + if [[ ! -f "${path}" ]]; then + mkdir -p "${KUBECTL_DIR}" + wget -O "${path}" "https://storage.googleapis.com/kubernetes-release/release/${full_kubectl_version}/bin/${kubectl_os}/amd64/kubectl" + echo "${kubectl_sha1} ${path}" | sha1sum -c + chmod +x "${path}" + fi + + ln -fs "${link_target}" "${KUBECTL_DIR}/kubectl" +} + +function dind::ensure-kubectl { + if [[ ! ${use_k8s_source} ]]; then + # already checked on startup + dind::ensure-downloaded-kubectl + return 0 + fi + if [ $(uname) = Darwin ]; then + if [ ! -f _output/local/bin/darwin/amd64/kubectl ]; then + dind::step "Building kubectl" + dind::step "+ make WHAT=cmd/kubectl" + make WHAT=cmd/kubectl 2>&1 | dind::filter-make-output + fi + elif ! force_local=y dind::check-binary kubectl; then + dind::make-for-linux y cmd/kubectl + fi +} + +function dind::ensure-binaries { + local -a to_build=() + for name in "$@"; do + if ! dind::check-binary "$(basename "${name}")"; then + to_build+=("${name}") + fi + done + if [ "${#to_build[@]}" -gt 0 ]; then + dind::make-for-linux n "${to_build[@]}" + fi + return 0 +} + +function dind::ensure-network { + if ! docker network inspect kubeadm-dind-net >&/dev/null; then + docker network create --subnet=10.192.0.0/16 kubeadm-dind-net >/dev/null + fi +} + +function dind::ensure-volume { + local reuse_volume= + if [[ $1 = -r ]]; then + reuse_volume=1 + shift + fi + local name="$1" + if dind::volume-exists "${name}"; then + if [[ ! {reuse_volume} ]]; then + docker volume rm "${name}" >/dev/null + fi + elif [[ ${reuse_volume} ]]; then + echo "*** Failed to locate volume: ${name}" 1>&2 + return 1 + fi + dind::create-volume "${name}" +} + +function dind::run { + local reuse_volume= + if [[ $1 = -r ]]; then + reuse_volume="-r" + shift + fi + local container_name="${1:-}" + local ip="${2:-}" + local netshift="${3:-}" + local portforward="${4:-}" + if [[ $# -gt 4 ]]; then + shift 4 + else + shift $# + fi + local -a opts=(--ip "${ip}" "$@") + local -a args=("systemd.setenv=CNI_PLUGIN=${CNI_PLUGIN}") + + if [[ ! "${container_name}" ]]; then + echo >&2 "Must specify container name" + exit 1 + fi + + # remove any previously created containers with the same name + docker rm -vf "${container_name}" >&/dev/null || true + + if [[ "$portforward" ]]; then + opts+=(-p "$portforward") + fi + + if [[ ${CNI_PLUGIN} = bridge && ${netshift} ]]; then + args+=("systemd.setenv=CNI_BRIDGE_NETWORK_OFFSET=0.0.${netshift}.0") + fi + + opts+=(${sys_volume_args[@]+"${sys_volume_args[@]}"}) + + dind::step "Starting DIND container:" "${container_name}" + + if [[ ! ${is_moby_linux} ]]; then + opts+=(-v /boot:/boot -v /lib/modules:/lib/modules) + fi + + volume_name="kubeadm-dind-${container_name}" + dind::ensure-network + dind::ensure-volume ${reuse_volume} "${volume_name}" + + # TODO: create named volume for binaries and mount it to /k8s + # in case of the source build + + # Start the new container. + docker run \ + -d --privileged \ + --net kubeadm-dind-net \ + --name "${container_name}" \ + --hostname "${container_name}" \ + -l mirantis.kubeadm_dind_cluster \ + -v ${volume_name}:/dind \ + -v /dev:/dev \ + -v /var/run/docker.sock:/opt/outer-docker.sock \ + ${opts[@]+"${opts[@]}"} \ + "${DIND_IMAGE}" \ + ${args[@]+"${args[@]}"} +} + +function dind::kubeadm { + local container_id="$1" + shift + dind::step "Running kubeadm:" "$*" + status=0 + # See image/bare/wrapkubeadm. + # Capturing output is necessary to grab flags for 'kubeadm join' + if ! docker exec "${container_id}" wrapkubeadm "$@" 2>&1 | tee /dev/fd/2; then + echo "*** kubeadm failed" >&2 + return 1 + fi + return ${status} +} + +# function dind::bare { +# local container_name="${1:-}" +# if [[ ! "${container_name}" ]]; then +# echo >&2 "Must specify container name" +# exit 1 +# fi +# shift +# run_opts=(${@+"$@"}) +# dind::run "${container_name}" +# } + +function dind::configure-kubectl { + dind::step "Setting cluster config" + "${kubectl}" config set-cluster dind --server="http://localhost:${APISERVER_PORT}" --insecure-skip-tls-verify=true + "${kubectl}" config set-context dind --cluster=dind + "${kubectl}" config use-context dind +} + +force_make_binaries= +function dind::set-master-opts { + master_opts=() + if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then + # share binaries pulled from the build container between nodes + dind::ensure-volume "dind-k8s-binaries" + dind::set-build-volume-args + master_opts+=("${build_volume_args[@]}" -v dind-k8s-binaries:/k8s) + local -a bins + if [[ ${BUILD_KUBEADM} ]]; then + master_opts+=(-e KUBEADM_SOURCE=build://) + bins+=(cmd/kubeadm) + fi + if [[ ${BUILD_HYPERKUBE} ]]; then + master_opts+=(-e HYPERKUBE_SOURCE=build://) + bins+=(cmd/hyperkube) + fi + if [[ ${force_make_binaries} ]]; then + dind::make-for-linux n "${bins[@]}" + else + dind::ensure-binaries "${bins[@]}" + fi + fi +} + +cached_use_rbac= +function dind::use-rbac { + # we use rbac in case of k8s 1.6 + if [[ ${cached_use_rbac} ]]; then + [[ ${cached_use_rbac} = 1 ]] && return 0 || return 1 + fi + cached_use_rbac=0 + if "${kubectl}" version --short >& /dev/null && ! "${kubectl}" version --short | grep -q 'Server Version: v1\.5\.'; then + cached_use_rbac=1 + return 0 + fi + return 1 +} + +function dind::deploy-dashboard { + dind::step "Deploying k8s dashboard" + "${kubectl}" create -f "${DASHBOARD_URL}" + if dind::use-rbac; then + # https://kubernetes-io-vnext-staging.netlify.com/docs/admin/authorization/rbac/#service-account-permissions + # Thanks @liggitt for the hint + "${kubectl}" create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default + fi +} + +function dind::init { + local -a opts + dind::set-master-opts + local master_ip="${dind_ip_base}.2" + local container_id=$(dind::run kube-master "${master_ip}" 1 127.0.0.1:${APISERVER_PORT}:8080 ${master_opts[@]+"${master_opts[@]}"}) + # FIXME: I tried using custom tokens with 'kubeadm ex token create' but join failed with: + # 'failed to parse response as JWS object [square/go-jose: compact JWS format must have three parts]' + # So we just pick the line from 'kubeadm init' output + kubeadm_join_flags="$(dind::kubeadm "${container_id}" init --pod-network-cidr=10.244.0.0/16 --skip-preflight-checks "$@" | grep '^ *kubeadm join' | sed 's/^ *kubeadm join //')" + dind::configure-kubectl + dind::deploy-dashboard +} + +function dind::create-node-container { + local reuse_volume= + if [[ $1 = -r ]]; then + reuse_volume="-r" + shift + fi + # if there's just one node currently, it's master, thus we need to use + # kube-node-1 hostname, if there are two nodes, we should pick + # kube-node-2 and so on + local next_node_index=${1:-$(docker ps -q --filter=label=mirantis.kubeadm_dind_cluster | wc -l | sed 's/^ *//g')} + local node_ip="${dind_ip_base}.$((next_node_index + 2))" + local -a opts + if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then + opts+=(-v dind-k8s-binaries:/k8s) + if [[ ${BUILD_KUBEADM} ]]; then + opts+=(-e KUBEADM_SOURCE=build://) + fi + if [[ ${BUILD_HYPERKUBE} ]]; then + opts+=(-e HYPERKUBE_SOURCE=build://) + fi + fi + dind::run ${reuse_volume} kube-node-${next_node_index} ${node_ip} $((next_node_index + 1)) "" ${opts[@]+"${opts[@]}"} +} + +function dind::join { + local container_id="$1" + shift + dind::kubeadm "${container_id}" join --skip-preflight-checks "$@" >/dev/null +} + +function dind::escape-e2e-name { + sed 's/[]\$*.^|()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n' +} + +function dind::accelerate-kube-dns { + dind::step "Patching kube-dns deployment to make it start faster" + # Could do this on the host, too, but we don't want to require jq here + # TODO: do this in wrapkubeadm + # 'kubectl version --short' is a quick check for kubectl 1.4 + # which doesn't support 'kubectl apply --force' + docker exec kube-master /bin/bash -c \ + "kubectl get deployment kube-dns -n kube-system -o json | jq '.spec.template.spec.containers[0].readinessProbe.initialDelaySeconds = 3|.spec.template.spec.containers[0].readinessProbe.periodSeconds = 3' | if kubectl version --short >&/dev/null; then kubectl apply --force -f -; else kubectl apply -f -; fi" +} + +function dind::component-ready { + local label="$1" + local out + if ! out="$("${kubectl}" get pod -l "${label}" -n kube-system \ + -o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then + return 1 + fi + if ! grep -v False <<<"${out}" | grep -q True; then + return 1 + fi + return 0 +} + +function dind::kill-failed-pods { + local pods + # workaround for https://github.com/kubernetes/kubernetes/issues/36482 + if ! pods="$(kubectl get pod -n kube-system -o jsonpath='{ .items[?(@.status.phase == "Failed")].metadata.name }' 2>/dev/null)"; then + return + fi + for name in ${pods}; do + kubectl delete pod --now -n kube-system "${name}" >&/dev/null || true + done +} + +function dind::wait-for-ready { + dind::step "Waiting for kube-proxy and the nodes" + local proxy_ready + local nodes_ready + local n=3 + while true; do + dind::kill-failed-pods + if "${kubectl}" get nodes 2>/dev/null| grep -q NotReady; then + nodes_ready= + else + nodes_ready=y + fi + if dind::component-ready k8s-app=kube-proxy; then + proxy_ready=y + else + proxy_ready= + fi + if [[ ${nodes_ready} && ${proxy_ready} ]]; then + if ((--n == 0)); then + echo "[done]" >&2 + break + fi + else + n=3 + fi + echo -n "." >&2 + sleep 1 + done + + dind::step "Bringing up kube-dns and kubernetes-dashboard" + "${kubectl}" scale deployment --replicas=1 -n kube-system kube-dns + "${kubectl}" scale deployment --replicas=1 -n kube-system kubernetes-dashboard + + while ! dind::component-ready k8s-app=kube-dns || ! dind::component-ready app=kubernetes-dashboard; do + echo -n "." >&2 + dind::kill-failed-pods + sleep 1 + done + echo "[done]" >&2 + + "${kubectl}" get nodes >&2 + dind::step "Access dashboard at:" "http://localhost:${APISERVER_PORT}/ui" +} + +function dind::up { + dind::down + dind::init + local master_ip="$(docker inspect --format="{{.NetworkSettings.IPAddress}}" kube-master)" + # pre-create node containers sequentially so they get predictable IPs + local -a node_containers + for ((n=1; n <= NUM_NODES; n++)); do + dind::step "Starting node container:" ${n} + if ! container_id="$(dind::create-node-container ${n})"; then + echo >&2 "*** Failed to start node container ${n}" + exit 1 + else + node_containers+=(${container_id}) + dind::step "Node container started:" ${n} + fi + done + status=0 + local -a pids + for ((n=1; n <= NUM_NODES; n++)); do + ( + dind::step "Joining node:" ${n} + container_id="${node_containers[n-1]}" + if ! dind::join ${container_id} ${kubeadm_join_flags}; then + echo >&2 "*** Failed to start node container ${n}" + exit 1 + else + dind::step "Node joined:" ${n} + fi + )& + pids[${n}]=$! + done + for pid in ${pids[*]}; do + wait ${pid} + done + case "${CNI_PLUGIN}" in + bridge) + ;; + flannel) + if dind::use-rbac; then + curl -sSL "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml?raw=true" | "${kubectl}" create -f - + # FIXME: current kube-flannel yaml causes problems after cluster restart + # kube-dns and kubernetes-dashboard stay in "ContainerCreating" state. + # Flannel pods become 'Running' for a brief amount of time and then + # die while trying to contact apiserver. + # Following the advice from https://github.com/kubernetes/kubernetes/issues/39701 (comments) + # we hardcode apiserver host into flannel pod definition + # (KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT) + # We could use 'kubectl convert' + 'jq' but we don't want + # to require jq on the host, and doing this via docker is cumbersome, + # so for now we're embedding patched flannel yaml here. + # For some reason the problem happens only on 1.6. + "${kubectl}" create -f - <&2 + ;; + esac + dind::accelerate-kube-dns + if [[ ${CNI_PLUGIN} != bridge ]]; then + # This is especially important in case of Calico - + # the cluster will not recover after snapshotting + # (at least not after restarting from the snapshot) + # if Calico installation is interrupted + dind::wait-for-ready + fi +} + +function dind::snapshot_container { + local container_name="$1" + docker exec -i ${container_name} /usr/local/bin/snapshot prepare + docker diff ${container_name} | docker exec -i ${container_name} /usr/local/bin/snapshot save +} + +function dind::snapshot { + dind::step "Taking snapshot of the cluster" + dind::snapshot_container kube-master + for ((n=1; n <= NUM_NODES; n++)); do + dind::snapshot_container "kube-node-${n}" + done + dind::wait-for-ready +} + +restore_cmd=restore +function dind::restore_container { + local container_id="$1" + docker exec ${container_id} /usr/local/bin/snapshot "${restore_cmd}" +} + +function dind::restore { + local master_ip="${dind_ip_base}.2" + dind::down + dind::step "Restoring master container" + dind::set-master-opts + for ((n=0; n <= NUM_NODES; n++)); do + ( + if [[ n -eq 0 ]]; then + dind::step "Restoring master container" + dind::restore_container "$(dind::run -r kube-master "${master_ip}" 1 127.0.0.1:${APISERVER_PORT}:8080 ${master_opts[@]+"${master_opts[@]}"})" + dind::step "Master container restored" + else + dind::step "Restoring node container:" ${n} + if ! container_id="$(dind::create-node-container -r ${n})"; then + echo >&2 "*** Failed to start node container ${n}" + exit 1 + else + dind::restore_container "${container_id}" + dind::step "Node container restored:" ${n} + fi + fi + )& + pids[${n}]=$! + done + for pid in ${pids[*]}; do + wait ${pid} + done + # Recheck kubectl config. It's possible that the cluster was started + # on this docker from different host + dind::configure-kubectl + dind::wait-for-ready +} + +function dind::down { + docker ps -a -q --filter=label=mirantis.kubeadm_dind_cluster | while read container_id; do + dind::step "Removing container:" "${container_id}" + docker rm -fv "${container_id}" + done +} + +function dind::remove-volumes { + # docker 1.13+: docker volume ls -q -f label=mirantis.kubeadm_dind_cluster + docker volume ls -q | (grep '^kubeadm-dind' || true) | while read volume_id; do + dind::step "Removing volume:" "${volume_id}" + docker volume rm "${volume_id}" + done +} + +function dind::check-for-snapshot { + if ! dind::volume-exists "kubeadm-dind-kube-master"; then + return 1 + fi + for ((n=1; n <= NUM_NODES; n++)); do + if ! dind::volume-exists "kubeadm-dind-kube-node-${n}"; then + return 1 + fi + done +} + +function dind::do-run-e2e { + local parallel="${1:-}" + local focus="${2:-}" + local skip="${3:-}" + dind::need-source + local test_args="--host=http://localhost:${APISERVER_PORT}" + local -a e2e_volume_opts=() + local term= + if [[ ${focus} ]]; then + test_args="--ginkgo.focus=${focus} ${test_args}" + fi + if [[ ${skip} ]]; then + test_args="--ginkgo.skip=${skip} ${test_args}" + fi + if [[ ${E2E_REPORT_DIR} ]]; then + test_args="--report-dir=/report ${test_args}" + e2e_volume_opts=(-v "${E2E_REPORT_DIR}:/report") + fi + dind::ensure-binaries cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo + dind::step "Running e2e tests with args:" "${test_args}" + dind::set-build-volume-args + if [ -t 1 ] ; then + term="-it" + test_args="--ginkgo.noColor ${test_args}" + fi + docker run \ + --rm ${term} \ + --net=host \ + "${build_volume_args[@]}" \ + -e KUBERNETES_PROVIDER=dind \ + -e KUBE_MASTER_IP=http://localhost:${APISERVER_PORT} \ + -e KUBE_MASTER=local \ + -e KUBERNETES_CONFORMANCE_TEST=y \ + -e GINKGO_PARALLEL=${parallel} \ + ${e2e_volume_opts[@]+"${e2e_volume_opts[@]}"} \ + -w /go/src/k8s.io/kubernetes \ + "${e2e_base_image}" \ + bash -c "cluster/kubectl.sh config set-cluster dind --server='http://localhost:${APISERVER_PORT}' --insecure-skip-tls-verify=true && + cluster/kubectl.sh config set-context dind --cluster=dind && + cluster/kubectl.sh config use-context dind && + go run hack/e2e.go --v --test -check_version_skew=false --test_args='${test_args}'" +} + +function dind::clean { + dind::down + # dind::remove-images + dind::remove-volumes + if docker network inspect kubeadm-dind-net >&/dev/null; then + docker network rm kubeadm-dind-net + fi +} + +function dind::run-e2e { + local focus="${1:-}" + local skip="${2:-\[Serial\]}" + if [[ "$focus" ]]; then + focus="$(dind::escape-e2e-name "${focus}")" + else + focus="\[Conformance\]" + fi + dind::do-run-e2e y "${focus}" "${skip}" +} + +function dind::run-e2e-serial { + local focus="${1:-}" + local skip="${2:-}" + dind::need-source + if [[ "$focus" ]]; then + focus="$(dind::escape-e2e-name "${focus}")" + else + focus="\[Serial\].*\[Conformance\]" + fi + dind::do-run-e2e n "${focus}" "${skip}" + # TBD: specify filter +} + +function dind::step { + local OPTS="" + if [ "$1" = "-n" ]; then + shift + OPTS+="-n" + fi + GREEN="$1" + shift + if [ -t 2 ] ; then + echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2 + else + echo ${OPTS} "* ${GREEN} $*" 1>&2 + fi +} + +case "${1:-}" in + up) + if [[ ! ( ${DIND_IMAGE} =~ local ) ]]; then + dind::step "Making sure DIND image is up to date" + docker pull "${DIND_IMAGE}" >&2 + fi + + dind::prepare-sys-mounts + dind::ensure-kubectl + if ! dind::check-for-snapshot; then + force_make_binaries=y dind::up + dind::snapshot + else + dind::restore + fi + ;; + reup) + dind::prepare-sys-mounts + dind::ensure-kubectl + if ! dind::check-for-snapshot; then + force_make_binaries=y dind::up + dind::snapshot + else + force_make_binaries=y + restore_cmd=update_and_restore + dind::restore + fi + ;; + down) + dind::down + ;; + init) + shift + dind::prepare-sys-mounts + dind::ensure-kubectl + dind::init "$@" + ;; + join) + shift + dind::prepare-sys-mounts + dind::ensure-kubectl + dind::join "$(dind::create-node-container)" "$@" + ;; + # bare) + # shift + # dind::bare "$@" + # ;; + snapshot) + shift + dind::snapshot + ;; + restore) + shift + dind::restore + ;; + clean) + dind::clean + ;; + e2e) + shift + dind::run-e2e "$@" + ;; + e2e-serial) + shift + dind::run-e2e-serial "$@" + ;; + *) + echo "usage:" >&2 + echo " $0 up" >&2 + echo " $0 reup" >&2 + echo " $0 down" >&2 + echo " $0 init kubeadm-args..." >&2 + echo " $0 join kubeadm-args..." >&2 + # echo " $0 bare container_name [docker_options...]" + echo " $0 clean" + echo " $0 e2e [test-name-substring]" >&2 + echo " $0 e2e-serial [test-name-substring]" >&2 + exit 1 + ;; +esac \ No newline at end of file diff --git a/e2e/framework/objects/command_arg.go b/e2e/framework/objects/command_arg.go new file mode 100644 index 000000000..d1c44da6e --- /dev/null +++ b/e2e/framework/objects/command_arg.go @@ -0,0 +1,17 @@ +package objects + +type CommandArgs struct { + Command string + SubCommand string + CmdArgs []string + OptionalArgs []string + PipeToStdIn string + EnvironmentVariable []string +} + +type CommandOut struct { + StdOut string + StdErr string + ExitCode int + Err error +} diff --git a/e2e/framework/objects/docker_context.go b/e2e/framework/objects/docker_context.go new file mode 100644 index 000000000..ed520959d --- /dev/null +++ b/e2e/framework/objects/docker_context.go @@ -0,0 +1,26 @@ +package objects + +import "github.com/rook/rook/e2e/framework/contracts" + +type DockerContext struct { + containerId string + dockerClient contracts.IDockerClient +} + +func SetDockerContext(dockerClient contracts.IDockerClient) DockerContext { + return DockerContext{dockerClient: dockerClient} +} + +func (d *DockerContext) Get_ContainerId() string { + return d.containerId +} + +func (d *DockerContext) Set_ContainerId(containerId string) string { + d.containerId = containerId + + return containerId +} + +func (d *DockerContext) Get_DockerClient() contracts.IDockerClient { + return d.dockerClient +} diff --git a/e2e/framework/objects/environment_manifest.go b/e2e/framework/objects/environment_manifest.go new file mode 100644 index 000000000..7f565b91e --- /dev/null +++ b/e2e/framework/objects/environment_manifest.go @@ -0,0 +1,23 @@ +package objects + +import ( + "flag" +) + +type EnvironmentManifest struct { + K8sVersion string + Platform string + RookTag string +} + +func NewManifest() EnvironmentManifest { + e := EnvironmentManifest{} + + flag.StringVar(&e.K8sVersion, "k8s_version", "v1.6", "Version of kubernetes to test rook in; v1.5 | v1.6 are the only version of kubernetes currently supported") + flag.StringVar(&e.Platform, "rook_platform", "Kubernetes", "Platform to install rook on; Kubernetes is the only platform currently supported") + flag.StringVar(&e.RookTag, "rook_version", "quay.io/rook/rookd:master-latest", "Docker tag of the rook operator to install, must be in quay.io or local environment") + + flag.Parse() + + return e +} diff --git a/e2e/framework/transport/docker_transport_client.go b/e2e/framework/transport/docker_transport_client.go new file mode 100644 index 000000000..aa0a2d4b5 --- /dev/null +++ b/e2e/framework/transport/docker_transport_client.go @@ -0,0 +1,40 @@ +package transport + +import ( + "github.com/rook/rook/e2e/framework/utils" +) + +type dockerClient struct { + env []string +} + +const docker_executable = "docker" + +func CreateDockerClient(dockerEnv []string) *dockerClient { + return &dockerClient{env: dockerEnv} +} + +func (k *dockerClient) Execute(cmdArgs []string) (stdout string, stderr string, err error) { + initialArgs := []string{"exec"} + cmdArgs = append(initialArgs, cmdArgs...) + + return utils.ExecuteCmdAndLogToConsole(docker_executable, cmdArgs, k.env) +} + +func (k *dockerClient) Run(cmdArgs []string) (stdout string, stderr string, err error) { + initialArgs := []string{"run"} + cmdArgs = append(initialArgs, cmdArgs...) + + return utils.ExecuteCmdAndLogToConsole(docker_executable, cmdArgs, k.env) +} + +func (k *dockerClient) Stop(cmdArgs []string) (stdout string, stderr string, err error) { + initialArgs := []string{"stop"} + cmdArgs = append(initialArgs, cmdArgs...) + + return utils.ExecuteCmdAndLogToConsole(docker_executable, cmdArgs, k.env) +} + +func (k *dockerClient) ExecuteCmd(cmdArgs []string) (stdout string, stderr string, err error) { + return utils.ExecuteCmdAndLogToConsole(docker_executable, cmdArgs, k.env) +} diff --git a/e2e/framework/transport/k8s_transport_client.go b/e2e/framework/transport/k8s_transport_client.go new file mode 100644 index 000000000..6f9d3d7e1 --- /dev/null +++ b/e2e/framework/transport/k8s_transport_client.go @@ -0,0 +1,69 @@ +package transport + +import ( + "github.com/rook/rook/e2e/framework/objects" + "github.com/rook/rook/e2e/framework/utils" +) + +type k8sTransportClient struct { +} + +func CreateNewk8sTransportClient() *k8sTransportClient { + return &k8sTransportClient{} +} + +const ( + kubectl_executable = "kubectl" +) + +func (k *k8sTransportClient) ExecuteCmd(cmd []string) (stdout string, stderr string, err error) { + return utils.ExecuteCmdAndLogToConsole(kubectl_executable, cmd, []string{}) + +} + +func (k *k8sTransportClient) Apply(cmdArgs []string) (stdout string, stderr string, err error) { + initialArgs := []string{"replace", "--force", "-f"} + cmdArgs = append(initialArgs, cmdArgs...) + return utils.ExecuteCmdAndLogToConsole("kubectl", cmdArgs, []string{}) +} + +func (k *k8sTransportClient) Execute(cmdArgs []string, optional []string) (stdout string, stderr string, exitCode int) { + + if optional != nil { + if len(optional) == 1 { + initialArgs := []string{"exec", optional[0], "--"} + cmdArgs = append(initialArgs, cmdArgs...) + } else if len(optional) == 2 { + initialArgs := []string{"exec", "-n", optional[1], optional[0], "--"} + cmdArgs = append(initialArgs, cmdArgs...) + } else { + return "", "invalid number of optional params used", 1 + } + } else { + initialArgs := []string{"exec", "-n", "rook", "rook-client", "--"} + cmdArgs = append(initialArgs, cmdArgs...) + } + + return utils.ExecuteCmd(kubectl_executable, cmdArgs) +} + +func (k *k8sTransportClient) CreateWithStdin(stdinText string) (stdout string, stderr string, exitCode int) { + cmdStruct := objects.CommandArgs{Command: kubectl_executable, PipeToStdIn: stdinText, CmdArgs: []string{"create", "-f", "-"}} + + cmdOut := utils.ExecuteCommand(cmdStruct) + + return cmdOut.StdOut, cmdOut.StdErr, cmdOut.ExitCode +} + +func (k *k8sTransportClient) Create(cmdArgs []string, optional []string) (stdout string, stderr string, exitCode int) { + initialArgs := []string{"create", "-f"} + cmdArgs = append(initialArgs, cmdArgs...) + return utils.ExecuteCmd(kubectl_executable, cmdArgs) +} + +func (k *k8sTransportClient) Delete(cmdArgs []string, optional []string) (stdout string, stderr string, exitCode int) { + + initialArgs := []string{"delete", "-f"} + cmdArgs = append(initialArgs, cmdArgs...) + return utils.ExecuteCmd(kubectl_executable, cmdArgs) +} diff --git a/e2e/framework/utils/exec_utils.go b/e2e/framework/utils/exec_utils.go new file mode 100644 index 000000000..64b669e64 --- /dev/null +++ b/e2e/framework/utils/exec_utils.go @@ -0,0 +1,209 @@ +package utils + +import ( + "bufio" + "bytes" + "fmt" + "github.com/rook/rook/e2e/framework/objects" + "log" + "os/exec" + "strings" + "syscall" +) + +func ExecuteCommand(cmdStruct objects.CommandArgs) objects.CommandOut { + var outBuffer, errBuffer bytes.Buffer + + cmd := exec.Command(cmdStruct.Command, cmdStruct.CmdArgs...) + + cmd.Env = append(cmd.Env, cmdStruct.EnvironmentVariable...) + + stdOut, err := cmd.StdoutPipe() + + if err != nil { + return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} + } + + stdin, err := cmd.StdinPipe() + + if err != nil { + return objects.CommandOut{Err: err} + } + + defer stdOut.Close() + + scanner := bufio.NewScanner(stdOut) + go func() { + for scanner.Scan() { + outBuffer.WriteString(scanner.Text()) + fmt.Printf("%s\n", scanner.Text()) + } + }() + + stdErr, err := cmd.StderrPipe() + + if err != nil { + return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} + } + + defer stdErr.Close() + + stdErrScanner := bufio.NewScanner(stdErr) + go func() { + for stdErrScanner.Scan() { + + txt := stdErrScanner.Text() + + if !strings.Contains(txt, "no buildable Go source files in") { + errBuffer.WriteString(txt) + fmt.Printf("%s\n", txt) + } + } + }() + + err = cmd.Start() + if err != nil { + return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} + } + + if cmdStruct.PipeToStdIn != "" { + stdin.Write([]byte(cmdStruct.PipeToStdIn)) + stdin.Close() + } + + if err := cmd.Wait(); err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + log.Printf("Exit Status: %d", status.ExitStatus()) + return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: exiterr} + } + } else { + return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: nil} + } + } + + return objects.CommandOut{StdErr: errBuffer.String(), StdOut: outBuffer.String(), Err: err} +} + +func ExecuteCmdWithEnv(Cmd string, cmdArgs []string, env []string) (stdout string, stderr string, exitCode int) { + + var outbuf, errbuf bytes.Buffer + cmd := exec.Command(Cmd, strings.Join(cmdArgs, " ")) + + cmd.Env = append(cmd.Env, env...) + + cmd.Stdout = &outbuf + cmd.Stderr = &errbuf + + err := cmd.Run() + stdout = outbuf.String() + stderr = errbuf.String() + + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + ws := exitError.Sys().(syscall.WaitStatus) + exitCode = ws.ExitStatus() + } else { + exitCode = 1 + if stderr == "" { + stderr = err.Error() + stdout + } + } + } else { + ws := cmd.ProcessState.Sys().(syscall.WaitStatus) + exitCode = ws.ExitStatus() + } + return +} + +func ExecuteCmd(Cmd string, cmdArgs []string) (stdout string, stderr string, exitCode int) { + + var outbuf, errbuf bytes.Buffer + cmd := exec.Command(Cmd, cmdArgs...) + cmd.Stdout = &outbuf + cmd.Stderr = &errbuf + + err := cmd.Run() + stdout = outbuf.String() + stderr = errbuf.String() + + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + ws := exitError.Sys().(syscall.WaitStatus) + exitCode = ws.ExitStatus() + } else { + exitCode = 1 + if stderr == "" { + stderr = err.Error() + stdout + } + } + } else { + ws := cmd.ProcessState.Sys().(syscall.WaitStatus) + exitCode = ws.ExitStatus() + } + return +} + +//TODO add timeout parameter +func ExecuteCmdAndLogToConsole(command string, cmdArgs []string, cmdEnv []string) (stdout string, stderr string, err error) { + var outbuf, errbuf bytes.Buffer + + cmd := exec.Command(command, cmdArgs...) + + cmd.Env = append(cmd.Env, cmdEnv...) + + stdOut, err := cmd.StdoutPipe() + if err != nil { + + return errbuf.String(), outbuf.String(), err + } + + defer stdOut.Close() + + scanner := bufio.NewScanner(stdOut) + go func() { + for scanner.Scan() { + outbuf.WriteString(scanner.Text()) + fmt.Printf("%s\n", scanner.Text()) + } + }() + + stdErr, err := cmd.StderrPipe() + if err != nil { + return errbuf.String(), outbuf.String(), err + } + + defer stdErr.Close() + + stdErrScanner := bufio.NewScanner(stdErr) + go func() { + for stdErrScanner.Scan() { + + txt := stdErrScanner.Text() + + errbuf.WriteString(txt) + fmt.Printf("%s\n", txt) + } + }() + + err = cmd.Start() + if err != nil { + + return errbuf.String(), outbuf.String(), err + } + + if err := cmd.Wait(); err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + log.Printf("Exit Status: %d", status.ExitStatus()) + return errbuf.String(), outbuf.String(), exiterr + } + } else { + return errbuf.String(), outbuf.String(), nil + } + } + + return errbuf.String(), outbuf.String(), err +} diff --git a/e2e/framework/utils/k8s_helper.go b/e2e/framework/utils/k8s_helper.go new file mode 100644 index 000000000..3c6817657 --- /dev/null +++ b/e2e/framework/utils/k8s_helper.go @@ -0,0 +1,253 @@ +package utils + +import ( + "encoding/json" + "fmt" + "github.com/jmoiron/jsonq" + "html/template" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +type K8sHelper struct { +} + +func CreatK8sHelper() *K8sHelper { + return &K8sHelper{} +} + +func (k8sh *K8sHelper) getMonIp(mon string) (string, error) { + //kubectl -n rook get pod mon0 -o json|jq ".status.podIP"| + cmdArgs := []string{"-n", "rook", "get", "pod", mon, "-o", "json"} + out, err, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + data := map[string]interface{}{} + dec := json.NewDecoder(strings.NewReader(out)) + dec.Decode(&data) + jq := jsonq.NewQuery(data) + ip, _ := jq.String("status", "podIP") + return ip + ":6790", nil + } else { + return err, fmt.Errorf("Error Getting Monitor IP") + } +} + +func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, poddefPath string) (string, error) { + + t, _ := template.ParseFiles(poddefPath) + mons := k8sh.getMonitorPods() + ip1, _ := k8sh.getMonIp(mons[0]) + ip2, _ := k8sh.getMonIp(mons[1]) + ip3, _ := k8sh.getMonIp(mons[2]) + + config := map[string]string{ + "mon0": ip1, + "mon1": ip2, + "mon2": ip3, + } + file, _ := ioutil.TempFile(os.TempDir(), "prefix") + t.Execute(file, config) + dir, _ := filepath.Abs(file.Name()) + + cmdArgs := []string{action, "-f", dir} + out, err, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + return out, nil + } else { + return out + " : " + err, fmt.Errorf("Could Not create resource in k8s") + } +} + +func (k8sh *K8sHelper) ResourceOperation(action string, poddefPath string) (string, error) { + + cmdArgs := []string{action, "-f", poddefPath} + out, err, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + return out, nil + } else { + return out + " : " + err, fmt.Errorf("Could Not create resource in k8s") + } +} + +func (k8sh *K8sHelper) CleanUpDymaincCreatedPVC(blockList map[string]blockListData) { + + for _, v := range blockList { + cmdArgs := []string{"exec", "-n", "rook", "rook-client", "--", "rook", "block", "delete", + "--name", v.name, "--pool-name", v.pool} + ExecuteCmd("kubectl", cmdArgs) + } +} + +func (k8sh *K8sHelper) getMonitorPods() []string { + mons := []string{} + monIdx := 0 + moncount := 0 + + for moncount < 3 { + m := "mon" + strconv.Itoa(monIdx) + cmdArgs := []string{"-n", "rook", "get", "pods", m} + _, _, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + mons = append(mons, m) + moncount++ + } + monIdx++ + } + + return mons +} + +func (k8sh *K8sHelper) IsPodRunning(name string) bool { + cmdArgs := []string{"get", "pod", name} + inc := 0 + for inc < 20 { + out, _, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + lines := strings.Split(out, "\n") + if len(lines) == 3 { + lines = lines[1 : len(lines)-1] + bktsrawdata := strings.Split(lines[0], " ") + var r []string + for _, str := range bktsrawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + } + if r[2] == "Running" { + return true + } + } + } + time.Sleep(5 * time.Second) + inc++ + + } + return false +} +func (k8sh *K8sHelper) IsPodRunningInNamespace(name string) bool { + cmdArgs := []string{"get", "pods", "-n", "rook", name} + inc := 0 + for inc < 20 { + out, _, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + lines := strings.Split(out, "\n") + if len(lines) == 3 { + lines = lines[1 : len(lines)-1] + bktsrawdata := strings.Split(lines[0], " ") + var r []string + for _, str := range bktsrawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + } + if r[2] == "Running" { + return true + } + } + } + time.Sleep(5 * time.Second) + inc++ + + } + return false +} + +//TODO: Combine these like functions +func (k8sh *K8sHelper) IsPodTerminated(name string) bool { + cmdArgs := []string{"get", "pods", name} + inc := 0 + for inc < 20 { + _, _, status := ExecuteCmd("kubectl", cmdArgs) + if status != 0 { + fmt.Println("Pod in default namespace terminated: " + name) + return true + } + time.Sleep(5 * time.Second) + inc++ + + } + fmt.Println("Pod in default namespace did not terminated: " + name) + return false +} + +//TODO: Combine these like functions +func (k8sh *K8sHelper) IsPodTerminatedInNamespace(name string) bool { + cmdArgs := []string{"get", "-n", "rook", "pods", name} + inc := 0 + for inc < 20 { + _, _, status := ExecuteCmd("kubectl", cmdArgs) + if status != 0 { + fmt.Println("Pod in rook namespace terminated: " + name) + return true + } + time.Sleep(5 * time.Second) + inc++ + + } + fmt.Println("Pod in rook namespace did not terminated: " + name) + return false +} + +func (k8sh *K8sHelper) IsServiceUp(name string) bool { + cmdArgs := []string{"get", "svc", name} + inc := 0 + for inc < 20 { + _, _, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + fmt.Println("Service in default namespace is up: " + name) + return true + } + time.Sleep(2 * time.Second) + inc++ + + } + fmt.Println("Service in default namespace is not up: " + name) + return false +} + +func (k8sh *K8sHelper) IsServiceUpInNameSpace(name string) bool { + + cmdArgs := []string{"get", "svc", "-n", "rook", name} + inc := 0 + for inc < 20 { + _, _, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + return true + } + time.Sleep(5 * time.Second) + inc++ + + } + fmt.Println("Service in rook namespace is not up: " + name) + return false +} + +func (k8sh *K8sHelper) GetService(servicename string) (string, error) { + cmdArgs := []string{"get", "svc", "-n", "rook", servicename} + sout, serr, status := ExecuteCmd("kubectl", cmdArgs) + if status != 0 { + return serr, fmt.Errorf("Cannot find service") + } + return sout, nil +} + +func (k8sh *K8sHelper) IsThirdPartyResourcePresent(tprname string) bool { + + cmdArgs := []string{"get", "thirdpartyresources", tprname} + inc := 0 + for inc < 20 { + _, _, exitCode := ExecuteCmd("kubectl", cmdArgs) + if exitCode == 0 { + fmt.Println("Found the thirdparty resource: " + tprname) + return true + } + time.Sleep(5 * time.Second) + inc++ + } + + return false +} diff --git a/e2e/framework/utils/rook_helper.go b/e2e/framework/utils/rook_helper.go new file mode 100644 index 000000000..76f65d6a0 --- /dev/null +++ b/e2e/framework/utils/rook_helper.go @@ -0,0 +1,246 @@ +package utils + +import ( + "fmt" + "strconv" + "strings" +) + +type RookHelper struct { +} + +type blockListData struct { + name string + pool string + size string + device string + mount string +} + +type fileSystemListData struct { + Name string + metadatapool string + datapool string +} + +type ObjectUserListData struct { + UserId string + DisplayName string + Email string +} + +type ObjectUserData struct { + UserId string + DisplayName string + Email string + AccessKey string + SecretKey string +} + +type ObjectConnectionData struct { + AwsHost string + AwsEndpoint string + AwsAccessKey string + AwsSecretKey string +} + +type ObjectBucketListData struct { + Name string + Owner string + Created string + Size int + NumberOfObjects int +} + +func CreateRookHelper() *RookHelper { + return &RookHelper{} +} + +func (rh *RookHelper) ParseBlockListData(rawdata string) map[string]blockListData { + data := make(map[string]blockListData) + + lines := strings.Split(rawdata, "\n") + if len(lines) <= 1 { + return data + } + lines = lines[1 : len(lines)-1] + if len(lines) >= 1 { + for line := range lines { + blockrawdata := strings.Split(lines[line], " ") + var r []string + for _, str := range blockrawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + + } + for len(r) < 5 { + r = append(r, "") + } + data[r[0]] = blockListData{r[0], r[1], r[2], r[3], r[4]} + } + } + return data +} + +func (rh *RookHelper) ParseFileSystemData(rawdata string) fileSystemListData { + lines := strings.Split(rawdata, "\n") + lines = lines[1 : len(lines)-1] + if len(lines) != 1 { + return fileSystemListData{"ERROR OCCURED", "", ""} + } + filerawdata := strings.Split(lines[0], " ") + var r []string + for _, str := range filerawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + + } + return fileSystemListData{r[0], r[1], r[2]} +} + +func (rh *RookHelper) ParserObjectUserListData(rawdata string) map[string]ObjectUserListData { + data := make(map[string]ObjectUserListData) + lines := strings.Split(rawdata, "\n") + if len(lines) <= 1 { + return data + } + lines = lines[1 : len(lines)-1] + if len(lines) >= 1 { + for line := range lines { + usersrawdata := strings.Split(lines[line], " ") + var r []string + for _, str := range usersrawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + + } + for len(r) < 3 { + r = append(r, "") + } + data[r[0]] = ObjectUserListData{r[0], r[1], r[2]} + } + + } + //TODO - + return data +} + +func (rh *RookHelper) ParserObjectUserData(rawdata string) ObjectUserData { + lines := strings.Split(rawdata, "\n") + lines = lines[:len(lines)-1] + var ( + userId string + displayName string + email string + accesskey string + secretkey string + ) + if len(lines) != 5 { + return ObjectUserData{UserId: "USER NOT FOUND"} + } else { + for line := range lines { + userrawdata := strings.Split(lines[line], ":") + switch userrawdata[0] { + case "User ID": + userId = strings.TrimSpace(userrawdata[1]) + case "Display Name": + displayName = strings.TrimSpace(userrawdata[1]) + case "Email": + email = strings.TrimSpace(userrawdata[1]) + case "Access Key": + accesskey = strings.TrimSpace(userrawdata[1]) + case "Secret Key": + secretkey = strings.TrimSpace(userrawdata[1]) + } + + } + } + return ObjectUserData{userId, displayName, email, accesskey, secretkey} +} + +func (rh *RookHelper) ParserObjectConnectionData(rawdata string) ObjectConnectionData { + lines := strings.Split(rawdata, "\n") + lines = lines[1 : len(lines)-1] + var ( + awshost string + endpoint string + accesskey string + secretkey string + ) + if len(lines) != 4 { + return ObjectConnectionData{AwsHost: "CONNECTION INFO NOT FOUND FOR GIVEN USERID"} + } else { + for line := range lines { + connrawdata := strings.Split(lines[line], " ") + var r []string + for _, str := range connrawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + } + switch r[0] { + case "AWS_HOST": + awshost = r[1] + case "AWS_ENDPOINT": + endpoint = r[1] + case "AWS_ACCESS_KEY_ID": + accesskey = r[1] + case "AWS_SECRET_ACCESS_KEY": + secretkey = r[1] + + } + } + } + return ObjectConnectionData{awshost, endpoint, accesskey, secretkey} +} + +func (rh *RookHelper) ParserObjectBucketListData(rawdata string) map[string]ObjectBucketListData { + data := make(map[string]ObjectBucketListData) + lines := strings.Split(rawdata, "\n") + if len(lines) <= 1 { + return data + } + lines = lines[1 : len(lines)-1] + if len(lines) >= 1 { + for line := range lines { + bktsrawdata := strings.Split(lines[line], " ") + var r []string + for _, str := range bktsrawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + + } + size, _ := strconv.Atoi(r[3]) + objs, _ := strconv.Atoi(r[4]) + data[r[0]] = ObjectBucketListData{r[0], r[1], r[2], size, objs} + } + } + return data +} + +func (rh *RookHelper) GetRgwServiceNodePort(rawdata string) (string, error) { + lines := strings.Split(rawdata, "\n") + if len(lines) <= 1 { + return "NOT FOUND", fmt.Errorf("Port not found") + } + lines = lines[1 : len(lines)-1] + if len(lines) == 1 { + bktsrawdata := strings.Split(lines[0], " ") + var r []string + for _, str := range bktsrawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + } + ports := strings.Split(r[3], ":") + port := strings.Replace(ports[1], "/TCP", "", -1) + return port, nil + + } else { + return "NOT FOUND", fmt.Errorf("Port not found") + } +} diff --git a/e2e/framework/utils/s3_operations.go b/e2e/framework/utils/s3_operations.go new file mode 100644 index 000000000..570265884 --- /dev/null +++ b/e2e/framework/utils/s3_operations.go @@ -0,0 +1,92 @@ +package utils + +import ( + "bytes" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "strings" +) + +type S3Helper struct { + s3client *s3.S3 +} + +func CreateNewS3Helper(endpoint string, keyId string, keySecret string) *S3Helper { + creds := credentials.NewStaticCredentials(keyId, keySecret, "") + _, err := creds.Get() + if err != nil { + fmt.Printf("bad credentials: %s", err) + } + cfg := aws.NewConfig().WithEndpoint(endpoint).WithRegion("us-west-1").WithCredentials(creds).WithDisableSSL(true) + + svc := s3.New(session.New(), cfg) + + return &S3Helper{svc} +} + +func (h *S3Helper) CreateBucket(name string) (bool, error) { + _, err := h.s3client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(name), + }) + if err != nil { + return false, err + + } + return true, nil +} + +func (h *S3Helper) DeleteBucket(name string) (bool, error) { + _, err := h.s3client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(name), + }) + if err != nil { + return false, err + + } + return true, nil +} + +func (h *S3Helper) PutObjectInBucket(bucketname string, body string, key string, + contentType string) (bool, error) { + _, err := h.s3client.PutObject(&s3.PutObjectInput{ + Body: strings.NewReader(body), + Bucket: &bucketname, + Key: &key, + ContentType: &contentType, + }) + if err != nil { + return false, err + + } + return true, nil +} + +func (h *S3Helper) GetObjectInBucket(bucketname string, key string) (string, error) { + result, err := h.s3client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucketname), + Key: aws.String(key), + }) + + if err != nil { + return "ERROR_ OBJCET NOT FOUND", err + + } + buf := new(bytes.Buffer) + buf.ReadFrom(result.Body) + + return buf.String(), nil +} +func (h *S3Helper) DeleteObjectInBucket(bucketname string, key string) (bool, error) { + _, err := h.s3client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(bucketname), + Key: aws.String(key), + }) + if err != nil { + return false, err + + } + return true, nil +} diff --git a/e2e/glide.lock b/e2e/glide.lock new file mode 100644 index 000000000..4b8ec1d4e --- /dev/null +++ b/e2e/glide.lock @@ -0,0 +1,50 @@ +hash: a292e770129d63132a567e4b88aa141d4c8803f8079f1cfcd4a8168e4153f67b +updated: 2017-04-19T20:52:43.827227707-07:00 +imports: +- name: github.com/aws/aws-sdk-go + version: 3981b48d0eca40d1ce35f20c605b3a153e64b4ad + subpackages: + - aws + - aws/awserr + - aws/awsutil + - aws/client + - aws/client/metadata + - aws/corehandlers + - aws/credentials + - aws/credentials/ec2rolecreds + - aws/credentials/endpointcreds + - aws/credentials/stscreds + - aws/defaults + - aws/ec2metadata + - aws/endpoints + - aws/request + - aws/session + - aws/signer/v4 + - private/protocol + - private/protocol/query + - private/protocol/query/queryutil + - private/protocol/rest + - private/protocol/restxml + - private/protocol/xml/xmlutil + - service/s3 + - service/sts +- name: github.com/go-ini/ini + version: e7fea39b01aea8d5671f6858f0532f56e8bff3a5 +- name: github.com/jmespath/go-jmespath + version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d +- name: github.com/jmoiron/jsonq + version: e874b168d07ecc7808bc950a17998a8aa3141d82 +testImports: +- name: github.com/davecgh/go-spew + version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 + subpackages: + - spew +- name: github.com/pmezard/go-difflib + version: d8ed2627bdf02c080bf22230dbb337003b7aba2d + subpackages: + - difflib +- name: github.com/stretchr/testify + version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 + subpackages: + - assert + - require diff --git a/e2e/glide.yaml b/e2e/glide.yaml new file mode 100644 index 000000000..893e23f57 --- /dev/null +++ b/e2e/glide.yaml @@ -0,0 +1,16 @@ +package: github.com/rook/rook/e2e +import: +- package: github.com/aws/aws-sdk-go + version: ^1.8.13 + subpackages: + - aws + - aws/credentials + - aws/session + - service/s3 +- package: github.com/jmoiron/jsonq +testImport: +- package: github.com/stretchr/testify + version: ^1.1.4 + subpackages: + - assert + - require diff --git a/e2e/images/Dockerfile b/e2e/images/Dockerfile new file mode 100644 index 000000000..46355ad7a --- /dev/null +++ b/e2e/images/Dockerfile @@ -0,0 +1,69 @@ +FROM ubuntu:16.04 +ENV container docker +ENV LC_ALL C +ENV DEBIAN_FRONTEND noninteractive + +# Install common dependencies +RUN apt-get -y update && apt-get install -qqyf \ + ca-certificates \ + apt-transport-https \ + curl \ + wget \ + git + +# Don't start any optional services except for the few we need. +RUN find /etc/systemd/system \ + /lib/systemd/system \ + -path '*.wants/*' \ + -not -name '*journald*' \ + -not -name '*systemd-tmpfiles*' \ + -not -name '*systemd-user-sessions*' \ + -exec rm \{} \; + +RUN systemctl set-default multi-user.target + + +#***************************************** +#Install k8s +#***************************************** +#Step 1 - Install kubeadm & dependencies +RUN curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - &&\ + echo deb http://apt.kubernetes.io/ kubernetes-xenial main >> /etc/apt/sources.list.d/kubernetes.list &&\ + apt-get update -qq && apt-get install -qqy \ + kubelet \ + kubectl \ + kubernetes-cni \ + docker.io +#***************************************** + + +#***************************************** +#Install go and configure +#***************************************** +RUN apt-get install -qqy golang &&\ + mkdir -p /workspace/go/bin &&\ + echo "export GOPATH=/workspace/go" >> ~/.bashrc &&\ + echo "export GOROOT=/usr/lib/go" >> ~/.bashrc &&\ + echo "export GOBIN=/workspace/go/bin" >> ~/.bashrc &&\ + echo "PATH=$PATH:$GOBIN:$GOPATH:$GOROOT" >> ~/.bashrc +#***************************************** + +ENV GOPATH /workspace/go +ENV GOROOT /usr/lib/go +ENV GOBIN /workspace/go/bin +ENV PATH $PATH:$GOBIN:$GOPATH:$GOROOT + +#Install glide +RUN curl https://glide.sh/get | sh + + +VOLUME /sys/fs/cgroup +VOLUME /lib/modules +VOLUME /sys + +RUN apt-get clean -y + +STOPSIGNAL SIGRTMIN+3 + +CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=journal 3>&1"] +#CMD /sbin/init diff --git a/e2e/images/README.md b/e2e/images/README.md new file mode 100644 index 000000000..2b7b0d9d5 --- /dev/null +++ b/e2e/images/README.md @@ -0,0 +1,33 @@ +# Rook Test Infrastructure + +The Rook Test Infrastructure is a docker project, +pre-configured with Kubernetes and Docker (DinD) software running +as systemd services on an Ubuntu 16.04 image. The goal of this environment +is to make the process of bringing up a k8s cluster, installing Rook +and running Rook related tests easier and isolated. + +## Requirements + +1. Docker version => 1.2 && < 17.0 + +## Installation + +Execute the following command to start the environment. + + docker run \ + --net=host \ + -d \ + --privileged \ + --security-opt=seccomp:unconfined \ + quay.io/quantum/rook-test \ + /sbin/init + +## Usage + +### Installing Kubernetes +After the container is running you can docker exec into the container +and use the standard Kubernetes means to bring up a Kubernetes cluster. + +### Installing Rook +You can follow the instructions at here +to install the Rook project. diff --git a/e2e/scripts/smoke_test.sh b/e2e/scripts/smoke_test.sh new file mode 100755 index 000000000..7bc07e4c2 --- /dev/null +++ b/e2e/scripts/smoke_test.sh @@ -0,0 +1,112 @@ +#!/bin/bash -e + +test_source_repo=$(pwd) +docker_test_repo=/workspace/go/src/github.com/rook/rook +git_smoke_test_directory=github.com/rook/rook/e2e/tests/smoke +container_image=quay.io/quantum/rook-test +tmp_docker_sock_path=/tmp/docker.sock +results_dir=results +results_filename=test.log + +#create the rook infrastructure container +rook_infra::create() { + export id=$(docker run \ + --net=host \ + -d \ + -e GOPATH=/workspace/go \ + -e GOROOT=/usr/lib/go \ + --privileged \ + --security-opt=seccomp:unconfined \ + -v /var/run/docker.sock:${tmp_docker_sock_path} \ + -v ${test_source_repo}:${docker_test_repo} \ + -w ${docker_test_repo}/e2e \ + ${container_image} \ + /sbin/init) + + rc=$?; if [[ $rc != 0 ]]; then set -e; fi +} + +#prepare the rook_infra container to run tests +rook_infra::init() { + echo Creating results directory + docker exec ${id} mkdir -p ${results_dir} + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + + echo Removing infra docker.sock + docker exec ${id} rm -rfv /var/run/docker.sock + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + + echo Creating sysmlink to host docker.sock + docker exec ${id} ln -s ${tmp_docker_sock_path} /var/run/docker.sock + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + echo Success... + + echo Installing rook-test-framework dependencies... + docker exec ${id} /bin/bash -c \ + "glide install" + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + echo Success... +} + +rook_infra::run_test() { + local test_name_regex=$1 + local tag_name=$2 + local rook_platform=$3 + local k8s_version=$4 + + docker exec -t ${id} /bin/bash -c \ + "go test -timeout 1200s -run ${test_name_regex} ${git_smoke_test_directory} --rook_platform=${rook_platform} --k8s_version=${k8s_version} --rook_version=${tag_name} -v 2>&1 | tee ${results_dir}/${results_filename}" + + rc=$?; if [[ $rc != 0 ]]; then set -e; fi +} + +rook_infra::gather_results() { + echo Gathering results... + + #install junit parser && create junit results + docker exec ${id} /bin/bash -c \ + "go get -u -f github.com/jstemmer/go-junit-report && cat ${results_dir}/test.log | go-junit-report > ${results_dir}/junit.xml" + + rc=$?; if [[ $rc != 0 ]]; then set -e; fi +} + +rook_infra::cleanup() { + echo Removing rook-test-framework container... + docker kill ${id} + docker rm ${id} +} + +{ #try + + if [ -z "$1" ]; then + tag_name="master-latest" + else + tag_name=$1 + fi + + if [ -z "$2" ]; then + rook_platform="Kubernetes" + else + rook_platform=$2 + fi + + if [ -z "$3" ]; then + k8s_version="v1.6" + else + k8s_version=$3 + fi + + rook_infra::create + sleep 5 + rook_infra::init + + rook_infra::run_test SmokeSuite ${tag_name} ${rook_platform} ${k8s_version} + + rook_infra::gather_results + +} || { #catch + rook_infra::gather_results +} + +#Delete the rook infrastructure container +rook_infra::cleanup diff --git a/e2e/tests/smoke/block_storage_smoke_test.go b/e2e/tests/smoke/block_storage_smoke_test.go new file mode 100644 index 000000000..25c15aedb --- /dev/null +++ b/e2e/tests/smoke/block_storage_smoke_test.go @@ -0,0 +1,114 @@ +package smoke + +import ( + "github.com/rook/rook/e2e/framework/enums" + "github.com/rook/rook/e2e/framework/manager" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "testing" +) + +func TestBlockStorageSmokeSuite(t *testing.T) { + suite.Run(t, new(BlockStorageTestSuite)) +} + +type BlockStorageTestSuite struct { + suite.Suite + rookPlatform enums.RookPlatformType + k8sVersion enums.K8sVersion + rookTag string +} + +func (suite *BlockStorageTestSuite) TearDownSuite() { + err, rookInfra := rook_test_infra.GetRookTestInfraManager(suite.rookPlatform, true, suite.k8sVersion) + + require.Nil(suite.T(), err) + + rookInfra.TearDownInfrastructureCreatedEnvironment() +} + +func (suite *BlockStorageTestSuite) SetupTest() { + var err error + + suite.rookPlatform, err = enums.GetRookPlatFormTypeFromString(env.Platform) + + require.Nil(suite.T(), err) + + suite.k8sVersion, err = enums.GetK8sVersionFromString(env.K8sVersion) + + require.Nil(suite.T(), err) + + suite.rookTag = env.RookTag + + require.NotEmpty(suite.T(), suite.rookTag, "RookTag parameter is required") +} + +func (suite *BlockStorageTestSuite) TestBlockStorage_SmokeTest() { + var err error + + err, rookInfra := rook_test_infra.GetRookTestInfraManager(suite.rookPlatform, true, suite.k8sVersion) + + require.Nil(suite.T(), err) + + rookInfra.ValidateAndSetupTestPlatform() + + err, _ = rookInfra.InstallRook(suite.rookTag) + + require.Nil(suite.T(), err) + + suite.T().Log("Block Storage Smoke Test - Create,Mount,write to, read from and Unmount Block") + sc, _ := CreateSmokeTestClient(rookInfra.GetRookPlatform()) + defer blockTestcleanup() + rh := sc.rookHelp + rbc := sc.GetBlockClient() + suite.T().Log("Step 0 : Get Initial List Block") + rawlistInit, _ := rbc.BlockList() + initblocklistMap := rh.ParseBlockListData(rawlistInit) + + suite.T().Log("step 1: Create block storage") + _, cb_err := sc.CreateBlockStorage() + require.Nil(suite.T(), cb_err) + rawlistAfterCreate, _ := rbc.BlockList() + blocklistMapAfterBlockCreate := rh.ParseBlockListData(rawlistAfterCreate) + require.Empty(suite.T(), len(initblocklistMap), len(blocklistMapAfterBlockCreate)+1, "Make sure a new block is created") + suite.T().Log("Block Storage created successfully") + + suite.T().Log("step 2: Mount block storage") + _, mt_err := sc.MountBlockStorage() + require.Nil(suite.T(), mt_err) + suite.T().Log("Block Storage Mounted successfully") + + suite.T().Log("step 3: Write to block storage") + _, wt_err := sc.WriteToBlockStorage("Test Data", "testFile1") + require.Nil(suite.T(), wt_err) + suite.T().Log("Write to Block storage successfully") + + suite.T().Log("step 4: Read from block storage") + read, r_err := sc.ReadFromBlockStorage("testFile1") + require.Nil(suite.T(), r_err) + require.Contains(suite.T(), read, "Test Data", "make sure content of the files is unchanged") + suite.T().Log("Read from Block storage successfully") + + suite.T().Log("step 5: Unmount block storage") + _, unmt_err := sc.UnMountBlockStorage() + require.Nil(suite.T(), unmt_err) + suite.T().Log("Block Storage unmounted successfully") + + suite.T().Log("step 6: Deleting block storage") + _, db_err := sc.DeleteBlockStorage() + require.Nil(suite.T(), db_err) + rawlistAfterDelete, _ := rbc.BlockList() + blocklistMapAfterBlockDelete := rh.ParseBlockListData(rawlistAfterDelete) + //This is a stop gap, block storage is not deleted when pods are deleted + sc.CleanUpDymanicBlockStorge() + require.Empty(suite.T(), len(initblocklistMap), len(blocklistMapAfterBlockDelete), "Make sure a new block is created") + suite.T().Log("Block Storage deleted successfully") + +} + +func blockTestcleanup() { + sc, _ := CreateSmokeTestClient(enums.Kubernetes) + sc.UnMountBlockStorage() + sc.DeleteBlockStorage() + sc.CleanUpDymanicBlockStorge() +} diff --git a/e2e/tests/smoke/file_storage_smoke_test.go b/e2e/tests/smoke/file_storage_smoke_test.go new file mode 100644 index 000000000..e798fd6e9 --- /dev/null +++ b/e2e/tests/smoke/file_storage_smoke_test.go @@ -0,0 +1,107 @@ +package smoke + +import ( + "github.com/rook/rook/e2e/framework/enums" + "github.com/rook/rook/e2e/framework/manager" + "github.com/rook/rook/e2e/framework/objects" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "testing" +) + +var env objects.EnvironmentManifest + +func init() { + env = objects.NewManifest() +} + +type FileSystemTestSuite struct { + suite.Suite + rookPlatform enums.RookPlatformType + k8sVersion enums.K8sVersion + rookTag string +} + +func TestFileSystemSmokeSuite(t *testing.T) { + + suite.Run(t, new(FileSystemTestSuite)) +} + +func (suite *FileSystemTestSuite) SetupTest() { + var err error + + suite.rookPlatform, err = enums.GetRookPlatFormTypeFromString(env.Platform) + + require.Nil(suite.T(), err) + + suite.k8sVersion, err = enums.GetK8sVersionFromString(env.K8sVersion) + + require.Nil(suite.T(), err) + + suite.rookTag = env.RookTag + + require.NotEmpty(suite.T(), suite.rookTag, "RookTag parameter is required") + +} + +func (suite *FileSystemTestSuite) TestFileStorage_SmokeTest() { + suite.T().Skip("Skipping test : existing issue https://github.com/rook/rook/issues/612") + var err error + + err, rookInfra := rook_test_infra.GetRookTestInfraManager(suite.rookPlatform, true, suite.k8sVersion) + + require.Nil(suite.T(), err) + + rookInfra.ValidateAndSetupTestPlatform() + + err, _ = rookInfra.InstallRook(suite.rookTag) + + require.Nil(suite.T(), err) + + suite.T().Log("File Storage Smoke Test - Create,Mount,write to, read from and Unmount Filesystem") + sc, _ := CreateSmokeTestClient(rookInfra.GetRookPlatform()) + defer fileSmokecleanUp() + rh := sc.rookHelp + rfc := sc.GetFileSystemClient() + + suite.T().Log("Step 1: Create file System") + _, fsc_err := sc.CreateFileStorage() + require.Nil(suite.T(), fsc_err) + rawlist, _ := rfc.FSList() + filesystemData := rh.ParseFileSystemData(rawlist) + require.Equal(suite.T(), "testfs", filesystemData.Name, "make sure filesystem name matches") + suite.T().Log("File system created") + + suite.T().Log("Step 2: Mount file System") + _, mtfs_err := sc.MountFileStorage() + require.Nil(suite.T(), mtfs_err) + suite.T().Log("File system mounted successfully") + + suite.T().Log("Step 3: Write to file system") + _, wfs_err := sc.WriteToFileStorage("Test data for file", "fsFile1") + require.Nil(suite.T(), wfs_err) + suite.T().Log("Write to file system successful") + + suite.T().Log("Step 4: Read from file system") + read, rd_err := sc.ReadFromFileStorage("fsFile1") + require.Nil(suite.T(), rd_err) + require.Contains(suite.T(), read, "Test data for file", "make sure content of the files is unchanged") + suite.T().Log("Read from file system successful") + + suite.T().Log("Step 5: Mount file System") + _, umtfs_err := sc.UnmountFileStorage() + require.Nil(suite.T(), umtfs_err) + suite.T().Log("File system mounted successfully") + + suite.T().Log("Step 6: Deleting file storage") + _, fsd_err := sc.DeleteFileStorage() + require.Nil(suite.T(), fsd_err) + //Delete is not actually deleting filesystem + suite.T().Log("File system deleted") +} + +func fileSmokecleanUp() { + sc, _ := CreateSmokeTestClient(enums.Kubernetes) + sc.UnmountFileStorage() + sc.DeleteFileStorage() +} diff --git a/e2e/tests/smoke/object_storage_smoke_test.go b/e2e/tests/smoke/object_storage_smoke_test.go new file mode 100644 index 000000000..ac2ce1667 --- /dev/null +++ b/e2e/tests/smoke/object_storage_smoke_test.go @@ -0,0 +1,184 @@ +package smoke + +//TODO - fix test, s3 library is having DNS issues resolving {bucketname}.{endpoint} since endpoint is not a regular amazon s3 endpoint +import ( + "errors" + "github.com/rook/rook/e2e/framework/enums" + "github.com/rook/rook/e2e/framework/manager" + "github.com/rook/rook/e2e/framework/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "testing" +) + +var ( + userid = "rook-user" + userdisplayname = "A rook RGW user" + bucketname = "smokebkt" + objBody = "Test Rook Object Data" + objectKey = "rookObj1" + contentType = "plain/text" +) + +func TestObjectStorageSmokeSuite(t *testing.T) { + suite.Run(t, new(ObjectStorageTestSuite)) +} + +type ObjectStorageTestSuite struct { + suite.Suite + rookPlatform enums.RookPlatformType + k8sVersion enums.K8sVersion + rookTag string +} + +func (suite *ObjectStorageTestSuite) SetupTest() { + var err error + + suite.rookPlatform, err = enums.GetRookPlatFormTypeFromString(env.Platform) + + require.Nil(suite.T(), err) + + suite.k8sVersion, err = enums.GetK8sVersionFromString(env.K8sVersion) + + require.Nil(suite.T(), err) + + suite.rookTag = env.RookTag + + require.NotEmpty(suite.T(), suite.rookTag, "RookTag parameter is required") + + sc, _ := CreateSmokeTestClient(enums.Kubernetes) + sc.CreateObjectStore() +} + +func (suite *ObjectStorageTestSuite) TestObjectStorage_SmokeTest() { + //NOTE - fix test, s3 library is having intermittent DNS issues resolving {bucketname}.{endpoint} since endpoint is not a regular amazon s3 endpoint + suite.T().Skip("Skipping test : existing issue https://github.com/rook/rook/issues/612") + var err error + + err, rookInfra := rook_test_infra.GetRookTestInfraManager(suite.rookPlatform, true, suite.k8sVersion) + + require.Nil(suite.T(), err) + + rookInfra.ValidateAndSetupTestPlatform() + + err, _ = rookInfra.InstallRook(suite.rookTag) + + require.Nil(suite.T(), err) + + suite.T().Log("Object Storage Smoke Test") + sc, _ := CreateSmokeTestClient(rookInfra.GetRookPlatform()) + defer objetTestcleanup() + + port, perr := sc.GetRGWPort() + assert.Nil(suite.T(), perr) + + suite.T().Log("Step 0 : Create Object Store") + _, cobs_err := sc.CreateObjectStore() + assert.Nil(suite.T(), cobs_err) + suite.T().Log("Object store created successfully") + + suite.T().Log("Step 1 : Create Object Store User") + initialUsers, _ := sc.GetObjectStoreUsers() + _, cosu_err := sc.CreateObjectStoreUser() + assert.Nil(suite.T(), cosu_err) + usersAfterCrate, _ := sc.GetObjectStoreUsers() + assert.Equal(suite.T(), len(initialUsers)+1, len(usersAfterCrate), "Make sure user list count is increaded by 1") + getuserData, gu_err := sc.GetObjectStoreUser(userid) + assert.Nil(suite.T(), gu_err) + assert.Equal(suite.T(), userid, getuserData.UserId, "Check user id returned") + assert.Equal(suite.T(), userdisplayname, getuserData.DisplayName, "Check user name returned") + suite.T().Log("Object store user created successfully") + + suite.T().Log("Step 2 : Get connection information") + conninfo, conninfo_error := sc.GetObjectStoreConnection(userid) + assert.Nil(suite.T(), conninfo_error) + + //TODO - need to figure out how to expose rgw endpoint and get host ip + //If for rook infra its localhost, for rook on coreos-k8s its 172.17.4.101 + s3endpoint := "localhost:" + port + s3client := utils.CreateNewS3Helper(s3endpoint, conninfo.AwsAccessKey, conninfo.AwsSecretKey) + + // Using s3 client + + suite.T().Log("Step 3 : Create bucket") + initialBuckets, _ := sc.GetObjectStoreBucketList() + s3client.CreateBucket(bucketname) + BucketsAfterCreate, _ := sc.GetObjectStoreBucketList() + assert.Equal(suite.T(), len(initialBuckets)+1, len(BucketsAfterCreate), "Make sure new bucket is created") + bkt, _ := getBucket(bucketname, BucketsAfterCreate) + assert.Equal(suite.T(), bucketname, bkt.Name) + assert.Equal(suite.T(), userid, bkt.Owner) + suite.T().Log("Bucket created in Object store successfully") + + suite.T().Log("Step 4 : Put Object on bucket") + initObjSize, initObjNum, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterCreate) + assert.Equal(suite.T(), 0, initObjSize) + assert.Equal(suite.T(), 0, initObjNum) + _, po_err := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType) + assert.Nil(suite.T(), po_err) + BucketsAfterPut, _ := sc.GetObjectStoreBucketList() + ObjSize, ObjNum, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterPut) + assert.NotEmpty(suite.T(), 0, ObjSize) + assert.Equal(suite.T(), 1, ObjNum) + suite.T().Log("Object Created on bucket successfully") + + suite.T().Log("Step 5 : Put Object from bucket") + read, err := s3client.GetObjectInBucket(bucketname, objectKey) + assert.Nil(suite.T(), err) + assert.Equal(suite.T(), objBody, read) + suite.T().Log("Object retrived from bucket successfully") + + suite.T().Log("Step 6 : Delete Object on bucket") + _, delobj_err := s3client.DeleteObjectInBucket(bucketname, objectKey) + assert.Nil(suite.T(), delobj_err) + BucketsAfterOjbDelete, _ := sc.GetObjectStoreBucketList() + ObjSize1, ObjNum1, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterOjbDelete) + assert.Equal(suite.T(), 0, ObjSize1) + assert.Equal(suite.T(), 0, ObjNum1) + suite.T().Log("Object deleted on bucket successfully") + + suite.T().Log("Step 6 : Delete bucket") + _, bkdel_err := s3client.DeleteBucket(bucketname) + assert.Nil(suite.T(), bkdel_err) + BucketsAfterDelete, _ := sc.GetObjectStoreBucketList() + assert.Equal(suite.T(), len(initialBuckets), len(BucketsAfterDelete), "Make sure new bucket is deleted") + suite.T().Log("Bucket deleted successfully") + + suite.T().Log("Step 7 : Delete User") + usersBeforeDelete, _ := sc.GetObjectStoreUsers() + _, dosu_err := sc.DeleteObjectStoreUser() + assert.Nil(suite.T(), dosu_err) + usersAfterDelete, _ := sc.GetObjectStoreUsers() + assert.Equal(suite.T(), len(usersBeforeDelete)-1, len(usersAfterDelete), "Make sure user list count is reducd by 1") + suite.T().Log("Object store user created successfully") + +} + +func objetTestcleanup() { + sc, _ := CreateSmokeTestClient(enums.Kubernetes) + conninfo, _ := sc.GetObjectStoreConnection(userid) + port, _ := sc.GetRGWPort() + //TODO - need to figure out how to expose rgw endpoint and get host ip. + s3endpoint := "http://172.17.4.101:" + port + s3client := utils.CreateNewS3Helper(s3endpoint, conninfo.AwsAccessKey, conninfo.AwsSecretKey) + s3client.DeleteObjectInBucket(bucketname, objectKey) + s3client.DeleteBucket(bucketname) + sc.DeleteObjectStoreUser() +} + +func getBucket(bucketname string, bucketdict map[string]utils.ObjectBucketListData) (utils.ObjectBucketListData, error) { + if val, ok := bucketdict[bucketname]; ok { + return val, nil + } else { + return utils.ObjectBucketListData{}, errors.New("Bucket not found") + } +} + +func getBucketSizeAndObjectes(bucketname string, bucketdict map[string]utils.ObjectBucketListData) (int, int, error) { + bkt, err := getBucket(bucketname, bucketdict) + if err != nil { + return 0, 0, errors.New("Bucket not found") + } + return bkt.Size, bkt.NumberOfObjects, nil +} diff --git a/e2e/tests/smoke/smoke_test_helper.go b/e2e/tests/smoke/smoke_test_helper.go new file mode 100644 index 000000000..77a43734f --- /dev/null +++ b/e2e/tests/smoke/smoke_test_helper.go @@ -0,0 +1,425 @@ +package smoke + +import ( + "fmt" + "github.com/rook/rook/e2e/framework/clients" + "github.com/rook/rook/e2e/framework/contracts" + "github.com/rook/rook/e2e/framework/enums" + "github.com/rook/rook/e2e/framework/utils" +) + +type SmokeTestHelper struct { + platform enums.RookPlatformType + rookclient *clients.RookClient + rookHelp *utils.RookHelper + k8sHelp *utils.K8sHelper +} + +type blockTestData struct { + name string + size int + mountpath string + pvcDefPath string + podDefPath string +} + +type fileTestData struct { + name string + mountpath string + podname string + podDefPath string +} + +type objectUserData struct { + userId string + displayname string + emailId string +} + +type objectConnectionData struct { + aws_endpoint string + aws_host string + aws_secret_key_id string + aws_secret_access_key string +} + +func createObjectUserData(userid string, displayname string, emailid string) objectUserData { + return objectUserData{userid, displayname, emailid} +} + +func createObjectConnectionData(endpoint string, host string, secretid string, secretkey string) objectConnectionData { + return objectConnectionData{endpoint, host, secretid, secretkey} +} + +func CreateSmokeTestClient(platform enums.RookPlatformType) (*SmokeTestHelper, error) { + + rc, err := getRookClient(platform) + + return &SmokeTestHelper{platform: platform, + rookclient: rc, + rookHelp: utils.CreateRookHelper(), + k8sHelp: utils.CreatK8sHelper()}, err + +} + +func getRookClient(platform enums.RookPlatformType) (*clients.RookClient, error) { + return clients.CreateRook_Client(platform) + +} +func (h *SmokeTestHelper) GetBlockClient() contracts.IRookBlock { + return h.rookclient.GetBlockClient() +} + +func (h *SmokeTestHelper) GetFileSystemClient() contracts.IRookFilesystem { + return h.rookclient.GetFileSystemClient() +} + +func (h *SmokeTestHelper) GetObjectClient() contracts.IRookObject { + return h.rookclient.GetObjectClient() +} + +func (h *SmokeTestHelper) getBlockTestData() (blockTestData, error) { + switch h.platform { + case enums.Kubernetes: + return blockTestData{"block-test", + 1048576, + "/tmp/rook1", + "../../data/smoke/pool_sc_pvc.yaml", + "../../data/smoke/block_mount.yaml"}, nil + case enums.StandAlone: + return blockTestData{"block-test", + 1048576, + "/tmp/rook1", + "", + ""}, nil + default: + return blockTestData{}, fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) getFileTestData() (fileTestData, error) { + switch h.platform { + case enums.Kubernetes: + return fileTestData{"testfs", + "/tmp/rookfs", + "file-test", + "../../data/smoke/file_mount.tmpl"}, nil + case enums.StandAlone: + return fileTestData{"testfs", + "/tmp/rookfs", + "", ""}, nil + default: + return fileTestData{}, fmt.Errorf("Unsupported Rook Platform Type") + } +} + +func (h *SmokeTestHelper) getObjectStoreUserData() objectUserData { + return objectUserData{"rook-user", "A rook RGW user", ""} +} +func (h *SmokeTestHelper) getRGWExtenalSevDef() string { + return "../../data/smoke/rgw_external.yaml" +} + +func (h *SmokeTestHelper) CreateBlockStorage() (string, error) { + switch h.platform { + case enums.Kubernetes: + blockData, _ := h.getBlockTestData() + // Create storage pool, storage class and pvc + return h.k8sHelp.ResourceOperation("create", blockData.pvcDefPath) + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) MountBlockStorage() (string, error) { + switch h.platform { + case enums.Kubernetes: + blockData, _ := h.getBlockTestData() + _, err := h.GetBlockClient().BlockMap(blockData.podDefPath, blockData.mountpath) + if err != nil { + return "MOUNT UNSUCCESSFUL", err + } + if h.k8sHelp.IsPodRunning(blockData.name) { + + return "MOUNT SUCCESSFUL", nil + } + return "MOUNT UNSUCCESSFUL", fmt.Errorf("Cannot mount block storage") + + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) WriteToBlockStorage(data string, filename string) (string, error) { + switch h.platform { + case enums.Kubernetes: + blockData, _ := h.getBlockTestData() + return h.GetBlockClient().BlockWrite(blockData.name, + blockData.mountpath, data, filename, "") + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) ReadFromBlockStorage(filename string) (string, error) { + switch h.platform { + case enums.Kubernetes: + blockData, _ := h.getBlockTestData() + return h.GetBlockClient().BlockRead(blockData.name, + blockData.mountpath, filename, "") + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) UnMountBlockStorage() (string, error) { + switch h.platform { + case enums.Kubernetes: + blockData, _ := h.getBlockTestData() + _, err := h.GetBlockClient().BlockUnmap(blockData.podDefPath, blockData.mountpath) + if err != nil { + return "UNMOUNT UNSUCCESSFUL", err + } + if h.k8sHelp.IsPodTerminated(blockData.name) { + return "UNMOUNT SUCCESSFUL", nil + } + return "UNMOUNT UNSUCCESSFUL", fmt.Errorf("Cannot unmount block storage") + + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) DeleteBlockStorage() (string, error) { + switch h.platform { + case enums.Kubernetes: + blockData, _ := h.getBlockTestData() + // Delete storage pool, storage class and pvc + return h.k8sHelp.ResourceOperation("delete", blockData.pvcDefPath) + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) CleanUpDymanicBlockStorge() { + switch h.platform { + case enums.Kubernetes: + // Delete storage pool, storage class and pvc + blocklistraw, _ := h.GetBlockClient().BlockList() + blocklistMap := h.rookHelp.ParseBlockListData(blocklistraw) + h.k8sHelp.CleanUpDymaincCreatedPVC(blocklistMap) + } +} + +func (h *SmokeTestHelper) CreateFileStorage() (string, error) { + switch h.platform { + case enums.Kubernetes: + fileTestData, _ := h.getFileTestData() + // Create storage pool, storage class and pvc + return h.GetFileSystemClient().FSCreate(fileTestData.name) + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) MountFileStorage() (string, error) { + switch h.platform { + case enums.Kubernetes: + fileTestData, _ := h.getFileTestData() + // Create pod that has has file sytem mounted + _, err := h.k8sHelp.ResourceOperationFromTemplate("create", fileTestData.podDefPath) + if err != nil { + return "MOUNT UNSUCCESSFUL", err + } + if h.k8sHelp.IsPodRunningInNamespace(fileTestData.podname) { + return "MOUNT SUCCESSFUL", nil + } + return "MOUNT UNSUCCESSFUL", fmt.Errorf("Cannot mount File storage") + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) WriteToFileStorage(data string, filename string) (string, error) { + switch h.platform { + case enums.Kubernetes: + fileTestData, _ := h.getFileTestData() + return h.GetFileSystemClient().FSWrite(fileTestData.podname, + fileTestData.mountpath, data, filename, "rook") + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) ReadFromFileStorage(filename string) (string, error) { + switch h.platform { + case enums.Kubernetes: + fileTestData, _ := h.getFileTestData() + return h.GetFileSystemClient().FSRead(fileTestData.podname, + fileTestData.mountpath, filename, "rook") + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) UnmountFileStorage() (string, error) { + switch h.platform { + case enums.Kubernetes: + fileTestData, _ := h.getFileTestData() + // Create pod that has has file sytem mounted + _, err := h.k8sHelp.ResourceOperationFromTemplate("delete", fileTestData.podDefPath) + if err != nil { + return "UNMOUNT UNSUCCESSFUL", err + } + if h.k8sHelp.IsPodTerminatedInNamespace(fileTestData.podname) { + return "UNMOUNT SUCCESSFUL", nil + } + return "UNMOUNT UNSUCCESSFUL", fmt.Errorf("Cannot unmount File storage") + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) DeleteFileStorage() (string, error) { + switch h.platform { + case enums.Kubernetes: + fileTestData, _ := h.getFileTestData() + // Create storage pool, storage class and pvc + return h.GetFileSystemClient().FSDelete(fileTestData.name) + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +} + +func (h *SmokeTestHelper) CreateObjectStore() (string, error) { + switch h.platform { + case enums.Kubernetes: + _, err := h.GetObjectClient().ObjectCreate() + if err != nil { + return "Couldn't create object store ", err + } + if h.k8sHelp.IsServiceUpInNameSpace("rgw") { + _, err := h.k8sHelp.GetService("rgw-external") + if err != nil { + h.k8sHelp.ResourceOperation("create", h.getRGWExtenalSevDef()) + } + + return "OJECT STORE CREATED", nil + } + return "Couldn't create object store ", fmt.Errorf("Cannot create object store") + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + } +} + +func (h *SmokeTestHelper) CreateObjectStoreUser() (string, error) { + objectUserdata := h.getObjectStoreUserData() + _, err := h.GetObjectClient().ObjectCreateUser(objectUserdata.userId, objectUserdata.displayname) + if err != nil { + return "USER NOT CREATED", fmt.Errorf("User not created for object store") + } + return "USER CREATED ", nil +} + +func (h *SmokeTestHelper) GetObjectStoreUsers() (map[string]utils.ObjectUserListData, error) { + rawdata, err := h.GetObjectClient().ObjectListUser() + if err != nil { + return nil, err + } else { + return h.rookHelp.ParserObjectUserListData(rawdata), nil + } + +} + +func (h *SmokeTestHelper) GetObjectStoreUser(userid string) (utils.ObjectUserData, error) { + rawdata, err := h.GetObjectClient().ObjectGetUser(userid) + if err != nil { + return utils.ObjectUserData{}, err + } else { + return h.rookHelp.ParserObjectUserData(rawdata), nil + } + +} + +func (h *SmokeTestHelper) GetObjectStoreConnection(userid string) (utils.ObjectConnectionData, error) { + rawdata, err := h.GetObjectClient().ObjectConnection(userid) + if err != nil { + return utils.ObjectConnectionData{}, err + } else { + return h.rookHelp.ParserObjectConnectionData(rawdata), nil + } + +} + +func (h *SmokeTestHelper) GetObjectStoreBucketList() (map[string]utils.ObjectBucketListData, error) { + rawdata, err := h.GetObjectClient().ObjectBucketList() + if err != nil { + return nil, err + } else { + return h.rookHelp.ParserObjectBucketListData(rawdata), nil + } +} + +func (h *SmokeTestHelper) DeleteObjectStoreUser() (string, error) { + objectUserdata := h.getObjectStoreUserData() + _, err := h.GetObjectClient().ObjectDeleteUser(objectUserdata.userId) + if err != nil { + return "USER NOT DELETED", fmt.Errorf("User not deleted for object store") + } + return "USER DELETED ", nil +} + +func (h *SmokeTestHelper) GetRGWPort() (string, error) { + switch h.platform { + case enums.Kubernetes: + rawdata, err := h.k8sHelp.GetService("rgw-external") + if err != nil { + return "port not found", err + } + return h.rookHelp.GetRgwServiceNodePort(rawdata) + case enums.StandAlone: + return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") + default: + return "", fmt.Errorf("Unsupported Rook Platform Type") + + } +}