From e44cfe713bb64bcbb2e8bbd39bbfcbfcc1e51fbb Mon Sep 17 00:00:00 2001 From: Divesh Angula Date: Mon, 8 May 2017 22:32:12 -0700 Subject: [PATCH] enable file and object store smoke test and fix naming conventions --- e2e/data/smoke/rgw_external.yaml | 1 + e2e/framework/manager/rook_test_infra.go | 42 +++++------ e2e/framework/utils/k8s_helper.go | 44 ++++++++++++ e2e/framework/utils/rook_helper.go | 24 ------- e2e/framework/utils/s3_operations.go | 27 ++++--- e2e/tests/smoke/file_storage_smoke_test.go | 1 - e2e/tests/smoke/object_storage_smoke_test.go | 75 +++++++++----------- e2e/tests/smoke/smoke_test_helper.go | 16 ++--- 8 files changed, 124 insertions(+), 106 deletions(-) diff --git a/e2e/data/smoke/rgw_external.yaml b/e2e/data/smoke/rgw_external.yaml index 1458da8f2..9ff46707c 100644 --- a/e2e/data/smoke/rgw_external.yaml +++ b/e2e/data/smoke/rgw_external.yaml @@ -12,6 +12,7 @@ spec: port: 53390 protocol: TCP targetPort: 53390 + nodePort: 30001 selector: app: rgw rook_cluster: rook diff --git a/e2e/framework/manager/rook_test_infra.go b/e2e/framework/manager/rook_test_infra.go index e325842e5..d45453ba6 100644 --- a/e2e/framework/manager/rook_test_infra.go +++ b/e2e/framework/manager/rook_test_infra.go @@ -34,7 +34,7 @@ var ( ) const ( - temp_image_file_name = "temp_image.tar" + tempImageFileName = "temp_image.tar" rookOperatorFileName = "rook-operator.yaml" rookClusterFileName = "rook-cluster.yaml" rookClientFileName = "rook-client.yml" @@ -48,9 +48,9 @@ const ( rookOperatorPrefix = "quay.io/rook/rookd" rookClientPrefix = "quay.io/rook/rook" rookOperatorCreatedTpr = "cluster.rook.io" - k8s_master_container_name = "kube-master" - k8s_node1_container_name = "kube-node-1" - k8s_node2_container_name = "kube-node-2" + masterContinerName = "kube-master" + node1ContinerName = "kube-node-1" + node2ContinerName = "kube-node-2" cephBaseImageName = "ceph/base" defaultTagName = "master-latest" ) @@ -170,30 +170,30 @@ func (r *rookTestInfraManager) copyImageToNode(containerId string, imageName str cmdOut := utils.ExecuteCommand(objects.CommandArgs{Command: "docker", CmdArgs: []string{"pull", imageName}}) if cmdOut.Err != nil || cmdOut.ExitCode != 0 { - return fmt.Errorf("Failed to pull image") + panic(fmt.Errorf("Failed to pull image : %s", imageName)) } fmt.Println("success...") } - fmt.Println("archiving the docker image: " + temp_image_file_name) - cmdOut = utils.ExecuteCommand(objects.CommandArgs{Command: "docker", CmdArgs: []string{"save", "-o", temp_image_file_name, imageName}}) + fmt.Println("archiving the docker image: " + tempImageFileName) + cmdOut = utils.ExecuteCommand(objects.CommandArgs{Command: "docker", CmdArgs: []string{"save", "-o", tempImageFileName, imageName}}) if cmdOut.Err != nil || cmdOut.ExitCode != 0 { - return fmt.Errorf("Failed to archive image") + panic(fmt.Errorf("Failed to archive image : %s", imageName)) } fmt.Println("success...") fmt.Println("copying archived image to node container") - _, _ = r.executeDockerCommand("", enums.Copy, temp_image_file_name, containerId+":/"+temp_image_file_name) + _, _ = r.executeDockerCommand("", enums.Copy, tempImageFileName, containerId+":/"+tempImageFileName) if cmdOut.Err != nil || cmdOut.ExitCode != 0 { - return fmt.Errorf("Failed to copy image to container") + panic(fmt.Errorf("Failed to copy image to container : %s", imageName)) } fmt.Println("success...") fmt.Println("importing archived image into container registry") - _, _ = r.executeDockerCommand(containerId, enums.Exec, "/bin/bash", "-c", "docker load -i /"+temp_image_file_name) + _, _ = r.executeDockerCommand(containerId, enums.Exec, "/bin/bash", "-c", "docker load -i /"+tempImageFileName) if cmdOut.Err != nil || cmdOut.ExitCode != 0 { - return fmt.Errorf("Failed load image into containers repository") + panic(fmt.Errorf("Failed load image into containers repository : %s", imageName)) } fmt.Println("success...") @@ -242,9 +242,9 @@ func (r *rookTestInfraManager) ValidateAndSetupTestPlatform() { } //if any of these methods fail, the executeDockerCommand will panic - k8sMasterContainerId := r.getContainerIdByName(k8s_master_container_name) - k8sNode1ContainerId := r.getContainerIdByName(k8s_node1_container_name) - k8sNode2ContainerId := r.getContainerIdByName(k8s_node2_container_name) + k8sMasterContainerId := r.getContainerIdByName(masterContinerName) + k8sNode1ContainerId := r.getContainerIdByName(node1ContinerName) + k8sNode2ContainerId := r.getContainerIdByName(node2ContinerName) //install rbd shim _, _ = r.executeDockerCommand("", enums.Copy, curdir+"/"+scriptsPath+"/rbd", k8sMasterContainerId+":/bin/rbd") @@ -362,31 +362,31 @@ func (r *rookTestInfraManager) InstallRook(tag string) (err error, client contra rookOperatorTag := rookOperatorPrefix + ":" + tag rookClientTag := rookClientPrefix + ":" + tag - err = r.copyImageToNode(r.getContainerIdByName(k8s_master_container_name), rookOperatorTag) + err = r.copyImageToNode(r.getContainerIdByName(masterContinerName), rookOperatorTag) if err != nil { panic(err) } - err = r.copyImageToNode(r.getContainerIdByName(k8s_node1_container_name), rookOperatorTag) + err = r.copyImageToNode(r.getContainerIdByName(node1ContinerName), rookOperatorTag) if err != nil { panic(err) } - err = r.copyImageToNode(r.getContainerIdByName(k8s_node2_container_name), rookOperatorTag) + err = r.copyImageToNode(r.getContainerIdByName(node2ContinerName), rookOperatorTag) if err != nil { panic(err) } - err = r.copyImageToNode(r.getContainerIdByName(k8s_master_container_name), rookClientTag) + err = r.copyImageToNode(r.getContainerIdByName(masterContinerName), rookClientTag) if err != nil { panic(err) } - err = r.copyImageToNode(r.getContainerIdByName(k8s_node1_container_name), rookClientTag) + err = r.copyImageToNode(r.getContainerIdByName(node1ContinerName), rookClientTag) if err != nil { panic(err) } - err = r.copyImageToNode(r.getContainerIdByName(k8s_node2_container_name), rookClientTag) + err = r.copyImageToNode(r.getContainerIdByName(node2ContinerName), rookClientTag) if err != nil { panic(err) } diff --git a/e2e/framework/utils/k8s_helper.go b/e2e/framework/utils/k8s_helper.go index 3c6817657..9bf52af73 100644 --- a/e2e/framework/utils/k8s_helper.go +++ b/e2e/framework/utils/k8s_helper.go @@ -251,3 +251,47 @@ func (k8sh *K8sHelper) IsThirdPartyResourcePresent(tprname string) bool { return false } + +func (k8sh *K8sHelper) GetPodDetails(podNamePattern string, namespace string) (string, error) { + cmdArgs := []string{"get", "pods", "-l", "app=" + podNamePattern, "-o", "wide", "--no-headers=true"} + if namespace != "" { + cmdArgs = append(cmdArgs, []string{"-n", namespace}...) + } + sout, serr, status := ExecuteCmd("kubectl", cmdArgs) + if status != 0 || strings.Contains(sout, "No resources found") { + return serr, fmt.Errorf("Cannot find pod in with name like %s in namespace : %s", podNamePattern, namespace) + } + return sout, nil +} + +func (k8sh *K8sHelper) GetPodHostId(podNamePattern string, namespace string) (string, error) { + data, err := k8sh.GetPodDetails(podNamePattern, namespace) + if err != nil { + return data, err + } + + // Handle case when no data is returned + lines := strings.Split(data, "\n") + + //extract name of the pod + lineRawdata := strings.Split(lines[0], " ") + var r []string + for _, str := range lineRawdata { + if str != "" { + r = append(r, strings.TrimSpace(str)) + } + } + + //get host Ip of the pod + cmdArgs := []string{"get", "pods", r[0], "-o", "jsonpath='{.status.hostIP}'"} + if namespace != "" { + cmdArgs = append(cmdArgs, []string{"-n", namespace}...) + } + sout, serr, status := ExecuteCmd("kubectl", cmdArgs) + if status == 0 { + hostIp := strings.Replace(sout, "'", "", -1) + return strings.TrimSpace(hostIp), nil + } else { + return serr, fmt.Errorf("Error Getting Monitor IP") + } +} diff --git a/e2e/framework/utils/rook_helper.go b/e2e/framework/utils/rook_helper.go index 76f65d6a0..4fe5e0c06 100644 --- a/e2e/framework/utils/rook_helper.go +++ b/e2e/framework/utils/rook_helper.go @@ -1,7 +1,6 @@ package utils import ( - "fmt" "strconv" "strings" ) @@ -221,26 +220,3 @@ func (rh *RookHelper) ParserObjectBucketListData(rawdata string) map[string]Obje } return data } - -func (rh *RookHelper) GetRgwServiceNodePort(rawdata string) (string, error) { - lines := strings.Split(rawdata, "\n") - if len(lines) <= 1 { - return "NOT FOUND", fmt.Errorf("Port not found") - } - lines = lines[1 : len(lines)-1] - if len(lines) == 1 { - bktsrawdata := strings.Split(lines[0], " ") - var r []string - for _, str := range bktsrawdata { - if str != "" { - r = append(r, strings.TrimSpace(str)) - } - } - ports := strings.Split(r[3], ":") - port := strings.Replace(ports[1], "/TCP", "", -1) - return port, nil - - } else { - return "NOT FOUND", fmt.Errorf("Port not found") - } -} diff --git a/e2e/framework/utils/s3_operations.go b/e2e/framework/utils/s3_operations.go index 570265884..380bdca28 100644 --- a/e2e/framework/utils/s3_operations.go +++ b/e2e/framework/utils/s3_operations.go @@ -2,29 +2,38 @@ package utils import ( "bytes" - "fmt" + "strings" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - "strings" ) type S3Helper struct { s3client *s3.S3 } +// create a s3 client for specfied endpoint and creds func CreateNewS3Helper(endpoint string, keyId string, keySecret string) *S3Helper { + creds := credentials.NewStaticCredentials(keyId, keySecret, "") - _, err := creds.Get() - if err != nil { - fmt.Printf("bad credentials: %s", err) - } - cfg := aws.NewConfig().WithEndpoint(endpoint).WithRegion("us-west-1").WithCredentials(creds).WithDisableSSL(true) - svc := s3.New(session.New(), cfg) + // create aws s3 config, must use 'other-v2-signature' region for ceph object store + awsConfig := aws.NewConfig(). + WithRegion("other-v2-signature"). + WithCredentials(creds). + WithEndpoint(endpoint). + WithS3ForcePathStyle(true). + WithDisableSSL(true) - return &S3Helper{svc} + //create new session + ses := session.New() + + //create new s3 client connection + c := s3.New(ses, awsConfig) + + return &S3Helper{c} } func (h *S3Helper) CreateBucket(name string) (bool, error) { diff --git a/e2e/tests/smoke/file_storage_smoke_test.go b/e2e/tests/smoke/file_storage_smoke_test.go index acd0bcda0..6ac5bce32 100644 --- a/e2e/tests/smoke/file_storage_smoke_test.go +++ b/e2e/tests/smoke/file_storage_smoke_test.go @@ -60,7 +60,6 @@ func (suite *FileSystemTestSuite) SetupTest() { func (suite *FileSystemTestSuite) TestFileStorage_SmokeTest() { - suite.T().Skip("Skipping test : existing issue https://github.com/rook/rook/issues/612") suite.T().Log("File Storage Smoke Test - Create,Mount,write to, read from and Unmount Filesystem") defer fileSmokecleanUp(suite.helper) diff --git a/e2e/tests/smoke/object_storage_smoke_test.go b/e2e/tests/smoke/object_storage_smoke_test.go index ac2ce1667..90429e92c 100644 --- a/e2e/tests/smoke/object_storage_smoke_test.go +++ b/e2e/tests/smoke/object_storage_smoke_test.go @@ -1,15 +1,15 @@ package smoke -//TODO - fix test, s3 library is having DNS issues resolving {bucketname}.{endpoint} since endpoint is not a regular amazon s3 endpoint import ( "errors" + "testing" + "github.com/rook/rook/e2e/framework/enums" "github.com/rook/rook/e2e/framework/manager" "github.com/rook/rook/e2e/framework/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "testing" ) var ( @@ -30,9 +30,11 @@ type ObjectStorageTestSuite struct { rookPlatform enums.RookPlatformType k8sVersion enums.K8sVersion rookTag string + helper *SmokeTestHelper } func (suite *ObjectStorageTestSuite) SetupTest() { + var err error suite.rookPlatform, err = enums.GetRookPlatFormTypeFromString(env.Platform) @@ -47,15 +49,6 @@ func (suite *ObjectStorageTestSuite) SetupTest() { require.NotEmpty(suite.T(), suite.rookTag, "RookTag parameter is required") - sc, _ := CreateSmokeTestClient(enums.Kubernetes) - sc.CreateObjectStore() -} - -func (suite *ObjectStorageTestSuite) TestObjectStorage_SmokeTest() { - //NOTE - fix test, s3 library is having intermittent DNS issues resolving {bucketname}.{endpoint} since endpoint is not a regular amazon s3 endpoint - suite.T().Skip("Skipping test : existing issue https://github.com/rook/rook/issues/612") - var err error - err, rookInfra := rook_test_infra.GetRookTestInfraManager(suite.rookPlatform, true, suite.k8sVersion) require.Nil(suite.T(), err) @@ -66,45 +59,44 @@ func (suite *ObjectStorageTestSuite) TestObjectStorage_SmokeTest() { require.Nil(suite.T(), err) - suite.T().Log("Object Storage Smoke Test") - sc, _ := CreateSmokeTestClient(rookInfra.GetRookPlatform()) - defer objetTestcleanup() + suite.helper, err = CreateSmokeTestClient(rookInfra.GetRookPlatform()) + require.Nil(suite.T(), err) - port, perr := sc.GetRGWPort() - assert.Nil(suite.T(), perr) +} +func (suite *ObjectStorageTestSuite) TestObjectStorage_SmokeTest() { + + suite.T().Log("Object Storage Smoke Test - Create Object Store, User,Bucket and read/write to bucket") + + defer objectTestcleanup(suite.helper) suite.T().Log("Step 0 : Create Object Store") - _, cobs_err := sc.CreateObjectStore() + _, cobs_err := suite.helper.CreateObjectStore() assert.Nil(suite.T(), cobs_err) suite.T().Log("Object store created successfully") suite.T().Log("Step 1 : Create Object Store User") - initialUsers, _ := sc.GetObjectStoreUsers() - _, cosu_err := sc.CreateObjectStoreUser() + initialUsers, _ := suite.helper.GetObjectStoreUsers() + _, cosu_err := suite.helper.CreateObjectStoreUser() assert.Nil(suite.T(), cosu_err) - usersAfterCrate, _ := sc.GetObjectStoreUsers() + usersAfterCrate, _ := suite.helper.GetObjectStoreUsers() assert.Equal(suite.T(), len(initialUsers)+1, len(usersAfterCrate), "Make sure user list count is increaded by 1") - getuserData, gu_err := sc.GetObjectStoreUser(userid) + getuserData, gu_err := suite.helper.GetObjectStoreUser(userid) assert.Nil(suite.T(), gu_err) assert.Equal(suite.T(), userid, getuserData.UserId, "Check user id returned") assert.Equal(suite.T(), userdisplayname, getuserData.DisplayName, "Check user name returned") suite.T().Log("Object store user created successfully") suite.T().Log("Step 2 : Get connection information") - conninfo, conninfo_error := sc.GetObjectStoreConnection(userid) + conninfo, conninfo_error := suite.helper.GetObjectStoreConnection(userid) assert.Nil(suite.T(), conninfo_error) - //TODO - need to figure out how to expose rgw endpoint and get host ip - //If for rook infra its localhost, for rook on coreos-k8s its 172.17.4.101 - s3endpoint := "localhost:" + port + s3endpoint, _ := suite.helper.GetRGWServiceUrl() s3client := utils.CreateNewS3Helper(s3endpoint, conninfo.AwsAccessKey, conninfo.AwsSecretKey) - // Using s3 client - suite.T().Log("Step 3 : Create bucket") - initialBuckets, _ := sc.GetObjectStoreBucketList() + initialBuckets, _ := suite.helper.GetObjectStoreBucketList() s3client.CreateBucket(bucketname) - BucketsAfterCreate, _ := sc.GetObjectStoreBucketList() + BucketsAfterCreate, _ := suite.helper.GetObjectStoreBucketList() assert.Equal(suite.T(), len(initialBuckets)+1, len(BucketsAfterCreate), "Make sure new bucket is created") bkt, _ := getBucket(bucketname, BucketsAfterCreate) assert.Equal(suite.T(), bucketname, bkt.Name) @@ -117,9 +109,9 @@ func (suite *ObjectStorageTestSuite) TestObjectStorage_SmokeTest() { assert.Equal(suite.T(), 0, initObjNum) _, po_err := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType) assert.Nil(suite.T(), po_err) - BucketsAfterPut, _ := sc.GetObjectStoreBucketList() + BucketsAfterPut, _ := suite.helper.GetObjectStoreBucketList() ObjSize, ObjNum, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterPut) - assert.NotEmpty(suite.T(), 0, ObjSize) + assert.NotEmpty(suite.T(), ObjSize) assert.Equal(suite.T(), 1, ObjNum) suite.T().Log("Object Created on bucket successfully") @@ -132,7 +124,7 @@ func (suite *ObjectStorageTestSuite) TestObjectStorage_SmokeTest() { suite.T().Log("Step 6 : Delete Object on bucket") _, delobj_err := s3client.DeleteObjectInBucket(bucketname, objectKey) assert.Nil(suite.T(), delobj_err) - BucketsAfterOjbDelete, _ := sc.GetObjectStoreBucketList() + BucketsAfterOjbDelete, _ := suite.helper.GetObjectStoreBucketList() ObjSize1, ObjNum1, _ := getBucketSizeAndObjectes(bucketname, BucketsAfterOjbDelete) assert.Equal(suite.T(), 0, ObjSize1) assert.Equal(suite.T(), 0, ObjNum1) @@ -141,30 +133,27 @@ func (suite *ObjectStorageTestSuite) TestObjectStorage_SmokeTest() { suite.T().Log("Step 6 : Delete bucket") _, bkdel_err := s3client.DeleteBucket(bucketname) assert.Nil(suite.T(), bkdel_err) - BucketsAfterDelete, _ := sc.GetObjectStoreBucketList() + BucketsAfterDelete, _ := suite.helper.GetObjectStoreBucketList() assert.Equal(suite.T(), len(initialBuckets), len(BucketsAfterDelete), "Make sure new bucket is deleted") suite.T().Log("Bucket deleted successfully") suite.T().Log("Step 7 : Delete User") - usersBeforeDelete, _ := sc.GetObjectStoreUsers() - _, dosu_err := sc.DeleteObjectStoreUser() + usersBeforeDelete, _ := suite.helper.GetObjectStoreUsers() + _, dosu_err := suite.helper.DeleteObjectStoreUser() assert.Nil(suite.T(), dosu_err) - usersAfterDelete, _ := sc.GetObjectStoreUsers() + usersAfterDelete, _ := suite.helper.GetObjectStoreUsers() assert.Equal(suite.T(), len(usersBeforeDelete)-1, len(usersAfterDelete), "Make sure user list count is reducd by 1") suite.T().Log("Object store user created successfully") } -func objetTestcleanup() { - sc, _ := CreateSmokeTestClient(enums.Kubernetes) - conninfo, _ := sc.GetObjectStoreConnection(userid) - port, _ := sc.GetRGWPort() - //TODO - need to figure out how to expose rgw endpoint and get host ip. - s3endpoint := "http://172.17.4.101:" + port +func objectTestcleanup(h *SmokeTestHelper) { + conninfo, _ := h.GetObjectStoreConnection(userid) + s3endpoint, _ := h.GetRGWServiceUrl() s3client := utils.CreateNewS3Helper(s3endpoint, conninfo.AwsAccessKey, conninfo.AwsSecretKey) s3client.DeleteObjectInBucket(bucketname, objectKey) s3client.DeleteBucket(bucketname) - sc.DeleteObjectStoreUser() + h.DeleteObjectStoreUser() } func getBucket(bucketname string, bucketdict map[string]utils.ObjectBucketListData) (utils.ObjectBucketListData, error) { diff --git a/e2e/tests/smoke/smoke_test_helper.go b/e2e/tests/smoke/smoke_test_helper.go index 77a43734f..f3b03527c 100644 --- a/e2e/tests/smoke/smoke_test_helper.go +++ b/e2e/tests/smoke/smoke_test_helper.go @@ -37,10 +37,10 @@ type objectUserData struct { } type objectConnectionData struct { - aws_endpoint string - aws_host string - aws_secret_key_id string - aws_secret_access_key string + awsEndpoint string + awsHost string + awsSecretKeyId string + awsSecretAccessKey string } func createObjectUserData(userid string, displayname string, emailid string) objectUserData { @@ -408,14 +408,14 @@ func (h *SmokeTestHelper) DeleteObjectStoreUser() (string, error) { return "USER DELETED ", nil } -func (h *SmokeTestHelper) GetRGWPort() (string, error) { +func (h *SmokeTestHelper) GetRGWServiceUrl() (string, error) { switch h.platform { case enums.Kubernetes: - rawdata, err := h.k8sHelp.GetService("rgw-external") + hostip, err := h.k8sHelp.GetPodHostId("rgw", "rook") if err != nil { - return "port not found", err + panic(fmt.Errorf("RGW pods not found/object store possibly not started")) } - return h.rookHelp.GetRgwServiceNodePort(rawdata) + return hostip + ":30001", err case enums.StandAlone: return "NEED TO IMPLEMENT", fmt.Errorf("NOT YET IMPLEMENTED") default: