From d426f47a195c198318d5d2fcd543d67228ad3269 Mon Sep 17 00:00:00 2001 From: Divesh Angula Date: Thu, 8 Jun 2017 12:19:48 -0700 Subject: [PATCH] adding long haul test for block . --- e2e/README.md | 18 +- e2e/framework/clients/rest_client.go | 17 +- e2e/framework/objects/environment_manifest.go | 10 +- e2e/framework/utils/mysql_helper.go | 8 +- e2e/scripts/k8sBlockLongHaul.sh | 181 ++++++++++++++++++ e2e/tests/baseSetup.go | 12 +- e2e/tests/block/create_block_test.go | 1 - .../k8s/longhaul/basicBlockonghaul_test.go | 135 +++++++++++++ e2e/tests/cleanup_test_infra_test.go | 7 +- 9 files changed, 367 insertions(+), 22 deletions(-) create mode 100755 e2e/scripts/k8sBlockLongHaul.sh create mode 100644 e2e/tests/block/k8s/longhaul/basicBlockonghaul_test.go diff --git a/e2e/README.md b/e2e/README.md index c7fd52eb1..1052ef009 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -35,6 +35,8 @@ rook_platform| platform rook needs to be installed on | kubernetes | kubernetes k8s_version | version of Kubernetes to be installed | v1.6 | v1.6 rook_version | rook version/tag to be installed | valid rook tag |master-latest skip_install_rook | skips installing rook (if already installed) | true or false | false +load_parallel_runs | performs concurrent operations (optional) | any number | 20 +count| go test paramater to repeat tests| any number | 1 if install_rook flag is set to false, then all the other flags are ignored, and tests are run without rook being installed and set up. Use this flag to run tests against @@ -54,15 +56,25 @@ are run. e.g. go test -run TestObjectStorageSmokeSuite github.com/rook/rook/e2e/tests/smoke --rook_version=v3.0.1 ``` -#### Run Tests on pre-installed/exisitng Rook +#### Run Tests on pre-installed/existing Rook To run tests against a pre-installed rook set ```--skip_install_rook=true``` . Setting skip_install_rook to true will ignore all other flags and run tests without installing rook first. e.g. ``` -go test -run TestFileStorageSmokeSuite github.com/rook/rook/e2e/tests/smoke --skip_install_rook=true +go test -run TestFileSystemSmokeSuite github.com/rook/rook/e2e/tests/smoke --skip_install_rook=true ``` - +#### Run Longhaul Tests +Using go test `-count` option, any tests can be repeated n number of times to simulate load or longhaul test. Although +any test can be converted to a longhaul test, it's ideal to run integration tests under load for extended period to simulate +real world scenarios. Also a new custom test flag `load_parallel_runs` is added to control the number of concurrent operations being performed. +For example look at the [block long haul test](/e2e/tests/block/k8s/longhaul/basicBlockonghaul_test.go) + + To run a longhaul test you can run any integration test with `-count` and `--load_parallel_runs` options + e.g. + ``` + go test -run TestK8sBlockLongHaul github.com/rook/rook/e2e/tests/block/k8s/longhaul --load_parallel_runs=20 -count=1000 + ``` Prerequisites : * Go installed and GO_PATH set up diff --git a/e2e/framework/clients/rest_client.go b/e2e/framework/clients/rest_client.go index 4470fe0ff..fce1b5d84 100644 --- a/e2e/framework/clients/rest_client.go +++ b/e2e/framework/clients/rest_client.go @@ -8,8 +8,10 @@ import ( "github.com/rook/rook/e2e/framework/utils" "github.com/rook/rook/pkg/model" rclient "github.com/rook/rook/pkg/rook/client" + "net" "os" "path/filepath" + "time" ) type RestAPIClient struct { @@ -38,8 +40,19 @@ func CreateRestAPIClient(platform enums.RookPlatformType) *RestAPIClient { default: panic(fmt.Errorf("platfrom type %s not yet supported", platform)) } - - client := rclient.NewRookNetworkRestClient(endpoint, http.DefaultClient) + httpclient := &http.Client{ + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 0, + }).Dial, + DisableKeepAlives: true, + DisableCompression: true, + MaxIdleConnsPerHost: 1, + ResponseHeaderTimeout: 30 * time.Second, + }, + } + client := rclient.NewRookNetworkRestClient(endpoint, httpclient) return &RestAPIClient{client} } diff --git a/e2e/framework/objects/environment_manifest.go b/e2e/framework/objects/environment_manifest.go index 1f59f7df3..8432ceef4 100644 --- a/e2e/framework/objects/environment_manifest.go +++ b/e2e/framework/objects/environment_manifest.go @@ -5,10 +5,11 @@ import ( ) type EnvironmentManifest struct { - K8sVersion string - Platform string - RookTag string - SkipInstallRook string + K8sVersion string + Platform string + RookTag string + SkipInstallRook string + LoadConcurrentRuns int } func NewManifest() EnvironmentManifest { @@ -18,6 +19,7 @@ func NewManifest() EnvironmentManifest { flag.StringVar(&e.Platform, "rook_platform", "Kubernetes", "Platform to install rook on; Kubernetes is the only platform currently supported") flag.StringVar(&e.RookTag, "rook_version", "master-latest", "Docker tag of the rook operator to install, must be in quay.io or local environment") flag.StringVar(&e.SkipInstallRook, "skip_install_rook", "false", "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed") + flag.IntVar(&e.LoadConcurrentRuns, "load_parallel_runs", 20, "number of routines for load test") flag.Parse() return e diff --git a/e2e/framework/utils/mysql_helper.go b/e2e/framework/utils/mysql_helper.go index d7aaaa676..e92bd6ebd 100644 --- a/e2e/framework/utils/mysql_helper.go +++ b/e2e/framework/utils/mysql_helper.go @@ -23,18 +23,20 @@ func CreateNewMySqlHelper(username string, password string, url string, dbname s return &MySqlHelper{DB: db} } - +func (h *MySqlHelper) CloseConnection() { + h.DB.Close() +} func (h *MySqlHelper) PingSuccess() bool { inc := 0 - for inc < 10 { + for inc < 30 { err := h.DB.Ping() if err == nil { return true } inc++ - time.Sleep(1 * time.Second) + time.Sleep(3 * time.Second) } return false diff --git a/e2e/scripts/k8sBlockLongHaul.sh b/e2e/scripts/k8sBlockLongHaul.sh new file mode 100755 index 000000000..2fc6557c5 --- /dev/null +++ b/e2e/scripts/k8sBlockLongHaul.sh @@ -0,0 +1,181 @@ +#!/bin/bash -e + +test_source_repo=$(pwd) +docker_test_repo=/workspace/go/src/github.com/rook/rook +git_smoke_test_directory=github.com/rook/rook/e2e/tests/smoke +git_longhaul_test_directory=github.com/rook/rook/e2e/tests/block/k8s/longhaul +git_test_directory=github.com/rook/rook/e2e/tests +container_image=quay.io/rook/rook-test +tmp_docker_sock_path=/var/run/docker.sock +results_dir=results +results_filename=test.log +longhaul_result_name=longhaul +temp_archive=forRookImage.tar + +#create the rook infrastructure container +rook_infra::create() { + export id=$(docker run \ + -d \ + -e GOPATH=/workspace/go \ + -e GOROOT=/usr/lib/go \ + --privileged \ + --security-opt=seccomp:unconfined \ + -v /lib/modules:/lib/modules \ + -v /sbin/modprobe:/sbin/modprobe \ + -v /sys/bus:/sys/bus \ + -v /dev:/dev \ + -v /var/run/docker.sock:${tmp_docker_sock_path} \ + -v ${test_source_repo}:${docker_test_repo} \ + -w ${docker_test_repo}/e2e \ + ${container_image} \ + /sbin/init) + + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + + local attempt_num=1 + local max_attempts=10 + + until [[ $(docker inspect -f {{.State.Running}} $id) == "true" ]] + do + if (( attempt_num == max_attempts )) + then + echo "rook-test infra container failed to start" + return 1 + else + sleep 1 + attempt_num++ + + fi + done + + echo "rook-test container is running..." +} + +#prepare the rook_infra container to run tests +rook_infra::init() { + echo Creating results directory + docker exec ${id} mkdir -p ${results_dir} + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + + echo Installing rook-test-framework dependencies... + docker exec ${id} /bin/bash -c \ + "glide install" + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + echo Success... +} + +rook_infra::run_test() { + local test_name_regex=$1 + local tag_name=$2 + local rook_platform=$3 + local k8s_version=$4 + + docker exec -t ${id} /bin/bash -c \ + "go test -timeout 1200s -run ${test_name_regex} ${git_smoke_test_directory} --rook_platform=${rook_platform} --k8s_version=${k8s_version} --rook_version=${tag_name} -v 2>&1 | tee ${results_dir}/${results_filename}" + + rc=$?; if [[ $rc != 0 ]]; then set -e; fi +} + +rook_infra::run_longhul() { + local test_name_regex=$1 + local total_runs=$2 + local test_runs=$3 + local parallel_runs=$4 + local test_intervel=$5 + for (( totalruns=1; totalruns<=${total_runs}; totalruns++ )) + do + + docker exec -t ${id} /bin/bash -c \ + "go test -run ${test_name_regex} ${git_longhaul_test_directory} --skip_install_rook=true --load_parallel_runs=${parallel_runs} -count=${test_runs} -v 2>&1 | tee ${results_dir}/${longhaul_result_name}${totalruns}.log" + docker exec ${id} /bin/bash -c \ + "go get -u -f github.com/jstemmer/go-junit-report && cat ${results_dir}/${longhaul_result_name}${totalruns}.log | go-junit-report > ${results_dir}/LongHauljunit${totalruns}.xml" + + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + sleep ${test_intervel}m + done +} + +rook_infra::try_copy_docker_image_to_rook_infra() { + local tag_name=$1 + + echo Searching local docker registry for ${tag_name} + export imageId=$(docker images -q ${tag_name}) + + if [ -z "$imageId" ]; then + echo Image not found + else + echo Image found... + + echo Archiving the docker image + docker save -o ${temp_archive} ${tag_name} + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + echo success... + + echo Copying archived image to rook-infra + docker cp forRookImage.tar ${id}:/${temp_archive} + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + echo success... + + echo Importing image into rook-infra image registry + docker exec ${id} /bin/bash -c "docker load -i /${temp_archive}" + rc=$?; if [[ $rc != 0 ]]; then set -e; fi + echo success... + fi +} + +rook_infra::gather_results() { + echo Gathering results... + + #install junit parser && create junit results + docker exec ${id} /bin/bash -c \ + "go get -u -f github.com/jstemmer/go-junit-report && cat ${results_dir}/test.log | go-junit-report > ${results_dir}/junit.xml" + + rc=$?; if [[ $rc != 0 ]]; then set -e; fi +} + +rook_infra::cleanup() { + local tag_name=$1 + local rook_platform=$2 + local k8s_version=$3 + + echo Removing rook-test-framework container and images... + docker kill ${id} || true + docker rm ${id} || true + docker images|grep 'rook-test\|kubeadm-dind-cluster\|ceph/base'|xargs docker rmi > /dev/null 2>&1 || true + docker volume ls -qf dangling=true | xargs -r docker volume rm > /dev/null 2>&1|| true + +} + + tag_name=${1:-"master-latest"} + rook_platform=${2:-"Kubernetes"} + k8s_version=${3:-"v1.6"} + total_runs=${4:-50} + total_per_runs=${5:-5} + parallel_runs=${6:-20} + test_intervel=${7:-10} + + + +{ #try + + rook_infra::create + rook_infra::try_copy_docker_image_to_rook_infra quay.io/rook/rookd:${tag_name} + rook_infra::try_copy_docker_image_to_rook_infra quay.io/rook/rook:${tag_name} + rook_infra::init + + rook_infra::run_test TestBlockStorageSmokeSuite ${tag_name} ${rook_platform} ${k8s_version} + + rook_infra::gather_results + +} || { #catch + rook_infra::gather_results +} +echo "Starting long haul test" +a=$SECONDS +rook_infra::run_longhul Test ${total_runs} ${total_per_runs} ${parallel_runs} ${test_intervel} +elapsedseconds=$(( SECONDS - a )) + +echo " Long haul Test completed - Ran for $elapsedseconds seconds" + + +rook_infra::cleanup ${tag_name} ${rook_platform} ${k8s_version} \ No newline at end of file diff --git a/e2e/tests/baseSetup.go b/e2e/tests/baseSetup.go index c5c4c43f4..bec1fbeb6 100644 --- a/e2e/tests/baseSetup.go +++ b/e2e/tests/baseSetup.go @@ -9,23 +9,23 @@ import ( ) var ( - env objects.EnvironmentManifest + Env objects.EnvironmentManifest Platform enums.RookPlatformType ) //One init function per package - initializes Rook infra and installs rook(if needed based on flags) func init() { - env = objects.NewManifest() + Env = objects.NewManifest() var err error - rookPlatform, err := enums.GetRookPlatFormTypeFromString(env.Platform) + rookPlatform, err := enums.GetRookPlatFormTypeFromString(Env.Platform) if err != nil { panic(fmt.Errorf("Cannot get platform", err)) } - k8sVersion, _ := enums.GetK8sVersionFromString(env.K8sVersion) + k8sVersion, _ := enums.GetK8sVersionFromString(Env.K8sVersion) - rookTag := env.RookTag + rookTag := Env.RookTag if err != nil { panic(fmt.Errorf("Rook Tag is required", err)) } @@ -34,7 +34,7 @@ func init() { if err != nil { panic(fmt.Errorf("Error during Rook Infra Setup", err)) } - skipRookInstall := strings.EqualFold(env.SkipInstallRook, "true") + skipRookInstall := strings.EqualFold(Env.SkipInstallRook, "true") rookInfra.ValidateAndSetupTestPlatform(skipRookInstall) err = rookInfra.InstallRook(rookTag, skipRookInstall) diff --git a/e2e/tests/block/create_block_test.go b/e2e/tests/block/create_block_test.go index a84b16076..3a1f5298f 100644 --- a/e2e/tests/block/create_block_test.go +++ b/e2e/tests/block/create_block_test.go @@ -51,7 +51,6 @@ func (s *BlockImageCreateSuite) TestCreatingNewBlockImageOnDefaultPool() { require.Contains(s.T(), cbi, "succeeded created image") b, _ := s.rc.GetBlockImages() require.Equal(s.T(), s.init_blockCount+1, len(b), "Make sure new block image is created") - s.T().Log("Test Passed") } diff --git a/e2e/tests/block/k8s/longhaul/basicBlockonghaul_test.go b/e2e/tests/block/k8s/longhaul/basicBlockonghaul_test.go new file mode 100644 index 000000000..d984ae56f --- /dev/null +++ b/e2e/tests/block/k8s/longhaul/basicBlockonghaul_test.go @@ -0,0 +1,135 @@ +package longhaul + +import ( + "bytes" + "github.com/rook/rook/e2e/framework/clients" + "github.com/rook/rook/e2e/framework/contracts" + "github.com/rook/rook/e2e/framework/objects" + "github.com/rook/rook/e2e/framework/utils" + "github.com/rook/rook/e2e/tests" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "html/template" + "sync" + "testing" +) + +// Rook Block Storage integration test +// Start MySql database that is using rook provisoned block storage. +// Make sure database is functional + +func TestK8sBlockLongHaul(t *testing.T) { + suite.Run(t, new(K8sBlockLongHaulSuite)) +} + +type K8sBlockLongHaulSuite struct { + suite.Suite + testClient *clients.TestClient + bc contracts.BlockOperator + kh *utils.K8sHelper + init_blockCount int + storageClass_Path string + mysqlApp_Path string + db *utils.MySqlHelper + wg sync.WaitGroup +} + +//Test set up - does the following in order +//create pool and storage class, create a PVC, Create a MySQL app/service that uses pvc +func (s *K8sBlockLongHaulSuite) SetupSuite() { + var err error + + s.testClient, err = clients.CreateTestClient(tests.Platform) + require.Nil(s.T(), err) + + s.bc = s.testClient.GetBlockClient() + s.kh = utils.CreatK8sHelper() + initialBlocks, err := s.bc.BlockList() + require.Nil(s.T(), err) + s.init_blockCount = len(initialBlocks) + s.storageClass_Path = "../../../../data/block/storageclass_pool.tmpl" + s.mysqlApp_Path = "../../../../data/integration/mysqlapp.yaml" + + //create storage class + if scp, _ := s.kh.IsStorageClassPresent("rook-block"); !scp { + _, _, scs := s.storageClassOperation("mysql-pool", "create") + require.Equal(s.T(), 0, scs) + + //make sure storageclass is created + present, err := s.kh.IsStorageClassPresent("rook-block") + require.Nil(s.T(), err) + require.True(s.T(), present, "Make sure storageclass is present") + } + //create mysql pod + if _, err := s.kh.GetPVCStatus("mysql-pv-claim"); err != nil { + + s.kh.ResourceOperation("create", s.mysqlApp_Path) + + //wait till mysql pod is up + require.True(s.T(), s.kh.IsPodInExpectedState("mysqldb", "", "Running")) + pvcStatus, err := s.kh.GetPVCStatus("mysql-pv-claim") + require.Nil(s.T(), err) + require.Contains(s.T(), pvcStatus, "Bound") + } + dbIp, err := s.kh.GetPodHostId("mysqldb", "") + require.Nil(s.T(), err) + //create database connection + s.db = utils.CreateNewMySqlHelper("mysql", "mysql", dbIp+":30003", "sample") + + require.True(s.T(), s.db.PingSuccess()) + + if exist := s.db.TableExists(); !exist { + s.db.CreateTable() + } + +} + +func (s *K8sBlockLongHaulSuite) TestBlockLonghaulRun() { + + s.wg.Add(tests.Env.LoadConcurrentRuns) + for i := 1; i <= tests.Env.LoadConcurrentRuns; i++ { + go s.dbOperation(i) + } + s.wg.Wait() +} + +func (s *K8sBlockLongHaulSuite) dbOperation(i int) { + defer s.wg.Done() + //InsertRandomData + s.db.InsertRandomData() + s.db.InsertRandomData() + s.db.InsertRandomData() + s.db.InsertRandomData() + s.db.InsertRandomData() + s.db.InsertRandomData() + + //delete Data + s.db.DeleteRandomRow() + +} +func (s *K8sBlockLongHaulSuite) TearDownSuite() { + s.db.CloseConnection() + s.testClient = nil + s.bc = nil + s.kh = nil + s.db = nil + s = nil +} +func (s *K8sBlockLongHaulSuite) storageClassOperation(poolName string, action string) (string, string, int) { + + t, _ := template.ParseFiles(s.storageClass_Path) + + var tpl bytes.Buffer + config := map[string]string{ + "poolName": poolName, + } + + t.Execute(&tpl, config) + + cmdStruct := objects.CommandArgs{Command: "kubectl", PipeToStdIn: tpl.String(), CmdArgs: []string{action, "-f", "-"}} + + cmdOut := utils.ExecuteCommand(cmdStruct) + + return cmdOut.StdOut, cmdOut.StdErr, cmdOut.ExitCode + +} diff --git a/e2e/tests/cleanup_test_infra_test.go b/e2e/tests/cleanup_test_infra_test.go index 280331d3c..5ea67d427 100644 --- a/e2e/tests/cleanup_test_infra_test.go +++ b/e2e/tests/cleanup_test_infra_test.go @@ -6,6 +6,7 @@ import ( "github.com/rook/rook/e2e/framework/manager" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "k8s.io/kubernetes/pkg/util/env" "testing" ) @@ -23,15 +24,15 @@ type CleanUpTestSuite struct { func (suite *CleanUpTestSuite) TestRookInfraCleanUpTest() { var err error - suite.rookPlatform, err = enums.GetRookPlatFormTypeFromString(env.Platform) + suite.rookPlatform, err = enums.GetRookPlatFormTypeFromString(Env.Platform) require.Nil(suite.T(), err) - suite.k8sVersion, err = enums.GetK8sVersionFromString(env.K8sVersion) + suite.k8sVersion, err = enums.GetK8sVersionFromString(Env.K8sVersion) require.Nil(suite.T(), err) - suite.rookTag = env.RookTag + suite.rookTag = Env.RookTag require.NotEmpty(suite.T(), suite.rookTag, "RookTag parameter is required")