Merge pull request #738 from dangula/blockLongHaul

adding long haul test for block .

e2e/README.md

@@ -35,6 +35,8 @@ rook_platform| platform rook needs to be installed on  | kubernetes | kubernetes
 k8s_version  | version of Kubernetes to be installed  | v1.6  | v1.6
 rook_version | rook version/tag to be installed | valid rook tag |master-latest 
 skip_install_rook | skips installing rook (if already installed) | true or false  | false
+load_parallel_runs | performs concurrent operations (optional) | any number | 20
+count| go test paramater to repeat tests| any number | 1
 
 if install_rook flag is set to false, then all the other flags are ignored,
 and tests are run without rook being installed and set up. Use this flag to run tests against
@@ -54,15 +56,25 @@ are run. e.g.
 go test -run TestObjectStorageSmokeSuite github.com/rook/rook/e2e/tests/smoke --rook_version=v3.0.1
 ```
 
-#### Run Tests on pre-installed/exisitng Rook
+#### Run Tests on pre-installed/existing Rook
 To run tests against a pre-installed rook set  ```--skip_install_rook=true``` . 
 Setting skip_install_rook to true will ignore all other flags and run tests without installing rook first.
 
 e.g.
 ```
-go test -run TestFileStorageSmokeSuite github.com/rook/rook/e2e/tests/smoke --skip_install_rook=true
+go test -run TestFileSystemSmokeSuite github.com/rook/rook/e2e/tests/smoke --skip_install_rook=true
 ```
-
+#### Run Longhaul Tests
+Using go test `-count` option, any tests can be repeated n number of times to simulate load or longhaul test. Although 
+any test can be converted to a longhaul test, it's ideal to run integration tests under load for extended period to simulate 
+real world scenarios. Also a new custom test flag `load_parallel_runs` is added to control the number of concurrent operations being performed.
+For example look at the [block long haul test](/e2e/tests/block/k8s/longhaul/basicBlockonghaul_test.go)
+ 
+ To run a longhaul test you can run any integration test with `-count` and `--load_parallel_runs` options
+ e.g.
+ ```
+ go test -run TestK8sBlockLongHaul github.com/rook/rook/e2e/tests/block/k8s/longhaul --load_parallel_runs=20 -count=1000
+ ```
 
 Prerequisites :
 * Go installed and GO_PATH set up

e2e/framework/clients/rest_client.go

@@ -8,8 +8,10 @@ import (
 	"github.com/rook/rook/e2e/framework/utils"
 	"github.com/rook/rook/pkg/model"
 	rclient "github.com/rook/rook/pkg/rook/client"
+	"net"
 	"os"
 	"path/filepath"
+	"time"
 )
 
 type RestAPIClient struct {
@@ -38,8 +40,19 @@ func CreateRestAPIClient(platform enums.RookPlatformType) *RestAPIClient {
 	default:
 		panic(fmt.Errorf("platfrom type %s not yet supported", platform))
 	}
-
-	client := rclient.NewRookNetworkRestClient(endpoint, http.DefaultClient)
+	httpclient := &http.Client{
+		Transport: &http.Transport{
+			Dial: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 0,
+			}).Dial,
+			DisableKeepAlives:     true,
+			DisableCompression:    true,
+			MaxIdleConnsPerHost:   1,
+			ResponseHeaderTimeout: 30 * time.Second,
+		},
+	}
+	client := rclient.NewRookNetworkRestClient(endpoint, httpclient)
 
 	return &RestAPIClient{client}
 }

e2e/framework/objects/environment_manifest.go

@@ -5,10 +5,11 @@ import (
 )
 
 type EnvironmentManifest struct {
-	K8sVersion      string
-	Platform        string
-	RookTag         string
-	SkipInstallRook string
+	K8sVersion         string
+	Platform           string
+	RookTag            string
+	SkipInstallRook    string
+	LoadConcurrentRuns int
 }
 
 func NewManifest() EnvironmentManifest {
@@ -18,6 +19,7 @@ func NewManifest() EnvironmentManifest {
 	flag.StringVar(&e.Platform, "rook_platform", "Kubernetes", "Platform to install rook on; Kubernetes is the only platform currently supported")
 	flag.StringVar(&e.RookTag, "rook_version", "master-latest", "Docker tag of the rook operator to install, must be in quay.io or local environment")
 	flag.StringVar(&e.SkipInstallRook, "skip_install_rook", "false", "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed")
+	flag.IntVar(&e.LoadConcurrentRuns, "load_parallel_runs", 20, "number of routines for load test")
 	flag.Parse()
 
 	return e

e2e/framework/utils/mysql_helper.go

@@ -23,18 +23,20 @@ func CreateNewMySqlHelper(username string, password string, url string, dbname s
 
 	return &MySqlHelper{DB: db}
 }
-
+func (h *MySqlHelper) CloseConnection() {
+	h.DB.Close()
+}
 func (h *MySqlHelper) PingSuccess() bool {
 	inc := 0
 
-	for inc < 10 {
+	for inc < 30 {
 		err := h.DB.Ping()
 
 		if err == nil {
 			return true
 		}
 		inc++
-		time.Sleep(1 * time.Second)
+		time.Sleep(3 * time.Second)
 	}
 
 	return false

e2e/scripts/k8sBlockLongHaul.sh

@@ -0,0 +1,181 @@
+#!/bin/bash -e
+
+test_source_repo=$(pwd)
+docker_test_repo=/workspace/go/src/github.com/rook/rook
+git_smoke_test_directory=github.com/rook/rook/e2e/tests/smoke
+git_longhaul_test_directory=github.com/rook/rook/e2e/tests/block/k8s/longhaul
+git_test_directory=github.com/rook/rook/e2e/tests
+container_image=quay.io/rook/rook-test
+tmp_docker_sock_path=/var/run/docker.sock
+results_dir=results
+results_filename=test.log
+longhaul_result_name=longhaul
+temp_archive=forRookImage.tar
+
+#create the rook infrastructure container
+rook_infra::create() {
+    export id=$(docker run \
+        -d \
+        -e GOPATH=/workspace/go \
+        -e GOROOT=/usr/lib/go \
+        --privileged \
+        --security-opt=seccomp:unconfined \
+        -v /lib/modules:/lib/modules \
+        -v /sbin/modprobe:/sbin/modprobe \
+        -v /sys/bus:/sys/bus \
+        -v /dev:/dev \
+        -v /var/run/docker.sock:${tmp_docker_sock_path} \
+        -v ${test_source_repo}:${docker_test_repo} \
+        -w ${docker_test_repo}/e2e \
+        ${container_image} \
+        /sbin/init)
+
+    rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+
+    local attempt_num=1
+    local max_attempts=10
+
+    until [[ $(docker inspect -f {{.State.Running}} $id) == "true" ]]
+    do
+        if (( attempt_num == max_attempts ))
+        then
+            echo "rook-test infra container failed to start"
+            return 1
+        else
+            sleep 1
+            attempt_num++
+
+        fi
+    done
+
+    echo "rook-test container is running..."
+}
+
+#prepare the rook_infra container to run tests
+rook_infra::init() {
+    echo Creating results directory
+    docker exec ${id} mkdir -p ${results_dir}
+    rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+
+    echo Installing rook-test-framework dependencies...
+    docker exec ${id} /bin/bash -c \
+    "glide install"
+    rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+    echo Success...
+}
+
+rook_infra::run_test() {
+    local test_name_regex=$1
+    local tag_name=$2
+    local rook_platform=$3
+    local k8s_version=$4
+
+    docker exec -t ${id} /bin/bash -c \
+        "go test -timeout 1200s -run ${test_name_regex} ${git_smoke_test_directory} --rook_platform=${rook_platform} --k8s_version=${k8s_version} --rook_version=${tag_name} -v 2>&1 | tee ${results_dir}/${results_filename}"
+
+    rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+}
+
+rook_infra::run_longhul() {
+    local test_name_regex=$1
+    local total_runs=$2
+    local test_runs=$3
+    local parallel_runs=$4
+    local test_intervel=$5
+    for (( totalruns=1; totalruns<=${total_runs}; totalruns++ ))
+	do
+
+   		docker exec -t ${id} /bin/bash -c \
+        	"go test -run ${test_name_regex} ${git_longhaul_test_directory} --skip_install_rook=true --load_parallel_runs=${parallel_runs} -count=${test_runs} -v 2>&1 | tee ${results_dir}/${longhaul_result_name}${totalruns}.log"
+         docker exec ${id} /bin/bash -c \
+        	"go get -u -f github.com/jstemmer/go-junit-report && cat ${results_dir}/${longhaul_result_name}${totalruns}.log | go-junit-report > ${results_dir}/LongHauljunit${totalruns}.xml"
+
+    	rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+    	sleep ${test_intervel}m
+	done
+}
+
+rook_infra::try_copy_docker_image_to_rook_infra() {
+    local tag_name=$1
+
+    echo Searching local docker registry for ${tag_name}
+    export imageId=$(docker images -q ${tag_name})
+
+    if [ -z "$imageId" ]; then
+        echo Image not found
+    else
+        echo Image found...
+
+        echo Archiving the docker image
+        docker save -o ${temp_archive} ${tag_name}
+        rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+        echo success...
+
+        echo Copying archived image to rook-infra
+        docker cp forRookImage.tar ${id}:/${temp_archive}
+        rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+        echo success...
+
+        echo Importing image into rook-infra image registry
+        docker exec ${id} /bin/bash -c "docker load -i /${temp_archive}"
+        rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+        echo success...
+    fi
+}
+
+rook_infra::gather_results() {
+    echo Gathering results...
+
+    #install junit parser && create junit results
+    docker exec ${id} /bin/bash -c \
+        "go get -u -f github.com/jstemmer/go-junit-report && cat ${results_dir}/test.log | go-junit-report > ${results_dir}/junit.xml"
+
+    rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+}
+
+rook_infra::cleanup() {
+    local tag_name=$1
+    local rook_platform=$2
+    local k8s_version=$3
+
+    echo Removing rook-test-framework container and images...
+    docker kill ${id} || true
+    docker rm ${id} || true
+    docker images|grep 'rook-test\|kubeadm-dind-cluster\|ceph/base'|xargs docker rmi > /dev/null 2>&1 || true
+    docker volume ls -qf dangling=true | xargs -r docker volume rm > /dev/null 2>&1|| true
+
+}
+
+ tag_name=${1:-"master-latest"}
+ rook_platform=${2:-"Kubernetes"}
+ k8s_version=${3:-"v1.6"}
+ total_runs=${4:-50}
+ total_per_runs=${5:-5}
+ parallel_runs=${6:-20}
+ test_intervel=${7:-10}
+
+
+
+{ #try
+
+    rook_infra::create
+    rook_infra::try_copy_docker_image_to_rook_infra quay.io/rook/rookd:${tag_name}
+    rook_infra::try_copy_docker_image_to_rook_infra quay.io/rook/rook:${tag_name}
+    rook_infra::init
+
+    rook_infra::run_test TestBlockStorageSmokeSuite ${tag_name} ${rook_platform} ${k8s_version}
+
+    rook_infra::gather_results
+
+} || { #catch
+    rook_infra::gather_results
+}
+echo "Starting long haul test"
+a=$SECONDS
+rook_infra::run_longhul Test ${total_runs} ${total_per_runs} ${parallel_runs}  ${test_intervel}
+elapsedseconds=$(( SECONDS - a ))
+
+echo " Long haul Test completed - Ran for $elapsedseconds seconds"
+
+
+rook_infra::cleanup ${tag_name} ${rook_platform} ${k8s_version}

e2e/tests/baseSetup.go

@@ -9,23 +9,23 @@ import (
 )
 
 var (
-	env      objects.EnvironmentManifest
+	Env      objects.EnvironmentManifest
 	Platform enums.RookPlatformType
 )
 
 //One init function per package - initializes Rook infra and installs rook(if needed based on flags)
 func init() {
-	env = objects.NewManifest()
+	Env = objects.NewManifest()
 	var err error
 
-	rookPlatform, err := enums.GetRookPlatFormTypeFromString(env.Platform)
+	rookPlatform, err := enums.GetRookPlatFormTypeFromString(Env.Platform)
 	if err != nil {
 		panic(fmt.Errorf("Cannot get platform", err))
 	}
 
-	k8sVersion, _ := enums.GetK8sVersionFromString(env.K8sVersion)
+	k8sVersion, _ := enums.GetK8sVersionFromString(Env.K8sVersion)
 
-	rookTag := env.RookTag
+	rookTag := Env.RookTag
 	if err != nil {
 		panic(fmt.Errorf("Rook Tag is required", err))
 	}
@@ -34,7 +34,7 @@ func init() {
 	if err != nil {
 		panic(fmt.Errorf("Error during Rook Infra Setup", err))
 	}
-	skipRookInstall := strings.EqualFold(env.SkipInstallRook, "true")
+	skipRookInstall := strings.EqualFold(Env.SkipInstallRook, "true")
 	rookInfra.ValidateAndSetupTestPlatform(skipRookInstall)
 
 	err = rookInfra.InstallRook(rookTag, skipRookInstall)

e2e/tests/block/create_block_test.go

@@ -51,7 +51,6 @@ func (s *BlockImageCreateSuite) TestCreatingNewBlockImageOnDefaultPool() {
 	require.Contains(s.T(), cbi, "succeeded created image")
 	b, _ := s.rc.GetBlockImages()
 	require.Equal(s.T(), s.init_blockCount+1, len(b), "Make sure new block image is created")
-	s.T().Log("Test Passed")
 
 }
 

e2e/tests/block/k8s/longhaul/basicBlockonghaul_test.go

@@ -0,0 +1,135 @@
+package longhaul
+
+import (
+	"bytes"
+	"github.com/rook/rook/e2e/framework/clients"
+	"github.com/rook/rook/e2e/framework/contracts"
+	"github.com/rook/rook/e2e/framework/objects"
+	"github.com/rook/rook/e2e/framework/utils"
+	"github.com/rook/rook/e2e/tests"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"html/template"
+	"sync"
+	"testing"
+)
+
+// Rook Block Storage integration test
+// Start MySql database that is using rook provisoned block storage.
+// Make sure database is functional
+
+func TestK8sBlockLongHaul(t *testing.T) {
+	suite.Run(t, new(K8sBlockLongHaulSuite))
+}
+
+type K8sBlockLongHaulSuite struct {
+	suite.Suite
+	testClient        *clients.TestClient
+	bc                contracts.BlockOperator
+	kh                *utils.K8sHelper
+	init_blockCount   int
+	storageClass_Path string
+	mysqlApp_Path     string
+	db                *utils.MySqlHelper
+	wg                sync.WaitGroup
+}
+
+//Test set up - does the following in order
+//create pool and storage class, create a PVC, Create a MySQL app/service that uses pvc
+func (s *K8sBlockLongHaulSuite) SetupSuite() {
+	var err error
+
+	s.testClient, err = clients.CreateTestClient(tests.Platform)
+	require.Nil(s.T(), err)
+
+	s.bc = s.testClient.GetBlockClient()
+	s.kh = utils.CreatK8sHelper()
+	initialBlocks, err := s.bc.BlockList()
+	require.Nil(s.T(), err)
+	s.init_blockCount = len(initialBlocks)
+	s.storageClass_Path = "../../../../data/block/storageclass_pool.tmpl"
+	s.mysqlApp_Path = "../../../../data/integration/mysqlapp.yaml"
+
+	//create storage class
+	if scp, _ := s.kh.IsStorageClassPresent("rook-block"); !scp {
+		_, _, scs := s.storageClassOperation("mysql-pool", "create")
+		require.Equal(s.T(), 0, scs)
+
+		//make sure storageclass is created
+		present, err := s.kh.IsStorageClassPresent("rook-block")
+		require.Nil(s.T(), err)
+		require.True(s.T(), present, "Make sure storageclass is present")
+	}
+	//create mysql pod
+	if _, err := s.kh.GetPVCStatus("mysql-pv-claim"); err != nil {
+
+		s.kh.ResourceOperation("create", s.mysqlApp_Path)
+
+		//wait till mysql pod is up
+		require.True(s.T(), s.kh.IsPodInExpectedState("mysqldb", "", "Running"))
+		pvcStatus, err := s.kh.GetPVCStatus("mysql-pv-claim")
+		require.Nil(s.T(), err)
+		require.Contains(s.T(), pvcStatus, "Bound")
+	}
+	dbIp, err := s.kh.GetPodHostId("mysqldb", "")
+	require.Nil(s.T(), err)
+	//create database connection
+	s.db = utils.CreateNewMySqlHelper("mysql", "mysql", dbIp+":30003", "sample")
+
+	require.True(s.T(), s.db.PingSuccess())
+
+	if exist := s.db.TableExists(); !exist {
+		s.db.CreateTable()
+	}
+
+}
+
+func (s *K8sBlockLongHaulSuite) TestBlockLonghaulRun() {
+
+	s.wg.Add(tests.Env.LoadConcurrentRuns)
+	for i := 1; i <= tests.Env.LoadConcurrentRuns; i++ {
+		go s.dbOperation(i)
+	}
+	s.wg.Wait()
+}
+
+func (s *K8sBlockLongHaulSuite) dbOperation(i int) {
+	defer s.wg.Done()
+	//InsertRandomData
+	s.db.InsertRandomData()
+	s.db.InsertRandomData()
+	s.db.InsertRandomData()
+	s.db.InsertRandomData()
+	s.db.InsertRandomData()
+	s.db.InsertRandomData()
+
+	//delete Data
+	s.db.DeleteRandomRow()
+
+}
+func (s *K8sBlockLongHaulSuite) TearDownSuite() {
+	s.db.CloseConnection()
+	s.testClient = nil
+	s.bc = nil
+	s.kh = nil
+	s.db = nil
+	s = nil
+}
+func (s *K8sBlockLongHaulSuite) storageClassOperation(poolName string, action string) (string, string, int) {
+
+	t, _ := template.ParseFiles(s.storageClass_Path)
+
+	var tpl bytes.Buffer
+	config := map[string]string{
+		"poolName": poolName,
+	}
+
+	t.Execute(&tpl, config)
+
+	cmdStruct := objects.CommandArgs{Command: "kubectl", PipeToStdIn: tpl.String(), CmdArgs: []string{action, "-f", "-"}}
+
+	cmdOut := utils.ExecuteCommand(cmdStruct)
+
+	return cmdOut.StdOut, cmdOut.StdErr, cmdOut.ExitCode
+
+}

e2e/tests/cleanup_test_infra_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/rook/rook/e2e/framework/manager"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	"k8s.io/kubernetes/pkg/util/env"
 	"testing"
 )
 
@@ -23,15 +24,15 @@ type CleanUpTestSuite struct {
 func (suite *CleanUpTestSuite) TestRookInfraCleanUpTest() {
 	var err error
 
-	suite.rookPlatform, err = enums.GetRookPlatFormTypeFromString(env.Platform)
+	suite.rookPlatform, err = enums.GetRookPlatFormTypeFromString(Env.Platform)
 
 	require.Nil(suite.T(), err)
 
-	suite.k8sVersion, err = enums.GetK8sVersionFromString(env.K8sVersion)
+	suite.k8sVersion, err = enums.GetK8sVersionFromString(Env.K8sVersion)
 
 	require.Nil(suite.T(), err)
 
-	suite.rookTag = env.RookTag
+	suite.rookTag = Env.RookTag
 
 	require.NotEmpty(suite.T(), suite.rookTag, "RookTag parameter is required")