Merge pull request #1398 from dangula/pr_perfTest_enhancements

adding test flags to control load size and test rook resilience

tests/README.md

@@ -80,16 +80,10 @@ In addition to standard go tests parameters, the following custom parameters are
  Parameter | Description | Possible values | Default
  --- |--- | --- | ---
 rook_platform| platform rook needs to be installed on  | kubernetes | kubernetes
-k8s_version  | version of Kubernetes to be installed  | v1.5+  | v1.7.5
+k8s_version  | version of Kubernetes to be installed  | v1.6+  | v1.8.5
 rook_image | rook image name to be installed | valid image name | rook/rook
 toolbox_image | toolbox image name to be installed | valid image name | rook/toolbox
 skip_install_rook | skips installing rook (if already installed) | true or false  | false
-load_parallel_runs | performs concurrent operations (optional used for load test) | any number | 20
-
-If the `install_rook` flag is set to false, then all the other flags are ignored
-and tests are run without rook being installed and setup. Use this flag to run tests against
-a pre-installed/configured rook.
-
 
 ### Running Tests with parameters.
 
@@ -111,24 +105,34 @@ go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration
 
 ##### To run specific without installing rook
 ```
-go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration --skip_install_rook=true
+go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration --skip_install_rook
 ```
 If the `skip_install_rook` flag is set to true, then rook is not uninstalled either. 
 
 #### Run Longhaul Tests
-Using the `go test -count n` option, any tests can be repeated `n` number of times to simulate load or longhaul test. Although 
-any test can be converted to a longhaul test, it's ideal to run integration tests under load for an extended period. Also a new custom test flag `load_parallel_runs` is added to control the number of concurrent operations being performed.
-For example look at the [block long haul test](/tests/block/k8s/longhaul/basicBlockonghaul_test.go)
- 
- To run a longhaul test you can run any integration test with `-count` and `--load_parallel_runs` options
- e.g.
+[Longhaul](/tests/block/k8s/longhaul) tests are integration tests that run for extended period of time. A load profile can be configured
+using the following load test flags 
+
+Parameter | Description | Possible values | Default
+ --- |--- | --- | ---
+load_parallel_runs | performs concurrent operations | any number | 20
+load_volumes | number of volumes | >1 | 1
+load_time | number of seconds to run  | >1 | 1800
+load_size | size of load profile (3M, 10M, or 50M per thread) | small, medium, or large | medium
+enable_chaos | kill random pods in rook cluster | true or false | false
+
+  e.g.
  ```
- go test -run TestK8sBlockLongHaul github.com/rook/rook/tests/longhaul --load_parallel_runs=20 -count=1000
+ go test -run TestObjectLongHaul github.com/rook/rook/tests/longhaul --load_parallel_runs=20 --load_time 1800 --load_size small --load_volumes 3
  ```
  The Longhaul test just like other test is going to install rook if it's not already installed, but it is not going to clean up test data or uninstall rook after the run. 
  Longhaul test is designed to run multiple times on the same setup and installation of rook to tests its stability. Test Data and rook should be cleaned up manually after the test.
  
  
+ You can measure memory, cpu, IOPS, throughput, and other settings on a cluster using Prometheus. The metrics collected during load test can be visualize using Grafana. 
+ Here a couple of helpful links to get prometheus and grafana started and collect metrics: 
+ [kube-prometheus](https://github.com/coreos/prometheus-operator/tree/master/contrib/kube-prometheus) and  [cluster-deploy-script](https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/hack/cluster-monitoring/deploy)
+ 
 
 Prerequisites :
 * Go installed and GO_PATH set

tests/framework/installer/install_helper.go

@@ -206,8 +206,7 @@ func SystemNamespace(namespace string) string {
 func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType, dataDirHostPath string, helmInstalled, useDevices bool, mons int) (bool, error) {
 	var err error
 	//flag used for local debuggin purpose, when rook is pre-installed
-	skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true")
-	if skipRookInstall {
+	if h.Env.SkipInstallRook {
 		return true, nil
 	}
 
@@ -270,8 +269,7 @@ func (h *InstallHelper) UninstallRook(helmInstalled bool, namespace string) {
 //UninstallRookFromK8s uninstalls rook from multiple namespaces in k8s
 func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNamespace string, namespaces ...string) {
 	//flag used for local debugging purpose, when rook is pre-installed
-	skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true")
-	if skipRookInstall {
+	if h.Env.SkipInstallRook {
 		return
 	}
 

tests/framework/objects/environment_manifest.go

@@ -25,10 +25,12 @@ type EnvironmentManifest struct {
 	HostType           string
 	RookImageName      string
 	ToolboxImageName   string
-	SkipInstallRook    string
+	SkipInstallRook    bool
 	LoadVolumeNumber   int
 	LoadConcurrentRuns int
 	LoadTime           int
+	LoadSize           string
+	EnableChaos        bool
 }
 
 var Env EnvironmentManifest
@@ -38,8 +40,10 @@ func init() {
 	flag.StringVar(&Env.HostType, "host_type", "localhost", "Host were tests are run eg - localhost,GCE or AWS")
 	flag.StringVar(&Env.RookImageName, "rook_image", "rook/rook", "Docker image name for the rook container to install, must be in docker hub or local environment")
 	flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/toolbox", "Docker image name of the toolbox container to install, must be in docker hub or local environment")
-	flag.StringVar(&Env.SkipInstallRook, "skip_install_rook", "false", "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed")
+	flag.BoolVar(&Env.SkipInstallRook, "skip_install_rook", false, "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed")
 	flag.IntVar(&Env.LoadConcurrentRuns, "load_parallel_runs", 20, "number of routines for load test")
 	flag.IntVar(&Env.LoadVolumeNumber, "load_volumes", 1, "number of volumes(file,object or block) to be created for load test")
 	flag.IntVar(&Env.LoadTime, "load_time", 1800, "number of seconds each thread perform operations for")
+	flag.StringVar(&Env.LoadSize, "load_size", "medium", "load size for each thread performing operations - small,medium or large.")
+	flag.BoolVar(&Env.EnableChaos, "enable_chaos", false, "used to determine if random pods in a namespace are to be killed during load test.")
 }

tests/framework/utils/k8s_helper.go

@@ -318,6 +318,18 @@ func (k8sh *K8sHelper) GetMonitorServices(namespace string) (map[string]string,
 	}, nil
 }
 
+func (k8sh *K8sHelper) IsPodWithLabelPresent(label string, namespace string) bool {
+	options := metav1.ListOptions{LabelSelector: label}
+	pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(options)
+	if errors.IsNotFound(err) {
+		return false
+	}
+	if len(pods.Items) == 0 {
+		return false
+	}
+	return true
+}
+
 //IsPodWithLabelRunning returns true if a Pod is running status or goes to Running status within 90s else returns false
 func (k8sh *K8sHelper) IsPodWithLabelRunning(label string, namespace string) bool {
 	options := metav1.ListOptions{LabelSelector: label}

tests/framework/utils/mysql_helper.go

@@ -67,7 +67,7 @@ func (h *MySQLHelper) PingSuccess() bool {
 
 //CreateTable func create sample Table
 func (h *MySQLHelper) CreateTable() sql.Result {
-	result, err := h.DB.Exec("CREATE TABLE LONGHAUL (id int NOT NULL AUTO_INCREMENT,number int,data varchar(32)" +
+	result, err := h.DB.Exec("CREATE TABLE LONGHAUL (id int NOT NULL AUTO_INCREMENT,number int,data1 varchar(10000),data2 varchar(10000),data3 varchar(10000),data4 varchar(10000),data5 varchar(10000)" +
 		", PRIMARY KEY (id) )")
 	if err != nil {
 		panic(err)
@@ -76,14 +76,14 @@ func (h *MySQLHelper) CreateTable() sql.Result {
 }
 
 //InsertRandomData Inserts random Data into the table
-func (h *MySQLHelper) InsertRandomData() sql.Result {
-	stmtIns, err := h.DB.Prepare("INSERT INTO LONGHAUL (number, data) VALUES ( ?, ? )") // ? = placeholder
+func (h *MySQLHelper) InsertRandomData(dataSize int) sql.Result {
+	stmtIns, err := h.DB.Prepare("INSERT INTO LONGHAUL (number, data1, data2, data3, data4, data5) VALUES ( ?, ?, ?, ?, ? )") // ? = placeholder
 	if err != nil {
 		panic(err)
 	}
 	defer stmtIns.Close()
 
-	result, err := stmtIns.Exec(rand.Intn(100000000), fake.CharactersN(20))
+	result, err := stmtIns.Exec(rand.Intn(100000000), fake.CharactersN(dataSize), fake.CharactersN(dataSize), fake.CharactersN(dataSize), fake.CharactersN(dataSize), fake.CharactersN(dataSize))
 	if err != nil {
 		panic(err)
 	}

tests/longhaul/base_block_test.go

@@ -1,12 +1,11 @@
 package longhaul
 
 import (
+	"strings"
 	"sync"
 	"testing"
-
 	"time"
 
-	"github.com/coreos/pkg/capnslog"
 	"github.com/rook/rook/tests/framework/clients"
 	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
@@ -14,7 +13,6 @@ import (
 )
 
 var (
-	logger           = capnslog.NewPackageLogger("github.com/rook/rook", "longhaul")
 	defaultNamespace = "default"
 )
 
@@ -22,9 +20,6 @@ var (
 func createStorageClassAndPool(t func() *testing.T, kh *utils.K8sHelper, namespace string, storageClassName string, poolName string) {
 	//create storage class
 	if scp, _ := kh.IsStorageClassPresent(storageClassName); !scp {
-
-		installer.BlockResourceOperation(kh, installer.GetBlockStorageClassDef(poolName, storageClassName, namespace), "create")
-
 		logger.Infof("Install pool and storage class for rook block")
 		_, err := installer.BlockResourceOperation(kh, installer.GetBlockPoolDef(poolName, namespace, "3"), "create")
 		require.NoError(t(), err)
@@ -99,24 +94,35 @@ func performBlockOperations(installer *installer.InstallHelper, db *utils.MySQLH
 	var wg sync.WaitGroup
 	for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ {
 		wg.Add(1)
-		go dbOperation(db, &wg, installer.Env.LoadTime)
+		go dbOperation(db, &wg, installer.Env.LoadTime, installer.Env.LoadSize)
 	}
 	wg.Wait()
 }
 
-func dbOperation(db *utils.MySQLHelper, wg *sync.WaitGroup, runtime int) {
+func dbOperation(db *utils.MySQLHelper, wg *sync.WaitGroup, runtime int, loadSize string) {
 	defer wg.Done()
+	ds := 100000
+	switch strings.ToLower(loadSize) {
+	case "small":
+		ds = 105000 //.1M * 5 columns * 6 = 3M per thread
+	case "medium":
+		ds = 419430 // .4M * 5 columns * 6 = 12M per thread
+	case "large":
+		ds = 2100000 // 2M * 5 columns * 6 = 60M per thread
+	default:
+		ds = 209715 // .2M * 5 columns * 6 = 15M per thread
+	}
 	start := time.Now()
 	elapsed := time.Since(start).Seconds()
 	for elapsed < float64(runtime) {
 		//InsertRandomData
-		db.InsertRandomData()
-		db.InsertRandomData()
-		db.InsertRandomData()
+		db.InsertRandomData(ds)
+		db.InsertRandomData(ds)
+		db.InsertRandomData(ds)
 		db.SelectRandomData(5)
-		db.InsertRandomData()
-		db.InsertRandomData()
-		db.InsertRandomData()
+		db.InsertRandomData(ds)
+		db.InsertRandomData(ds)
+		db.InsertRandomData(ds)
 		db.SelectRandomData(10)
 
 		//delete Data
@@ -155,6 +161,13 @@ func (o BaseLoadTestOperations) SetUp() {
 		isRookInstalled, err := o.installer.InstallRookOnK8sWithHostPathAndDevices(o.namespace, "bluestore", "/temp/rookBackup", false, true, 3)
 		require.NoError(o.T(), err)
 		require.True(o.T(), isRookInstalled)
+
+	}
+
+	// Enable chaos monkey if enable_chaos flag is present
+	if o.installer.Env.EnableChaos {
+		c := NewChaosHelper(o.namespace, o.kh)
+		go c.Monkey()
 	}
 }
 

tests/longhaul/base_object_test.go

@@ -1,10 +1,26 @@
+/*
+Copyright 2016 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package longhaul
 
 import (
 	"math/rand"
+	"strings"
 	"sync"
 	"testing"
-
 	"time"
 
 	"github.com/icrowley/fake"
@@ -58,13 +74,24 @@ func performObjectStoreOperations(installer *installer.InstallHelper, s3 *utils.
 	var wg sync.WaitGroup
 	for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ {
 		wg.Add(1)
-		go bucketOperations(s3, bucketName, &wg, installer.Env.LoadTime)
+		go bucketOperations(s3, bucketName, &wg, installer.Env.LoadTime, installer.Env.LoadSize)
 	}
 	wg.Wait()
 }
 
-func bucketOperations(s3 *utils.S3Helper, bucketName string, wg *sync.WaitGroup, runtime int) {
+func bucketOperations(s3 *utils.S3Helper, bucketName string, wg *sync.WaitGroup, runtime int, loadSize string) {
 	defer wg.Done()
+	ds := 512000
+	switch strings.ToLower(loadSize) {
+	case "small":
+		ds = 524288 //.5M * 5 = 2.5M per thread
+	case "medium":
+		ds = 2097152 //2M * 5 = 10M per thread
+	case "large":
+		ds = 10485760 //10M * 5 = 50M per thread
+	default:
+		ds = 1048576 // 1M * 5 = 5M per thread
+	}
 	start := time.Now()
 	elapsed := time.Since(start).Seconds()
 	for elapsed < float64(runtime) {
@@ -72,11 +99,11 @@ func bucketOperations(s3 *utils.S3Helper, bucketName string, wg *sync.WaitGroup,
 		key2 := fake.CharactersN(30)
 		key3 := fake.CharactersN(30)
 		key4 := fake.CharactersN(30)
-		s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key1, "plain/text")
-		s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key2, "plain/text")
-		s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key3, "plain/text")
-		s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key4, "plain/text")
-		s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key1, "plain/text")
+		s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key1, "plain/text")
+		s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key2, "plain/text")
+		s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key3, "plain/text")
+		s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key4, "plain/text")
+		s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key1, "plain/text")
 		s3.GetObjectInBucket(bucketName, key1)
 		s3.GetObjectInBucket(bucketName, key2)
 		s3.GetObjectInBucket(bucketName, key3)

tests/longhaul/block_test.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2016 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package longhaul
 
 import (
@@ -29,14 +45,16 @@ type BlockLongHaulSuite struct {
 	suite.Suite
 	kh        *utils.K8sHelper
 	installer *installer.InstallHelper
+	namespace string
 	op        contracts.Setup
 }
 
 //Test set up - does the following in order
 //create pool and storage class, create a PVC, Create a MySQL app/service that uses pvc
 func (s *BlockLongHaulSuite) SetupSuite() {
-	s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, "longhaul-ns")
-	createStorageClassAndPool(s.T, s.kh, "longhaul-ns", "rook-block", "rook-pool")
+	s.namespace = "longhaul-ns"
+	s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, s.namespace)
+	createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-block", "rook-pool")
 }
 
 //create a n number  ofPVC, Create a MySQL app/service that uses pvc

tests/longhaul/block_with_fencing_test.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2016 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package longhaul
 
 import (
@@ -31,6 +47,7 @@ type BlockLongHaulSuiteWithFencing struct {
 	installer  *installer.InstallHelper
 	testClient *clients.TestClient
 	bc         contracts.BlockOperator
+	namespace  string
 	op         contracts.Setup
 }
 
@@ -39,9 +56,10 @@ type BlockLongHaulSuiteWithFencing struct {
 //Write some data to the pvc and unmount the pod
 func (s *BlockLongHaulSuiteWithFencing) SetupSuite() {
 	var err error
-	s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, "longhaul-ns")
-	createStorageClassAndPool(s.T, s.kh, "longhaul-ns", "rook-block", "rook-pool")
-	s.testClient, err = clients.CreateTestClient(s.kh, "longhaul-ns")
+	s.namespace = "longhaul-ns"
+	s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, s.namespace)
+	createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-block", "rook-pool")
+	s.testClient, err = clients.CreateTestClient(s.kh, s.namespace)
 	require.Nil(s.T(), err)
 	s.bc = s.testClient.GetBlockClient()
 	if _, err := s.kh.GetPVCStatus(defaultNamespace, "block-pv-one"); err != nil {

tests/longhaul/longhaul_util.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2016 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package longhaul
+
+import (
+	"math/rand"
+
+	"github.com/coreos/pkg/capnslog"
+	"github.com/rook/rook/tests/framework/utils"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"time"
+)
+
+//utility to install and promethous and grafana to monitor rook cluster
+var (
+	logger = capnslog.NewPackageLogger("github.com/rook/rook", "longhaul")
+)
+
+type ChaosHelper struct {
+	namespace string
+	kh        *utils.K8sHelper
+}
+
+func NewChaosHelper(namespace string, kh *utils.K8sHelper) *ChaosHelper {
+	return &ChaosHelper{namespace: namespace, kh: kh}
+
+}
+
+func (h *ChaosHelper) Monkey() {
+	time.Sleep(5 * time.Minute)
+	for {
+		logger.Info("Attempt to kill a Random Pod in namespace %s", h.namespace)
+		listOpts := metav1.ListOptions{}
+		pods, err := h.kh.Clientset.CoreV1().Pods(h.namespace).List(listOpts)
+		if err != nil {
+			logger.Infof("Error getting list of pods ,err ->%v", err)
+			return
+		}
+		kp := pods.Items[rand.Intn(len(pods.Items)-1)]
+		logger.Info("Killing pod with Name= %s", kp.Name)
+		propagation := metav1.DeletePropagationForeground
+		delOptions := &metav1.DeleteOptions{PropagationPolicy: &propagation}
+		err = h.kh.Clientset.CoreV1().Pods(h.namespace).Delete(kp.Name, delOptions)
+		if err != nil {
+			logger.Infof("Cannot delete pod %s, err->%v", kp.Name, err)
+			continue
+		}
+		logger.Infof("pod %s deleted", kp.Name)
+		time.Sleep(5 * time.Minute)
+	}
+}

tests/longhaul/object_test.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2016 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package longhaul
 
 import (
@@ -33,13 +49,15 @@ type ObjectLongHaulSuite struct {
 	kh        *utils.K8sHelper
 	installer *installer.InstallHelper
 	tc        *clients.TestClient
+	namespace string
 	op        contracts.Setup
 }
 
 func (s *ObjectLongHaulSuite) SetupSuite() {
 	var err error
-	s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, "longhaul-ns")
-	s.tc, err = clients.CreateTestClient(s.kh, "longhaul-ns")
+	s.namespace = "longhaul-ns"
+	s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, s.namespace)
+	s.tc, err = clients.CreateTestClient(s.kh, s.namespace)
 	require.Nil(s.T(), err)
 
 }
@@ -50,9 +68,9 @@ func (s *ObjectLongHaulSuite) TestObjectLonghaulRun() {
 	wg.Add(s.installer.Env.LoadVolumeNumber)
 	for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ {
 		if i == 1 {
-			go ObjectStoreOperations(s, &wg, "longhaul-ns", storeName+strconv.Itoa(i), false)
+			go ObjectStoreOperations(s, &wg, s.namespace, storeName+strconv.Itoa(i), false)
 		} else {
-			go ObjectStoreOperations(s, &wg, "longhaul-ns", storeName+strconv.Itoa(i), randomBool())
+			go ObjectStoreOperations(s, &wg, s.namespace, storeName+strconv.Itoa(i), randomBool())
 		}
 
 	}
@@ -62,7 +80,7 @@ func (s *ObjectLongHaulSuite) TestObjectLonghaulRun() {
 func ObjectStoreOperations(s *ObjectLongHaulSuite, wg *sync.WaitGroup, namespace string, storeName string, deleteStore bool) {
 	defer wg.Done()
 	bucketName := "loadbucket"
-	s3 := createObjectStoreAndUser(s.T, s.kh, s.tc, "longhaul-ns", storeName, "longhaul", "LongHaulTest")
+	s3 := createObjectStoreAndUser(s.T, s.kh, s.tc, s.namespace, storeName, "longhaul", "LongHaulTest")
 	isFound, err := s3.IsBucketPresent(bucketName)
 	if err == nil {
 		if !isFound {

tests/pipeline/Jenkinsfile.longhaul

@@ -7,9 +7,11 @@ pipeline {
           choice(name: 'key', choices: 'master\nPR', description: 'Select master or a PR to run tests aganist - defaults to master')
           string(name: 'PR', defaultValue: '', description: 'PR number, required if key value is set to PR')
           string(name: 'version', defaultValue: '', description: 'Version to run tests against')
-          string(name: 'Load_test', defaultValue: 'TestBlockLongHaul', description: 'select a load test to run ')
+          string(name: 'Load_test', defaultValue: 'TestObjectLongHaul', description: 'select a load test to run ')
           string(name: 'load_factor', defaultValue: '20', description: 'load factor - number of concurrent threads running per test')
           string(name: 'load_time', defaultValue: '1800', description: 'load time - number of seconds each thread runs')
+          choice(name: 'load_size', choices: 'S\nM\nL', description: 'select load size')
+          choice(name: 'enable_chaos', choices: 'false\ntrue', description: 'enable deliberate chaos - kill pods randomly')
           string(name: 'volume_count', defaultValue: '3', description: 'volume count - number of concurrent volumes being create(file/block or object')
           string(name: 'runs', defaultValue: '20', description: 'number of test runs ')
           string(name: 'run_interval', defaultValue: '600', description: 'interval between test runs in seconds')
@@ -56,7 +58,7 @@ pipeline {
                             kubectl get nodes
                             chmod +x longhaul
                             mkdir -p _output/tests/
-                            ./longhaul -test.v -test.run '''+ "${env.Load_test}" + ''' -test.timeout 3600s --load_parallel_runs '''+"${env.load_factor}"+''' --load_volumes '''+"${env.volume_count}"+''' --load_time '''+"${env.load_time}"+ ''' 2>&1 | tee _output/tests/longHaulTest_'''+"$i"+'''.log
+                            ./longhaul -test.v -test.run '''+ "${env.Load_test}" + ''' -test.timeout 3600s --load_parallel_runs '''+"${env.load_factor}"+''' --load_volumes '''+"${env.volume_count}"+''' --load_time '''+"${env.load_time}"+''' --load_size '''+"${env.load_size}"+''' --enable_chaos='''+"${env.enable_chaos}"+''' 2>&1 | tee _output/tests/longHaulTest_'''+"$i"+'''.log
                              cat _output/tests/longHaulTest_''' + "$i"+'''.log | _output/go-junit-report > _output/tests/longHaulTest_'''+"$i"+'''.xml'''
                         }
                         finally{