diff --git a/tests/README.md b/tests/README.md index a1a4e6f58..4104f6a92 100644 --- a/tests/README.md +++ b/tests/README.md @@ -80,16 +80,10 @@ In addition to standard go tests parameters, the following custom parameters are Parameter | Description | Possible values | Default --- |--- | --- | --- rook_platform| platform rook needs to be installed on | kubernetes | kubernetes -k8s_version | version of Kubernetes to be installed | v1.5+ | v1.7.5 +k8s_version | version of Kubernetes to be installed | v1.6+ | v1.8.5 rook_image | rook image name to be installed | valid image name | rook/rook toolbox_image | toolbox image name to be installed | valid image name | rook/toolbox skip_install_rook | skips installing rook (if already installed) | true or false | false -load_parallel_runs | performs concurrent operations (optional used for load test) | any number | 20 - -If the `install_rook` flag is set to false, then all the other flags are ignored -and tests are run without rook being installed and setup. Use this flag to run tests against -a pre-installed/configured rook. - ### Running Tests with parameters. @@ -111,24 +105,34 @@ go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration ##### To run specific without installing rook ``` -go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration --skip_install_rook=true +go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration --skip_install_rook ``` If the `skip_install_rook` flag is set to true, then rook is not uninstalled either. #### Run Longhaul Tests -Using the `go test -count n` option, any tests can be repeated `n` number of times to simulate load or longhaul test. Although -any test can be converted to a longhaul test, it's ideal to run integration tests under load for an extended period. Also a new custom test flag `load_parallel_runs` is added to control the number of concurrent operations being performed. -For example look at the [block long haul test](/tests/block/k8s/longhaul/basicBlockonghaul_test.go) - - To run a longhaul test you can run any integration test with `-count` and `--load_parallel_runs` options - e.g. +[Longhaul](/tests/block/k8s/longhaul) tests are integration tests that run for extended period of time. A load profile can be configured +using the following load test flags + +Parameter | Description | Possible values | Default + --- |--- | --- | --- +load_parallel_runs | performs concurrent operations | any number | 20 +load_volumes | number of volumes | >1 | 1 +load_time | number of seconds to run | >1 | 1800 +load_size | size of load profile (3M, 10M, or 50M per thread) | small, medium, or large | medium +enable_chaos | kill random pods in rook cluster | true or false | false + + e.g. ``` - go test -run TestK8sBlockLongHaul github.com/rook/rook/tests/longhaul --load_parallel_runs=20 -count=1000 + go test -run TestObjectLongHaul github.com/rook/rook/tests/longhaul --load_parallel_runs=20 --load_time 1800 --load_size small --load_volumes 3 ``` The Longhaul test just like other test is going to install rook if it's not already installed, but it is not going to clean up test data or uninstall rook after the run. Longhaul test is designed to run multiple times on the same setup and installation of rook to tests its stability. Test Data and rook should be cleaned up manually after the test. + You can measure memory, cpu, IOPS, throughput, and other settings on a cluster using Prometheus. The metrics collected during load test can be visualize using Grafana. + Here a couple of helpful links to get prometheus and grafana started and collect metrics: + [kube-prometheus](https://github.com/coreos/prometheus-operator/tree/master/contrib/kube-prometheus) and [cluster-deploy-script](https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/hack/cluster-monitoring/deploy) + Prerequisites : * Go installed and GO_PATH set diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/install_helper.go index 1fc56ef49..293064d29 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/install_helper.go @@ -206,8 +206,7 @@ func SystemNamespace(namespace string) string { func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType, dataDirHostPath string, helmInstalled, useDevices bool, mons int) (bool, error) { var err error //flag used for local debuggin purpose, when rook is pre-installed - skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true") - if skipRookInstall { + if h.Env.SkipInstallRook { return true, nil } @@ -270,8 +269,7 @@ func (h *InstallHelper) UninstallRook(helmInstalled bool, namespace string) { //UninstallRookFromK8s uninstalls rook from multiple namespaces in k8s func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNamespace string, namespaces ...string) { //flag used for local debugging purpose, when rook is pre-installed - skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true") - if skipRookInstall { + if h.Env.SkipInstallRook { return } diff --git a/tests/framework/objects/environment_manifest.go b/tests/framework/objects/environment_manifest.go index a90d3a961..4142907f2 100644 --- a/tests/framework/objects/environment_manifest.go +++ b/tests/framework/objects/environment_manifest.go @@ -25,10 +25,12 @@ type EnvironmentManifest struct { HostType string RookImageName string ToolboxImageName string - SkipInstallRook string + SkipInstallRook bool LoadVolumeNumber int LoadConcurrentRuns int LoadTime int + LoadSize string + EnableChaos bool } var Env EnvironmentManifest @@ -38,8 +40,10 @@ func init() { flag.StringVar(&Env.HostType, "host_type", "localhost", "Host were tests are run eg - localhost,GCE or AWS") flag.StringVar(&Env.RookImageName, "rook_image", "rook/rook", "Docker image name for the rook container to install, must be in docker hub or local environment") flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/toolbox", "Docker image name of the toolbox container to install, must be in docker hub or local environment") - flag.StringVar(&Env.SkipInstallRook, "skip_install_rook", "false", "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed") + flag.BoolVar(&Env.SkipInstallRook, "skip_install_rook", false, "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed") flag.IntVar(&Env.LoadConcurrentRuns, "load_parallel_runs", 20, "number of routines for load test") flag.IntVar(&Env.LoadVolumeNumber, "load_volumes", 1, "number of volumes(file,object or block) to be created for load test") flag.IntVar(&Env.LoadTime, "load_time", 1800, "number of seconds each thread perform operations for") + flag.StringVar(&Env.LoadSize, "load_size", "medium", "load size for each thread performing operations - small,medium or large.") + flag.BoolVar(&Env.EnableChaos, "enable_chaos", false, "used to determine if random pods in a namespace are to be killed during load test.") } diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index f93d2328b..01a7d249a 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -318,6 +318,18 @@ func (k8sh *K8sHelper) GetMonitorServices(namespace string) (map[string]string, }, nil } +func (k8sh *K8sHelper) IsPodWithLabelPresent(label string, namespace string) bool { + options := metav1.ListOptions{LabelSelector: label} + pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(options) + if errors.IsNotFound(err) { + return false + } + if len(pods.Items) == 0 { + return false + } + return true +} + //IsPodWithLabelRunning returns true if a Pod is running status or goes to Running status within 90s else returns false func (k8sh *K8sHelper) IsPodWithLabelRunning(label string, namespace string) bool { options := metav1.ListOptions{LabelSelector: label} diff --git a/tests/framework/utils/mysql_helper.go b/tests/framework/utils/mysql_helper.go index 7dd091d9c..91b821e64 100644 --- a/tests/framework/utils/mysql_helper.go +++ b/tests/framework/utils/mysql_helper.go @@ -67,7 +67,7 @@ func (h *MySQLHelper) PingSuccess() bool { //CreateTable func create sample Table func (h *MySQLHelper) CreateTable() sql.Result { - result, err := h.DB.Exec("CREATE TABLE LONGHAUL (id int NOT NULL AUTO_INCREMENT,number int,data varchar(32)" + + result, err := h.DB.Exec("CREATE TABLE LONGHAUL (id int NOT NULL AUTO_INCREMENT,number int,data1 varchar(10000),data2 varchar(10000),data3 varchar(10000),data4 varchar(10000),data5 varchar(10000)" + ", PRIMARY KEY (id) )") if err != nil { panic(err) @@ -76,14 +76,14 @@ func (h *MySQLHelper) CreateTable() sql.Result { } //InsertRandomData Inserts random Data into the table -func (h *MySQLHelper) InsertRandomData() sql.Result { - stmtIns, err := h.DB.Prepare("INSERT INTO LONGHAUL (number, data) VALUES ( ?, ? )") // ? = placeholder +func (h *MySQLHelper) InsertRandomData(dataSize int) sql.Result { + stmtIns, err := h.DB.Prepare("INSERT INTO LONGHAUL (number, data1, data2, data3, data4, data5) VALUES ( ?, ?, ?, ?, ? )") // ? = placeholder if err != nil { panic(err) } defer stmtIns.Close() - result, err := stmtIns.Exec(rand.Intn(100000000), fake.CharactersN(20)) + result, err := stmtIns.Exec(rand.Intn(100000000), fake.CharactersN(dataSize), fake.CharactersN(dataSize), fake.CharactersN(dataSize), fake.CharactersN(dataSize), fake.CharactersN(dataSize)) if err != nil { panic(err) } diff --git a/tests/longhaul/base_block_test.go b/tests/longhaul/base_block_test.go index 0663a0f45..fb8eb6f4a 100644 --- a/tests/longhaul/base_block_test.go +++ b/tests/longhaul/base_block_test.go @@ -1,12 +1,11 @@ package longhaul import ( + "strings" "sync" "testing" - "time" - "github.com/coreos/pkg/capnslog" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" @@ -14,7 +13,6 @@ import ( ) var ( - logger = capnslog.NewPackageLogger("github.com/rook/rook", "longhaul") defaultNamespace = "default" ) @@ -22,9 +20,6 @@ var ( func createStorageClassAndPool(t func() *testing.T, kh *utils.K8sHelper, namespace string, storageClassName string, poolName string) { //create storage class if scp, _ := kh.IsStorageClassPresent(storageClassName); !scp { - - installer.BlockResourceOperation(kh, installer.GetBlockStorageClassDef(poolName, storageClassName, namespace), "create") - logger.Infof("Install pool and storage class for rook block") _, err := installer.BlockResourceOperation(kh, installer.GetBlockPoolDef(poolName, namespace, "3"), "create") require.NoError(t(), err) @@ -99,24 +94,35 @@ func performBlockOperations(installer *installer.InstallHelper, db *utils.MySQLH var wg sync.WaitGroup for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ { wg.Add(1) - go dbOperation(db, &wg, installer.Env.LoadTime) + go dbOperation(db, &wg, installer.Env.LoadTime, installer.Env.LoadSize) } wg.Wait() } -func dbOperation(db *utils.MySQLHelper, wg *sync.WaitGroup, runtime int) { +func dbOperation(db *utils.MySQLHelper, wg *sync.WaitGroup, runtime int, loadSize string) { defer wg.Done() + ds := 100000 + switch strings.ToLower(loadSize) { + case "small": + ds = 105000 //.1M * 5 columns * 6 = 3M per thread + case "medium": + ds = 419430 // .4M * 5 columns * 6 = 12M per thread + case "large": + ds = 2100000 // 2M * 5 columns * 6 = 60M per thread + default: + ds = 209715 // .2M * 5 columns * 6 = 15M per thread + } start := time.Now() elapsed := time.Since(start).Seconds() for elapsed < float64(runtime) { //InsertRandomData - db.InsertRandomData() - db.InsertRandomData() - db.InsertRandomData() + db.InsertRandomData(ds) + db.InsertRandomData(ds) + db.InsertRandomData(ds) db.SelectRandomData(5) - db.InsertRandomData() - db.InsertRandomData() - db.InsertRandomData() + db.InsertRandomData(ds) + db.InsertRandomData(ds) + db.InsertRandomData(ds) db.SelectRandomData(10) //delete Data @@ -155,6 +161,13 @@ func (o BaseLoadTestOperations) SetUp() { isRookInstalled, err := o.installer.InstallRookOnK8sWithHostPathAndDevices(o.namespace, "bluestore", "/temp/rookBackup", false, true, 3) require.NoError(o.T(), err) require.True(o.T(), isRookInstalled) + + } + + // Enable chaos monkey if enable_chaos flag is present + if o.installer.Env.EnableChaos { + c := NewChaosHelper(o.namespace, o.kh) + go c.Monkey() } } diff --git a/tests/longhaul/base_object_test.go b/tests/longhaul/base_object_test.go index 402967d8e..c57bfbf2d 100644 --- a/tests/longhaul/base_object_test.go +++ b/tests/longhaul/base_object_test.go @@ -1,10 +1,26 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package longhaul import ( "math/rand" + "strings" "sync" "testing" - "time" "github.com/icrowley/fake" @@ -58,13 +74,24 @@ func performObjectStoreOperations(installer *installer.InstallHelper, s3 *utils. var wg sync.WaitGroup for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ { wg.Add(1) - go bucketOperations(s3, bucketName, &wg, installer.Env.LoadTime) + go bucketOperations(s3, bucketName, &wg, installer.Env.LoadTime, installer.Env.LoadSize) } wg.Wait() } -func bucketOperations(s3 *utils.S3Helper, bucketName string, wg *sync.WaitGroup, runtime int) { +func bucketOperations(s3 *utils.S3Helper, bucketName string, wg *sync.WaitGroup, runtime int, loadSize string) { defer wg.Done() + ds := 512000 + switch strings.ToLower(loadSize) { + case "small": + ds = 524288 //.5M * 5 = 2.5M per thread + case "medium": + ds = 2097152 //2M * 5 = 10M per thread + case "large": + ds = 10485760 //10M * 5 = 50M per thread + default: + ds = 1048576 // 1M * 5 = 5M per thread + } start := time.Now() elapsed := time.Since(start).Seconds() for elapsed < float64(runtime) { @@ -72,11 +99,11 @@ func bucketOperations(s3 *utils.S3Helper, bucketName string, wg *sync.WaitGroup, key2 := fake.CharactersN(30) key3 := fake.CharactersN(30) key4 := fake.CharactersN(30) - s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key1, "plain/text") - s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key2, "plain/text") - s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key3, "plain/text") - s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key4, "plain/text") - s3.PutObjectInBucket(bucketName, fake.CharactersN(200), key1, "plain/text") + s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key1, "plain/text") + s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key2, "plain/text") + s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key3, "plain/text") + s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key4, "plain/text") + s3.PutObjectInBucket(bucketName, fake.CharactersN(ds), key1, "plain/text") s3.GetObjectInBucket(bucketName, key1) s3.GetObjectInBucket(bucketName, key2) s3.GetObjectInBucket(bucketName, key3) diff --git a/tests/longhaul/block_test.go b/tests/longhaul/block_test.go index b11c5b90b..dafb223a6 100644 --- a/tests/longhaul/block_test.go +++ b/tests/longhaul/block_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package longhaul import ( @@ -29,14 +45,16 @@ type BlockLongHaulSuite struct { suite.Suite kh *utils.K8sHelper installer *installer.InstallHelper + namespace string op contracts.Setup } //Test set up - does the following in order //create pool and storage class, create a PVC, Create a MySQL app/service that uses pvc func (s *BlockLongHaulSuite) SetupSuite() { - s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, "longhaul-ns") - createStorageClassAndPool(s.T, s.kh, "longhaul-ns", "rook-block", "rook-pool") + s.namespace = "longhaul-ns" + s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, s.namespace) + createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-block", "rook-pool") } //create a n number ofPVC, Create a MySQL app/service that uses pvc diff --git a/tests/longhaul/block_with_fencing_test.go b/tests/longhaul/block_with_fencing_test.go index e992c896d..16cbbbb9d 100644 --- a/tests/longhaul/block_with_fencing_test.go +++ b/tests/longhaul/block_with_fencing_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package longhaul import ( @@ -31,6 +47,7 @@ type BlockLongHaulSuiteWithFencing struct { installer *installer.InstallHelper testClient *clients.TestClient bc contracts.BlockOperator + namespace string op contracts.Setup } @@ -39,9 +56,10 @@ type BlockLongHaulSuiteWithFencing struct { //Write some data to the pvc and unmount the pod func (s *BlockLongHaulSuiteWithFencing) SetupSuite() { var err error - s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, "longhaul-ns") - createStorageClassAndPool(s.T, s.kh, "longhaul-ns", "rook-block", "rook-pool") - s.testClient, err = clients.CreateTestClient(s.kh, "longhaul-ns") + s.namespace = "longhaul-ns" + s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, s.namespace) + createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-block", "rook-pool") + s.testClient, err = clients.CreateTestClient(s.kh, s.namespace) require.Nil(s.T(), err) s.bc = s.testClient.GetBlockClient() if _, err := s.kh.GetPVCStatus(defaultNamespace, "block-pv-one"); err != nil { diff --git a/tests/longhaul/longhaul_util.go b/tests/longhaul/longhaul_util.go new file mode 100644 index 000000000..b08825b95 --- /dev/null +++ b/tests/longhaul/longhaul_util.go @@ -0,0 +1,65 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package longhaul + +import ( + "math/rand" + + "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/tests/framework/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "time" +) + +//utility to install and promethous and grafana to monitor rook cluster +var ( + logger = capnslog.NewPackageLogger("github.com/rook/rook", "longhaul") +) + +type ChaosHelper struct { + namespace string + kh *utils.K8sHelper +} + +func NewChaosHelper(namespace string, kh *utils.K8sHelper) *ChaosHelper { + return &ChaosHelper{namespace: namespace, kh: kh} + +} + +func (h *ChaosHelper) Monkey() { + time.Sleep(5 * time.Minute) + for { + logger.Info("Attempt to kill a Random Pod in namespace %s", h.namespace) + listOpts := metav1.ListOptions{} + pods, err := h.kh.Clientset.CoreV1().Pods(h.namespace).List(listOpts) + if err != nil { + logger.Infof("Error getting list of pods ,err ->%v", err) + return + } + kp := pods.Items[rand.Intn(len(pods.Items)-1)] + logger.Info("Killing pod with Name= %s", kp.Name) + propagation := metav1.DeletePropagationForeground + delOptions := &metav1.DeleteOptions{PropagationPolicy: &propagation} + err = h.kh.Clientset.CoreV1().Pods(h.namespace).Delete(kp.Name, delOptions) + if err != nil { + logger.Infof("Cannot delete pod %s, err->%v", kp.Name, err) + continue + } + logger.Infof("pod %s deleted", kp.Name) + time.Sleep(5 * time.Minute) + } +} diff --git a/tests/longhaul/object_test.go b/tests/longhaul/object_test.go index ecaf54b11..35e65f302 100644 --- a/tests/longhaul/object_test.go +++ b/tests/longhaul/object_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package longhaul import ( @@ -33,13 +49,15 @@ type ObjectLongHaulSuite struct { kh *utils.K8sHelper installer *installer.InstallHelper tc *clients.TestClient + namespace string op contracts.Setup } func (s *ObjectLongHaulSuite) SetupSuite() { var err error - s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, "longhaul-ns") - s.tc, err = clients.CreateTestClient(s.kh, "longhaul-ns") + s.namespace = "longhaul-ns" + s.op, s.kh, s.installer = NewBaseLoadTestOperations(s.T, s.namespace) + s.tc, err = clients.CreateTestClient(s.kh, s.namespace) require.Nil(s.T(), err) } @@ -50,9 +68,9 @@ func (s *ObjectLongHaulSuite) TestObjectLonghaulRun() { wg.Add(s.installer.Env.LoadVolumeNumber) for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ { if i == 1 { - go ObjectStoreOperations(s, &wg, "longhaul-ns", storeName+strconv.Itoa(i), false) + go ObjectStoreOperations(s, &wg, s.namespace, storeName+strconv.Itoa(i), false) } else { - go ObjectStoreOperations(s, &wg, "longhaul-ns", storeName+strconv.Itoa(i), randomBool()) + go ObjectStoreOperations(s, &wg, s.namespace, storeName+strconv.Itoa(i), randomBool()) } } @@ -62,7 +80,7 @@ func (s *ObjectLongHaulSuite) TestObjectLonghaulRun() { func ObjectStoreOperations(s *ObjectLongHaulSuite, wg *sync.WaitGroup, namespace string, storeName string, deleteStore bool) { defer wg.Done() bucketName := "loadbucket" - s3 := createObjectStoreAndUser(s.T, s.kh, s.tc, "longhaul-ns", storeName, "longhaul", "LongHaulTest") + s3 := createObjectStoreAndUser(s.T, s.kh, s.tc, s.namespace, storeName, "longhaul", "LongHaulTest") isFound, err := s3.IsBucketPresent(bucketName) if err == nil { if !isFound { diff --git a/tests/pipeline/Jenkinsfile.longhaul b/tests/pipeline/Jenkinsfile.longhaul index 9bc105191..87626fa2e 100644 --- a/tests/pipeline/Jenkinsfile.longhaul +++ b/tests/pipeline/Jenkinsfile.longhaul @@ -7,9 +7,11 @@ pipeline { choice(name: 'key', choices: 'master\nPR', description: 'Select master or a PR to run tests aganist - defaults to master') string(name: 'PR', defaultValue: '', description: 'PR number, required if key value is set to PR') string(name: 'version', defaultValue: '', description: 'Version to run tests against') - string(name: 'Load_test', defaultValue: 'TestBlockLongHaul', description: 'select a load test to run ') + string(name: 'Load_test', defaultValue: 'TestObjectLongHaul', description: 'select a load test to run ') string(name: 'load_factor', defaultValue: '20', description: 'load factor - number of concurrent threads running per test') string(name: 'load_time', defaultValue: '1800', description: 'load time - number of seconds each thread runs') + choice(name: 'load_size', choices: 'S\nM\nL', description: 'select load size') + choice(name: 'enable_chaos', choices: 'false\ntrue', description: 'enable deliberate chaos - kill pods randomly') string(name: 'volume_count', defaultValue: '3', description: 'volume count - number of concurrent volumes being create(file/block or object') string(name: 'runs', defaultValue: '20', description: 'number of test runs ') string(name: 'run_interval', defaultValue: '600', description: 'interval between test runs in seconds') @@ -56,7 +58,7 @@ pipeline { kubectl get nodes chmod +x longhaul mkdir -p _output/tests/ - ./longhaul -test.v -test.run '''+ "${env.Load_test}" + ''' -test.timeout 3600s --load_parallel_runs '''+"${env.load_factor}"+''' --load_volumes '''+"${env.volume_count}"+''' --load_time '''+"${env.load_time}"+ ''' 2>&1 | tee _output/tests/longHaulTest_'''+"$i"+'''.log + ./longhaul -test.v -test.run '''+ "${env.Load_test}" + ''' -test.timeout 3600s --load_parallel_runs '''+"${env.load_factor}"+''' --load_volumes '''+"${env.volume_count}"+''' --load_time '''+"${env.load_time}"+''' --load_size '''+"${env.load_size}"+''' --enable_chaos='''+"${env.enable_chaos}"+''' 2>&1 | tee _output/tests/longHaulTest_'''+"$i"+'''.log cat _output/tests/longHaulTest_''' + "$i"+'''.log | _output/go-junit-report > _output/tests/longHaulTest_'''+"$i"+'''.xml''' } finally{