From 2376e36a8c3dd50f214b5ea8b6e79af12467dc90 Mon Sep 17 00:00:00 2001 From: Divesh Angula Date: Fri, 15 Sep 2017 10:35:39 -0700 Subject: [PATCH] create new jenkins pipeline - setup 3 node k8s - complete regression - longhaul testing --- tests/pipeline/Jenkinsfile.k8sSetup | 113 ++++++++++++++++++++++++++ tests/pipeline/Jenkinsfile.longhaul | 78 ++++++++++++++++++ tests/pipeline/Jenkinsfile.regression | 83 +++++++++++++++++++ 3 files changed, 274 insertions(+) create mode 100644 tests/pipeline/Jenkinsfile.k8sSetup create mode 100644 tests/pipeline/Jenkinsfile.longhaul create mode 100644 tests/pipeline/Jenkinsfile.regression diff --git a/tests/pipeline/Jenkinsfile.k8sSetup b/tests/pipeline/Jenkinsfile.k8sSetup new file mode 100644 index 000000000..0b8293dd7 --- /dev/null +++ b/tests/pipeline/Jenkinsfile.k8sSetup @@ -0,0 +1,113 @@ +// Jenkins Pipelines for setting up 3 node rook cluster + +pipeline { + + parameters { + choice(name: 'node', choices: 'gce-reg\ngce-load', description: 'select env to run tests in ') + choice(name: 'key', choices: 'master\nPR', description: 'Select master or a PR to run tests aganist - defaults to master') + string(name: 'PR', defaultValue: '', description: 'PR number, required if key value is set to PR') + string(name: 'version', defaultValue: 'v0.5.0-111.ga4a1a98', description: 'Version to run tests against') + string(name: 'kube_version', defaultValue: 'v1.7.5', description: 'Version of kubernetes to tests against') + } + + agent { label "${params.node}-master" } + + options { + disableConcurrentBuilds() + timestamps() + } + + stages { + stage('set up rook images') { + steps { + //checkout scripts + sh "mkdir -p tests/scripts/" + sh "mkdir -p _output/charts/" + sh "wget -O tests/scripts/kubeadm.sh https://raw.githubusercontent.com/rook/rook/master/tests/scripts/kubeadm.sh" + sh "wget -O tests/scripts/kubeadm-rbd https://raw.githubusercontent.com/rook/rook/master/tests/scripts/kubeadm-rbd" + sh "wget -O tests/scripts/kubeadm-install.sh https://raw.githubusercontent.com/rook/rook/master/tests/scripts/kubeadm-install.sh" + sh "chmod +x tests/scripts/*.sh" + script { + if ( "${params.key}" == 'PR' ){ + if ( "${params.PR}" == '' ){ + echo '[FAILURE] Need A valid PR number to continue' + currentBuild.result = 'FAILURE' + error 'Exit pipeline need a pr number' + } + sh "wget -O rook.tar.gz https://release.rook.io/build/pr/${params.PR}/${params.version}/images/linux_amd64/rook.tar.gz" + sh "wget -O toolbox.tar.gz https://release.rook.io/build/pr/${params.PR}/${params.version}/images/linux_amd64/toolbox.tar.gz" + sh "wget -O version https://release.rook.io/build/pr/${params.PR}/${params.version}/images/linux_amd64/version" + + }else { + sh "wget -O rook.tar.gz https://release.rook.io/build/master/${params.version}/images/linux_amd64/rook.tar.gz" + sh "wget -O toolbox.tar.gz https://release.rook.io/build/master/${params.version}/images/linux_amd64/toolbox.tar.gz" + sh "wget -O version https://release.rook.io/build/master/${params.version}/version" + + } + //tag rook images need for testing + env.versionId = readFile('version').trim() + sh''' + #!/bin/bash + docker load -i rook.tar.gz + docker load -i toolbox.tar.gz + docker images + rookImg=$(docker images |grep rook | awk '{print $1}') + echo $rookImg + toolboxImg=$(docker images |grep toolbox | awk '{print $1}') + echo $toolboxImg + echo "helm Image:" + echo ${versionId} + docker tag $rookImg rook/rook:master + docker tag $rookImg rook/rook:${versionId} + docker tag $toolboxImg rook/toolbox:master + ''' + + } + + stash name: 'test-scripts',includes: 'tests/scripts/' + + } + } + //start 3 node k8s cluster + stage ("Set Up k8s") { + steps{ + script{ + withEnv(["KUBE_VERSION=${env.kube_version}"]){ + echo "setting up 3 node k8s ${env.kube_version} cluster" + sh '''#!/bin/bash + echo " setting up master" + export KUBE_VERSION='''+ "${env.kube_version}" + ''' + kubeadm_join_flags="$(tests/scripts/kubeadm.sh install master | grep '^ *kubeadm join' | sed 's/^ *kubeadm join //')" + echo ${kubeadm_join_flags} > ktoken''' + script{ + token = readFile('ktoken').trim() + } + node("${params.node}-n1"){ + unstash 'test-scripts' + sh '''#!/bin/bash + echo " setting up node 1" + export KUBE_VERSION='''+ "${env.kube_version}" + ''' + tests/scripts/kubeadm.sh install node ''' + "${token}" + } + node("${params.node}-n2"){ + unstash 'test-scripts' + sh '''#!/bin/bash + echo " setting up node 2" + export KUBE_VERSION='''+ "${env.kube_version}" + ''' + tests/scripts/kubeadm.sh install node ''' + "${token}" + } + sh '''#!/bin/bash + export KUBECONFIG=$HOME/admin.conf + tests/scripts/kubeadm.sh wait 3''' + } + } + } + } + } + + post { + always { + deleteDir() + } + } +} diff --git a/tests/pipeline/Jenkinsfile.longhaul b/tests/pipeline/Jenkinsfile.longhaul new file mode 100644 index 000000000..727719b61 --- /dev/null +++ b/tests/pipeline/Jenkinsfile.longhaul @@ -0,0 +1,78 @@ +// Jenkins Pipelines for longhaul + +pipeline { + + parameters { + choice(name: 'node', choices: 'gce-reg\ngce-load', description: 'select env to run tests in ') + choice(name: 'key', choices: 'master\nPR', description: 'Select master or a PR to run tests aganist - defaults to master') + string(name: 'PR', defaultValue: '', description: 'PR number, required if key value is set to PR') + string(name: 'version', defaultValue: 'v0.5.0-111.ga4a1a98', description: 'Version to run tests against') + string(name: 'Load_test', defaultValue: 'BlockLongHaul', description: 'select a load test to run ') + string(name: 'testCount', defaultValue: '50', description: 'Number of times to repeat test per run') + string(name: 'load_factor', defaultValue: '20', description: 'load factor - number of concurrent threads running per test') + string(name: 'runs', defaultValue: '100', description: 'number of runs') + string(name: 'run_interval', defaultValue: '600', description: 'interval between test runs in seconds') + } + + agent { label "${params.node}-master" } + + options { + disableConcurrentBuilds() + timestamps() + } + + stages { + stage('checkout test scripts') { + steps { + sh "mkdir -p _output" + script { + if ( "${params.key}" == 'PR' ){ + if ( "${params.PR}" == '' ){ + echo '[FAILURE] Need A valid PR number to continue' + currentBuild.result = 'FAILURE' + return + } + sh "wget -O longhaul https://release.rook.io/build/pr/${params.PR}/${params.version}/tests/linux_amd64/longhaul" + sh "wget -O _output/go-junit-report https://release.rook.io/build/pr/${params.PR}/${params.version}/go-junit-report" + + }else { + sh "wget -O longhaul https://release.rook.io/build/master/${params.version}/tests/linux_amd64/longhaul" + sh "wget -O _output/go-junit-report https://release.rook.io/build/master/${params.version}/go-junit-report" + } + sh "chmod +x _output/go-junit-report" + } + } + } + stage ("Run Longhaul") { + steps{ + script { + for(i=0; i< "${env.runs}".toInteger() ; i++) { + try{ + sh '''#!/bin/bash + set -o pipefail + export KUBECONFIG=$HOME/admin.conf + kubectl config view + kubectl get nodes + chmod +x longhaul + mkdir -p _output/tests/ + ./longhaul -test.v -test.run '''+ "${env.Load_test}" + ''' -test.timeout 3600s --test.count '''+"${env.testCount}"+''' --load_parallel_runs '''+"${env.load_factor}"+''' 2>&1 | tee _output/tests/integrationTests_'''+"$i"+'''.log + cat _output/tests/integrationTests_''' + "$i"+'''.log | _output/go-junit-report > _output/tests/integrationTests_'''+"$i"+'''.xml''' + } + finally{ + archive '_output/tests/*.log,_output/tests/*.xml' + junit allowEmptyResults: true, keepLongStdio: true, testResults: '_output/tests/*.xml' + } + sleep "${env.run_interval}".toInteger() + } + } + } + } + } + + post { + always { + junit allowEmptyResults: true, keepLongStdio: true, testResults: '_output/tests/*.xml' + deleteDir() + } + } +} \ No newline at end of file diff --git a/tests/pipeline/Jenkinsfile.regression b/tests/pipeline/Jenkinsfile.regression new file mode 100644 index 000000000..f3bd0b538 --- /dev/null +++ b/tests/pipeline/Jenkinsfile.regression @@ -0,0 +1,83 @@ +// Jenkins Pipelines for Rook Regression + +pipeline { + + parameters { + choice(name: 'node', choices: 'gce-reg\ngce-load', description: 'select env to run tests in ') + choice(name: 'key', choices: 'master\nPR', description: 'Select master or a PR to run tests aganist - defaults to master') + string(name: 'PR', defaultValue: '', description: 'PR number, required if key value is set to PR') + string(name: 'version', defaultValue: 'v0.5.0-111.ga4a1a98', description: 'Version to run tests against') + } + + agent { label "${params.node}-master" } + + options { + disableConcurrentBuilds() + timestamps() + } + + stages { + stage('checkout test scripts') { + steps { + sh "mkdir -p _output/charts/" + sh "wget -O helm.sh https://raw.githubusercontent.com/rook/rook/master/tests/scripts/helm.sh" + script { + if ( "${params.key}" == 'PR' ){ + if ( "${params.PR}" == '' ){ + echo '[FAILURE] Need A valid PR number to continue' + currentBuild.result = 'FAILURE' + error 'Exit pipeline need a pr number' + } + sh "wget -O integration https://release.rook.io/build/pr/${params.PR}/${params.version}/tests/linux_amd64/integration" + sh "wget -O _output/go-junit-report https://release.rook.io/build/pr/${params.PR}/${params.version}/go-junit-report" + sh "wget -O version https://release.rook.io/build/pr/${params.PR}/${params.version}/images/linux_amd64/version" + versionId = readFile('version').trim() + sh "wget -O _output/charts/index.yaml https://release.rook.io/build/pr/${params.PR}/${params.version}/charts/index.yaml" + sh "wget -O _output/charts/rook-${versionId}.tgz https://release.rook.io/build/pr/${params.PR}/${params.version}/charts/rook-${versionId}.tgz" + + }else { + sh "wget -O integration https://release.rook.io/build/master/${params.version}/tests/linux_amd64/integration" + sh "wget -O _output/go-junit-report https://release.rook.io/build/master/${params.version}/go-junit-report" + sh "wget -O version https://release.rook.io/build/master/${params.version}/version" + versionId = readFile('version').trim() + sh "wget -O _output/charts/index.yaml https://release.rook.io/build/master/${params.version}/charts/index.yaml" + sh "wget -O _output/charts/rook-${versionId}.tgz https://release.rook.io/build/master/${params.version}/charts/rook-${versionId}.tgz" + } + sh "chmod +x _output/go-junit-report" + } + } + } + //Run regression + stage ("Run Regression") { + steps{ + script { + sh '''#!/bin/bash + export KUBECONFIG=$HOME/admin.conf + chmod +x helm.sh + ./helm.sh up''' + try{ + sh '''#!/bin/bash + set -o pipefail + export KUBECONFIG=$HOME/admin.conf + kubectl config view + kubectl get nodes + chmod +x integration + mkdir -p _output/tests/ + ./integration -test.v -test.timeout 1800s 2>&1 | tee _output/tests/regressionTests.log''' + } + finally{ + sh "cat _output/tests/regressionTests.log | _output/go-junit-report > _output/tests/regressionTests.xml" + archive '_output/tests/*.log,_output/tests/*.xml' + } + } + } + } + } + + post { + always { + junit allowEmptyResults: true, keepLongStdio: true, testResults: '_output/tests/*.xml' + deleteDir() + } + } +}