From 5d8d2b5b5acbdf0016e965e59cae9963c8c8e1b7 Mon Sep 17 00:00:00 2001 From: Alexander Trost Date: Wed, 22 Aug 2018 16:14:59 +0200 Subject: [PATCH] build: Only install helm if needed and use a temp path Resolves #981 Signed-off-by: Alexander Trost --- Jenkinsfile | 10 ++- tests/README.md | 12 ++- tests/framework/installer/ceph_installer.go | 2 +- tests/framework/installer/environment.go | 4 +- tests/framework/utils/helm_helper.go | 7 +- tests/scripts/helm.sh | 84 ++++++++++++++------- 6 files changed, 77 insertions(+), 42 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index de0ad5b84..281f6319a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -141,17 +141,19 @@ def RunIntegrationTest(k, v) { echo "Running Smoke Tests" sh '''#!/bin/bash set -o pipefail - export KUBECONFIG=$HOME/admin.conf + export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \ + KUBECONFIG=$HOME/admin.conf kubectl config view - _output/tests/linux_amd64/integration -test.v -test.timeout 600s -test.run SmokeSuite --host_type '''+"${k}"+''' 2>&1 | tee _output/tests/integrationTests.log''' + _output/tests/linux_amd64/integration -test.v -test.timeout 600s -test.run SmokeSuite --host_type '''+"${k}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log''' } else { echo "Running full regression" sh '''#!/bin/bash set -o pipefail - export KUBECONFIG=$HOME/admin.conf + export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \ + KUBECONFIG=$HOME/admin.conf kubectl config view - _output/tests/linux_amd64/integration -test.v -test.timeout 2400s --host_type '''+"${k}"+''' 2>&1 | tee _output/tests/integrationTests.log''' + _output/tests/linux_amd64/integration -test.v -test.timeout 2400s --host_type '''+"${k}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log''' } } finally{ diff --git a/tests/README.md b/tests/README.md index f8ddc8b15..f07347a17 100644 --- a/tests/README.md +++ b/tests/README.md @@ -58,13 +58,17 @@ To use an alternate version, simply set this variable before running the relevan For example, if you wanted to use `v1.9.6`, you would run `export KUBE_VERSION=v1.9.6` first before running `up`. ### Install Helm -Use [helm.sh](/tests/scripts/helm.sh) to install helm and set up Rook charts defined under `_output/charts` (generated by build): - - To install and set up Helm charts for Rook run `tests/scripts/helm.sh up`. - - To clean up `tests/scripts/helm.sh clean`. +Use [helm.sh](/tests/scripts/helm.sh) to install Helm and set up Rook charts defined under `_output/charts` (generated by build): -**Note:** `*kubeadm.sh`, `minikube.sh` and `helm.sh` scripts depend on some artifacts under the `_output/` directory generated during build time, +- To install and set up Helm charts for Rook run `tests/scripts/helm.sh up`. +- To clean up `tests/scripts/helm.sh clean`. + +**NOTE:** `*kubeadm.sh`, `minikube.sh` and `helm.sh` scripts depend on some artifacts under the `_output/` directory generated during build time, these scripts should be run from project root. e.g., `tests/script/kubeadm.sh up`. +**NOTE**: If Helm is not available in your `PATH`, Helm will be downloaded to a temporary directory (`/tmp/rook-tests-scripts-helm`) and used from that directory. +The temporary directory the Helm binary is in needs to be given to the `integration` binary calls. The flag for that is `--helm=HELM_BINARY`, e.g., `--helm=/tmp/rook-tests-scripts-helm/linux-amd64/helm`. + ## Run Tests From the project root do the following: #### 1. Build rook: diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 2d78c109c..c153de07b 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -460,7 +460,7 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rook h := &CephInstaller{ Manifests: NewCephManifests(rookVersion), k8shelper: k8shelp, - helmHelper: utils.NewHelmHelper(), + helmHelper: utils.NewHelmHelper(Env.Helm), k8sVersion: version.String(), T: t, } diff --git a/tests/framework/installer/environment.go b/tests/framework/installer/environment.go index 63f76a3d6..52cdfed84 100644 --- a/tests/framework/installer/environment.go +++ b/tests/framework/installer/environment.go @@ -23,6 +23,7 @@ import ( //EnvironmentManifest contains information about system under test type EnvironmentManifest struct { HostType string + Helm string RookImageName string ToolboxImageName string SkipInstallRook bool @@ -37,7 +38,8 @@ var Env EnvironmentManifest func init() { Env = EnvironmentManifest{} - flag.StringVar(&Env.HostType, "host_type", "localhost", "Host where tests are run eg - localhost,GCE or AWS") + flag.StringVar(&Env.HostType, "host_type", "localhost", "Host where tests are run eg - localhost, GCE or AWS") + flag.StringVar(&Env.Helm, "helm", "helm", "Path to helm binary") flag.StringVar(&Env.RookImageName, "rook_image", "rook/ceph", "Docker image name for the rook container to install, must be in docker hub or local environment") flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/ceph-toolbox", "Docker image name of the toolbox container to install, must be in docker hub or local environment") flag.BoolVar(&Env.SkipInstallRook, "skip_install_rook", false, "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed") diff --git a/tests/framework/utils/helm_helper.go b/tests/framework/utils/helm_helper.go index f7b4eac59..b1557e452 100644 --- a/tests/framework/utils/helm_helper.go +++ b/tests/framework/utils/helm_helper.go @@ -27,18 +27,19 @@ import ( //HelmHelper is wrapper for running helm commands type HelmHelper struct { executor *exec.CommandExecutor + HelmPath string } //NewHelmHelper creates a instance of HelmHelper -func NewHelmHelper() *HelmHelper { +func NewHelmHelper(helmPath string) *HelmHelper { executor := &exec.CommandExecutor{} - return &HelmHelper{executor: executor} + return &HelmHelper{executor: executor, HelmPath: helmPath} } //Execute is wrapper for executing helm commands func (h *HelmHelper) Execute(args ...string) (string, error) { - result, err := h.executor.ExecuteCommandWithOutput(false, "", "helm", args...) + result, err := h.executor.ExecuteCommandWithOutput(false, "", h.HelmPath, args...) if err != nil { logger.Errorf("Errors Encountered while executing helm command %v: %v", result, err) return result, fmt.Errorf("Failed to run helm commands on args %v : %v , err -> %v", args, result, err) diff --git a/tests/scripts/helm.sh b/tests/scripts/helm.sh index 35e4dae1c..0706fa949 100755 --- a/tests/scripts/helm.sh +++ b/tests/scripts/helm.sh @@ -1,18 +1,46 @@ #!/bin/bash +e + scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +temp="/tmp/rook-tests-scripts-helm" + +HELM="helm" +arch="${ARCH:-}" + +detectArch() { + case "$(uname -m)" in + "x86_64" | "amd64") + arch="amd64" + ;; + "aarch64") + arch="arm64" + ;; + "i386") + arch="i386" + ;; + *) + echo "Couldn't translate 'uname -m' output to an available arch." + echo "Try setting ARCH environment variable to your system arch:" + echo "amd64, x86_64. aarch64, i386" + exit 1 + ;; + esac +} install() { - #Download and unpack helm - local dist - dist="$(uname -s)" - # shellcheck disable=SC2021 - dist=$(echo "${dist}" | tr "[A-Z]" "[a-z]") - wget "https://storage.googleapis.com/kubernetes-helm/helm-v2.6.0-${dist}-amd64.tar.gz" - tar -zxvf "helm-v2.6.0-${dist}-amd64.tar.gz" - sudo mv "${dist}-amd64/helm" /usr/local/bin/helm + if ! helm_loc="$(type -p "helm")" || [[ -z ${helm_loc} ]]; then + # Download and unpack helm + local dist + dist="$(uname -s)" + # shellcheck disable=SC2021 + dist=$(echo "${dist}" | tr "[A-Z]" "[a-z]") + mkdir -p "${temp}" + wget "https://storage.googleapis.com/kubernetes-helm/helm-v2.6.0-${dist}-${arch}.tar.gz" -O "${temp}/helm-v2.6.0-${dist}-${arch}.tar.gz" + tar -C "${temp}" -zxvf "${temp}/helm-v2.6.0-${dist}-${arch}.tar.gz" + HELM="${temp}/${dist}-${arch}/helm" + fi - #Init helm - helm init + # Init helm + "${HELM}" init sleep 5 @@ -20,7 +48,7 @@ install() { INC=0 until [[ "${helm_ready}" == "Running" || $INC -gt 20 ]]; do sleep 10 - ((++INC)) + (( ++INC )) helm_ready=$(kubectl get pods -l app=helm -n kube-system -o jsonpath='{.items[0].status.phase}') echo "helm pod status: $(helm_ready)" done @@ -38,39 +66,37 @@ install() { kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller kubectl -n kube-system patch deploy/tiller-deploy -p '{"spec": {"template": {"spec": {"serviceAccountName": "tiller"}}}}' - #set up local repo for helm and add local/rook-ceph - helm repo remove local - helm repo remove stable + # set up local repo for helm and add local/rook-ceph + "${HELM}" repo remove local + "${HELM}" repo remove stable - helm repo index _output/charts/ --url http://127.0.0.1:8879 - nohup helm serve --repo-path _output/charts/ > /dev/null 2>&1 & + "${HELM}" repo index _output/charts/ --url http://127.0.0.1:8879 + nohup "${HELM}" serve --repo-path _output/charts/ > /dev/null 2>&1 & sleep 10 # wait for helm serve to start - helm repo add local http://127.0.0.1:8879 - helm search rook-ceph + "${HELM}" repo add local http://127.0.0.1:8879 + "${HELM}" search rook-ceph } helm_reset() { - helm reset - local dist - dist="$(uname -s)" + "${HELM}" reset # shellcheck disable=SC2021 - dist=$(echo "${dist}" | tr "[A-Z]" "[a-z]") - sudo rm /usr/local/bin/helm - rm -rf "${dist}-amd64/" - rm "helm-v2.6.0-${dist}-amd64.tar.gz"* - pgrep helm | grep -v grep | awk '{print $2}'| xargs kill -9 - + pgrep "${HELM}" | grep -v grep | awk '{print $2}'| xargs kill -9 + rm -rf "${temp}" } +if [ -z "${arch}" ]; then + detectArch +fi + case "${1:-}" in up) install # shellcheck disable=2002 - cat _output/version | xargs "${scriptdir}/makeTestImages.sh" tag amd64 || true - helm repo add stable https://kubernetes-charts.storage.googleapis.com/ + cat _output/version | xargs "${scriptdir}/makeTestImages.sh" tag "${arch}" || true + "${HELM}" repo add stable https://kubernetes-charts.storage.googleapis.com/ ;; clean) helm_reset