From ebe21c8f6887a2c0bd05a15d0380361f4d2b6e4d Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Fri, 6 Aug 2021 12:01:57 +0530 Subject: [PATCH] ci: add action for shellcheck linter we are adding new linter for shellcheck. As we are writing more shell scripts this will help maintain quality. Also, doing all the changes required in bash files to pass this shellcheck. Closes: https://github.com/rook/rook/issues/8431 Signed-off-by: subhamkrai --- .github/workflows/shellcheck.yaml | 28 + build/common.sh | 51 +- images/cross/run.sh | 12 +- tests/integration/ceph_base_object_test.go | 4 +- tests/scripts/deploy-validate-vault.sh | 24 +- tests/scripts/deploy_admission_controller.sh | 47 +- .../deploy_admission_controller_test.sh | 10 +- tests/scripts/dind-cluster-rbd | 2 - tests/scripts/dind-cluster.sh | 990 ------------------ tests/scripts/gen_release_notes.sh | 10 +- tests/scripts/generate-tls-config.sh | 0 tests/scripts/github-action-helper.sh | 67 +- tests/scripts/helm.sh | 18 +- tests/scripts/k8s-vagrant-multi-node.sh | 108 -- tests/scripts/kubeadm-dind.sh | 78 -- tests/scripts/kubeadm-install.sh | 9 +- tests/scripts/kubeadm.sh | 26 +- tests/scripts/localPathPV.sh | 8 +- tests/scripts/makeTestImages.sh | 8 +- tests/scripts/minikube.sh | 4 +- tests/scripts/minishift.sh | 19 +- tests/scripts/webhook-patch-ca-bundle.sh | 3 +- 22 files changed, 207 insertions(+), 1319 deletions(-) create mode 100644 .github/workflows/shellcheck.yaml delete mode 100644 tests/scripts/dind-cluster-rbd delete mode 100755 tests/scripts/dind-cluster.sh mode change 100644 => 100755 tests/scripts/generate-tls-config.sh delete mode 100755 tests/scripts/k8s-vagrant-multi-node.sh delete mode 100755 tests/scripts/kubeadm-dind.sh diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml new file mode 100644 index 000000000..8e188f70a --- /dev/null +++ b/.github/workflows/shellcheck.yaml @@ -0,0 +1,28 @@ +name: ShellCheck +on: + push: + tags: + - v* + branches: + - master + - release-* + pull_request: + branches: + - master + - release-* + +jobs: + shellcheck: + name: Shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + with: + severity: warning + check_together: 'yes' + disable_matcher: false + additional_files: build/run build/reset build/sed-in-place + ignore: olm + format: gcc diff --git a/build/common.sh b/build/common.sh index 3e9069f32..c5534fcc9 100644 --- a/build/common.sh +++ b/build/common.sh @@ -1,5 +1,5 @@ #!/bin/bash -e - +set -u # Copyright 2016 The Rook Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,31 +18,41 @@ BUILD_HOST=$(hostname) BUILD_REPO=github.com/rook/rook BUILD_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd -P) SHA256CMD=${SHA256CMD:-shasum -a 256} -BUILD_REGISTRY=build-$(echo ${BUILD_HOST}-${BUILD_ROOT} | ${SHA256CMD} | cut -c1-8) +BUILD_REGISTRY=build-$(echo "${BUILD_HOST}"-"${BUILD_ROOT}" | ${SHA256CMD} | cut -c1-8) DOCKERCMD=${DOCKERCMD:-docker} -OUTPUT_DIR=${BUILD_ROOT}/_output -WORK_DIR=${BUILD_ROOT}/.work +export scriptdir +scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export OUTPUT_DIR=${BUILD_ROOT}/_output +export WORK_DIR=${BUILD_ROOT}/.work CACHE_DIR=${BUILD_ROOT}/.cache -KUBEADM_DIND_DIR=${CACHE_DIR}/kubeadm-dind +export KUBEADM_DIND_DIR=${CACHE_DIR}/kubeadm-dind CROSS_IMAGE=${BUILD_REGISTRY}/cross-amd64 CROSS_IMAGE_VOLUME=cross-volume CROSS_RSYNC_PORT=10873 function ver() { - printf "%d%03d%03d%03d" $(echo "$1" | tr '.' ' ') + local full_ver maj min bug build + full_ver="$1" # functions should name input params for easier understanding + maj="$(echo "${full_ver}" | cut -f1 -d'.')" # when splitting a param, name the components for easier understanding + min="$(echo "${full_ver}" | cut -f2 -d'.')" + bug="$(echo "${full_ver}" | cut -f3 -d'.')" + build="$(echo "${full_ver}" | cut -f4 -d'.')" + printf "%d%03d%03d%03d" "${maj}" "${min}" "${bug}" "${build}" } function check_git() { # git version 2.6.6+ through 2.8.3 had a bug with submodules. this makes it hard # to share a cloned directory between host and container # see https://github.com/git/git/blob/master/Documentation/RelNotes/2.8.3.txt#L33 - local gitversion=$(git --version | cut -d" " -f3) - if (( $(ver ${gitversion}) > $(ver 2.6.6) && $(ver ${gitversion}) < $(ver 2.8.3) )); then - echo WARN: your running git version ${gitversion} which has a bug related to relative + local gitversion + gitversion=$(git --version | cut -d" " -f3) + + if (( $(ver "${gitversion}") > $(ver 2.6.6) && $(ver "${gitversion}") < $(ver 2.8.3) )); then + echo WARN: you are running git version "${gitversion}" which has a bug related to relative echo WARN: submodule paths. Please consider upgrading to 2.8.3 or later fi } @@ -56,18 +66,18 @@ function start_rsync_container() { -p ${CROSS_RSYNC_PORT}:873 \ -v ${CROSS_IMAGE_VOLUME}:/volume \ --entrypoint "/tini" \ - ${CROSS_IMAGE} \ + "${CROSS_IMAGE}" \ -- /build/rsyncd.sh } function wait_for_rsync() { # wait for rsync to come up local tries=100 - while (( ${tries} > 0 )) ; do + while (( tries > 0 )); do if rsync "rsync://localhost:${CROSS_RSYNC_PORT}/" &> /dev/null ; then return 0 fi - tries=$(( ${tries} - 1 )) + (( tries-- )) sleep 0.1 done echo ERROR: rsyncd did not come up >&2 @@ -77,8 +87,8 @@ function wait_for_rsync() { function stop_rsync_container() { local id=$1 - ${DOCKERCMD} stop ${id} &> /dev/null || true - ${DOCKERCMD} rm ${id} &> /dev/null || true + ${DOCKERCMD} stop "${id}" &> /dev/null || true + ${DOCKERCMD} rm "${id}" &> /dev/null || true } function run_rsync() { @@ -90,10 +100,11 @@ function run_rsync() { # run the container as an rsyncd daemon so that we can copy the # source tree to the container volume. - local id=$(start_rsync_container) + local id + id=$(start_rsync_container) # wait for rsync to come up - wait_for_rsync || stop_rsync_container ${id} + wait_for_rsync || { stop_rsync_container "${id}"; return 1; } # NOTE: add --progress to show files being syncd rsync \ @@ -101,15 +112,15 @@ function run_rsync() { --delete \ --prune-empty-dirs \ "$@" \ - $src $dst || { stop_rsync_container ${id}; return 1; } + "$src" "$dst" || { stop_rsync_container "${id}"; return 1; } - stop_rsync_container ${id} + stop_rsync_container "${id}" } function rsync_host_to_container() { - run_rsync ${scriptdir}/.. rsync://localhost:${CROSS_RSYNC_PORT}/volume/go/src/${BUILD_REPO} "$@" + run_rsync "${scriptdir}"/.. rsync://localhost:${CROSS_RSYNC_PORT}/volume/go/src/${BUILD_REPO} "$@" } function rsync_container_to_host() { - run_rsync rsync://localhost:${CROSS_RSYNC_PORT}/volume/go/src/${BUILD_REPO}/ ${scriptdir}/.. "$@" + run_rsync rsync://localhost:${CROSS_RSYNC_PORT}/volume/go/src/${BUILD_REPO}/ "${scriptdir}"/.. "$@" } diff --git a/images/cross/run.sh b/images/cross/run.sh index 038d9013c..da0997406 100755 --- a/images/cross/run.sh +++ b/images/cross/run.sh @@ -14,9 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARGS="$@" +ARGS=( "$@" ) if [ $# -eq 0 ]; then - ARGS=/bin/bash + ARGS=( /bin/bash ) fi BUILDER_USER=${BUILDER_USER:-rook} @@ -24,11 +24,11 @@ BUILDER_GROUP=${BUILDER_GROUP:-rook} BUILDER_UID=${BUILDER_UID:-1000} BUILDER_GID=${BUILDER_GID:-1000} -groupadd -o -g $BUILDER_GID $BUILDER_GROUP 2> /dev/null -useradd -o -m -g $BUILDER_GID -u $BUILDER_UID $BUILDER_USER 2> /dev/null +groupadd -o -g "$BUILDER_GID" "$BUILDER_GROUP" 2> /dev/null +useradd -o -m -g "$BUILDER_GID" -u "$BUILDER_UID" "$BUILDER_USER" 2> /dev/null echo "$BUILDER_USER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers export HOME=/home/${BUILDER_USER} echo "127.0.0.1 $(cat /etc/hostname)" >> /etc/hosts [[ -S /var/run/docker.sock ]] && chmod 666 /var/run/docker.sock -chown -R $BUILDER_UID:$BUILDER_GID $HOME -exec chpst -u :$BUILDER_UID:$BUILDER_GID ${ARGS} +chown -R "$BUILDER_UID":"$BUILDER_GID" "$HOME" +exec chpst -u :"$BUILDER_UID":"$BUILDER_GID" "${ARGS[0]}" diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index cb54a0165..5b968564a 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -445,8 +445,8 @@ func generateRgwTlsCertSecret(s suite.Suite, helper *clients.TestClient, k8sh *u tlscertdir, err := ioutil.TempDir(root, "tlscertdir") require.NoError(s.T(), err, "failed to create directory for TLS certs") defer os.RemoveAll(tlscertdir) - cmdArgs := utils.CommandArgs{Command: filepath.Join(root, "tests/scripts/github-action-helper.sh"), - CmdArgs: []string{"generate_tls_config", tlscertdir, rgwServiceName, namespace}} + cmdArgs := utils.CommandArgs{Command: filepath.Join(root, "tests/scripts/generate-tls-config.sh"), + CmdArgs: []string{tlscertdir, rgwServiceName, namespace}} cmdOut := utils.ExecuteCommand(cmdArgs) require.NoError(s.T(), cmdOut.Err) tlsKeyIn, err := ioutil.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".key")) diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index dea3cad9b..3e54399e6 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -36,7 +36,7 @@ function create_secret_generic { --from-file=vault.key="${TMPDIR}"/vault.key \ --from-file=vault.crt="${TMPDIR}"/vault.crt \ --from-file=vault.ca="${TMPDIR}"/vault.ca - + # for rook kubectl create secret generic vault-ca-cert --namespace ${ROOK_NAMESPACE} --from-file=cert="${TMPDIR}"/vault.ca kubectl create secret generic vault-client-cert --namespace ${ROOK_NAMESPACE} --from-file=cert="${TMPDIR}"/vault.crt @@ -44,7 +44,7 @@ function create_secret_generic { } function vault_helm_tls { - + cat <"${TMPDIR}/"custom-values.yaml global: enabled: true @@ -73,7 +73,7 @@ server: path = "/vault/data" } EOF - + } function deploy_vault { @@ -82,12 +82,12 @@ function deploy_vault { bash "${scriptdir}"/generate-tls-config.sh "${TMPDIR}" ${SERVICE} ${NAMESPACE} create_secret_generic vault_helm_tls - + # Install Vault with Helm helm repo add hashicorp https://helm.releases.hashicorp.com helm install vault hashicorp/vault --values "${TMPDIR}/"custom-values.yaml timeout 120 sh -c 'until kubectl get pods -l app.kubernetes.io/name=vault --field-selector=status.phase=Running|grep vault-0; do sleep 5; done' - + # Unseal Vault VAULT_INIT_TEMP_DIR=$(mktemp) kubectl exec -ti vault-0 -- vault operator init -format "json" -ca-cert /vault/userconfig/vault-server-tls/vault.crt | tee -a "$VAULT_INIT_TEMP_DIR" @@ -95,10 +95,10 @@ function deploy_vault { kubectl exec -ti vault-0 -- vault operator unseal -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$(jq -r ".unseal_keys_b64[$i]" "$VAULT_INIT_TEMP_DIR")" done kubectl get pods -l app.kubernetes.io/name=vault - + # Wait for vault to be ready once unsealed while [[ $(kubectl get pods -l app.kubernetes.io/name=vault -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "waiting vault to be ready" && sleep 1; done - + # Configure Vault ROOT_TOKEN=$(jq -r '.root_token' "$VAULT_INIT_TEMP_DIR") kubectl exec -it vault-0 -- vault login -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$ROOT_TOKEN" @@ -107,7 +107,7 @@ function deploy_vault { kubectl exec -ti vault-0 -- vault secrets enable -ca-cert /vault/userconfig/vault-server-tls/vault.crt -path=rook/ver2 kv-v2 kubectl exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver1 || true # failure is expected kubectl exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver2 || true # failure is expected - + # Configure Vault Policy for Rook echo ' path "rook/*" { @@ -116,10 +116,10 @@ function deploy_vault { path "sys/mounts" { capabilities = ["read"] }'| kubectl exec -i vault-0 -- vault policy write -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook - - + # Create a token for Rook ROOK_TOKEN=$(kubectl exec vault-0 -- vault token create -policy=rook -format json -ca-cert /vault/userconfig/vault-server-tls/vault.crt|jq -r '.auth.client_token'|base64) - + # Configure cluster sed -i "s|ROOK_TOKEN|${ROOK_TOKEN//[$'\t\r\n']}|" tests/manifests/test-kms-vault.yaml } @@ -132,7 +132,7 @@ function validate_rgw_token { RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=) VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=) VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE) - + #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then @@ -152,7 +152,7 @@ function validate_rgw_deployment { function validate_osd_secret { NB_OSD_PVC=$(kubectl -n rook-ceph get pvc|grep -c set1) NB_VAULT_SECRET=$(kubectl -n default exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver1|grep -c set1) - + if [ "$NB_OSD_PVC" -ne "$NB_VAULT_SECRET" ]; then echo "number of osd pvc is $NB_OSD_PVC and number of vault secret is $NB_VAULT_SECRET, mismatch" exit 1 diff --git a/tests/scripts/deploy_admission_controller.sh b/tests/scripts/deploy_admission_controller.sh index c1f4ae644..10970d074 100755 --- a/tests/scripts/deploy_admission_controller.sh +++ b/tests/scripts/deploy_admission_controller.sh @@ -5,10 +5,10 @@ set -eEo pipefail function cleanup() { set +e - kubectl -n rook-ceph delete validatingwebhookconfigurations $WEBHOOK_CONFIG_NAME + kubectl -n rook-ceph delete validatingwebhookconfigurations "$WEBHOOK_CONFIG_NAME" kubectl -n rook-ceph delete certificate rook-admission-controller-cert kubectl -n rook-ceph delete issuers selfsigned-issuer - kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/$CERT_VERSION/cert-manager.yaml + kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/"$CERT_VERSION"/cert-manager.yaml set -e } @@ -17,23 +17,23 @@ function error_log() { kubectl -n rook-ceph get issuer kubectl -n rook-ceph get certificate kubectl -n rook-ceph get secret | grep rook-ceph-admission-controller - kubectl -n rook-ceph get validatingwebhookconfigurations.admissionregistration.k8s.io - kubectl describe validatingwebhookconfigurations.admissionregistration.k8s.io cert-manager-webhook - kubectl describe validatingwebhookconfigurations.admissionregistration.k8s.io rook-ceph-webhook + kubectl -n rook-ceph get validatingwebhookconfigurations.admissionregistration.k8s.io + kubectl describe validatingwebhookconfigurations.admissionregistration.k8s.io cert-manager-webhook + kubectl describe validatingwebhookconfigurations.admissionregistration.k8s.io rook-ceph-webhook kubectl -n cert-manager logs deploy/cert-manager-webhook --tail=10 kubectl -n cert-manager logs deploy/cert-manager-cainjector --tail=10 set -e +x cleanup } -trap cleanup SIGINT +trap cleanup SIGINT trap error_log ERR # Minimum 1.16.0 kubernetes version is required to start the admission controller SERVER_VERSION=$(kubectl version --short | awk -F "." '/Server Version/ {print $2}') MINIMUM_VERSION=16 -if [ ${SERVER_VERSION} -lt ${MINIMUM_VERSION} ]; then +if [ "${SERVER_VERSION}" -lt ${MINIMUM_VERSION} ]; then echo "required minimum kubernetes version 1.$MINIMUM_VERSION.0" exit fi @@ -51,10 +51,33 @@ echo "$BASE_DIR" echo "Deploying cert-manager" kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/$CERT_VERSION/cert-manager.yaml -timeout 150 sh -c 'until [ $(kubectl -n cert-manager get pods --field-selector=status.phase=Running|grep -c ^cert-) -eq 3 ]; do sleep 1 && echo "waiting for cert-manager pods to be in running state"; done' -timeout 20 sh -c 'until [ $(kubectl -n cert-manager get pods -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 3 ]; do sleep 1 && echo "waiting for the pods to be in ready state"; done' -timeout 25 sh -c 'until [ $(kubectl get validatingwebhookconfigurations cert-manager-webhook -o jsonpath='{.webhooks[*].clientConfig.caBundle}' | wc -c) -gt 1 ]; do sleep 1 && echo "waiting for caInjector to inject in caBundle for cert-manager validating webhook"; done' -timeout 25 sh -c 'until [ $(kubectl get mutatingwebhookconfigurations cert-manager-webhook -o jsonpath='{.webhooks[*].clientConfig.caBundle}' | wc -c) -gt 1 ]; do sleep 1 && echo "waiting for caInjector to inject in caBundle for cert-managers mutating webhook"; done' +timeout 150 bash <<-'EOF' + until [ $(kubectl -n cert-manager get pods --field-selector=status.phase=Running | grep -c ^cert-) -eq 3 ]; do + echo "waiting for cert-manager pods to be in running state" + sleep 1 + done +EOF + +timeout 20 bash <<-'EOF' + until [ $(kubectl -n cert-manager get pods -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 3 ]; do + echo "waiting for the pods to be in ready state" + sleep 1 + done +EOF + +timeout 25 bash <<-'EOF' + until [ $(kubectl get validatingwebhookconfigurations cert-manager-webhook -o jsonpath='{.webhooks[*].clientConfig.caBundle}' | wc -c) -gt 1 ]; do + echo "waiting for caInjector to inject in caBundle for cert-manager validating webhook" + sleep 1 + done +EOF + +timeout 25 bash <<-'EOF' + until [ $(kubectl get mutatingwebhookconfigurations cert-manager-webhook -o jsonpath='{.webhooks[*].clientConfig.caBundle}' | wc -c) -gt 1 ]; do + echo "waiting for caInjector to inject in caBundle for cert-managers mutating webhook" + sleep 1 + done +EOF echo "Successfully deployed cert-manager" @@ -87,7 +110,7 @@ EOF echo "Successfully created Issuer and Certificate" echo "Deploying webhook config" -cat ${BASE_DIR}/webhook-config.yaml | \ +< "${BASE_DIR}"/webhook-config.yaml \ "${BASE_DIR}"/webhook-patch-ca-bundle.sh | \ sed -e "s|\${NAMESPACE}|${NAMESPACE}|g" | \ sed -e "s|\${WEBHOOK_CONFIG_NAME}|${WEBHOOK_CONFIG_NAME}|g" | \ diff --git a/tests/scripts/deploy_admission_controller_test.sh b/tests/scripts/deploy_admission_controller_test.sh index 3ba1d0bb1..cd7145c1c 100755 --- a/tests/scripts/deploy_admission_controller_test.sh +++ b/tests/scripts/deploy_admission_controller_test.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash -BASE_DIR=$(cd "$(dirname "$0")"; pwd) +BASE_DIR=$(cd "$(dirname "$0")" && pwd) if [ -z "$KUBECONFIG" ];then if [ -f "$HOME/.kube/config" ]; then export KUBECONFIG="$HOME/.kube/config" else - sudo cp /etc/kubernetes/admin.conf $HOME/ - sudo chown $(id -u):$(id -g) $HOME/admin.conf + sudo cp /etc/kubernetes/admin.conf "$HOME"/ + sudo chown "$(id -u)":"$(id -g)" "$HOME"/admin.conf export KUBECONFIG=$HOME/admin.conf fi fi @@ -15,5 +15,5 @@ fi if [ -f "${BASE_DIR}"/deploy_admission_controller.sh ];then bash "${BASE_DIR}"/deploy_admission_controller.sh else - echo ""${BASE_DIR}"/deploy_admission_controller.sh not found !" -fi \ No newline at end of file + echo "${BASE_DIR}/deploy_admission_controller.sh not found !" +fi diff --git a/tests/scripts/dind-cluster-rbd b/tests/scripts/dind-cluster-rbd deleted file mode 100644 index e0d3b0ab6..000000000 --- a/tests/scripts/dind-cluster-rbd +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -DOCKER_HOST=unix:///opt/outer-docker.sock /usr/bin/docker run --rm -v /sys:/sys --net=host --privileged=true ceph/base rbd "$@" diff --git a/tests/scripts/dind-cluster.sh b/tests/scripts/dind-cluster.sh deleted file mode 100755 index e6eb0080e..000000000 --- a/tests/scripts/dind-cluster.sh +++ /dev/null @@ -1,990 +0,0 @@ -#!/bin/bash -# Copyright 2017 Mirantis -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -if [ $(uname) = Darwin ]; then - readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} -else - readlinkf(){ readlink -f "$1"; } -fi -DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" - -RUN_ON_BTRFS_ANYWAY="${RUN_ON_BTRFS_ANYWAY:-}" -if [[ ! ${RUN_ON_BTRFS_ANYWAY} ]] && docker info| grep -q '^Storage Driver: btrfs'; then - echo "ERROR: Docker is using btrfs storage driver which is unsupported by kubeadm-dind-cluster" >&2 - echo "Please refer to the documentation for more info." >&2 - echo "Set RUN_ON_BTRFS_ANYWAY to non-empty string to continue anyway." >&2 - exit 1 -fi - -# In case of moby linux, -v will not work so we can't -# mount /lib/modules and /boot -is_moby_linux= -if docker info|grep -s '^Kernel Version: .*-moby$' > /dev/null 2>&1; then - is_moby_linux=1 -fi - -#%CONFIG% - -if [[ ! ${EMBEDDED_CONFIG:-} ]]; then - source "${DIND_ROOT}/config.sh" -fi - -CNI_PLUGIN="${CNI_PLUGIN:-bridge}" -DIND_SUBNET="${DIND_SUBNET:-10.192.0.0}" -dind_ip_base="$(echo "${DIND_SUBNET}" | sed 's/\.0$//')" -DIND_IMAGE="${DIND_IMAGE:-}" -BUILD_KUBEADM="${BUILD_KUBEADM:-}" -BUILD_HYPERKUBE="${BUILD_HYPERKUBE:-}" -APISERVER_PORT=${APISERVER_PORT:-8080} -NUM_NODES=${NUM_NODES:-2} -LOCAL_KUBECTL_VERSION=${LOCAL_KUBECTL_VERSION:-} -KUBECTL_DIR="${KUBECTL_DIR:-${HOME}/.kubeadm-dind-cluster}" -DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml}" -SKIP_SNAPSHOT="${SKIP_SNAPSHOT:-}" -E2E_REPORT_DIR="${E2E_REPORT_DIR:-}" - -if [[ ! ${LOCAL_KUBECTL_VERSION:-} && ${DIND_IMAGE:-} =~ :(v[0-9]+\.[0-9]+)$ ]]; then - LOCAL_KUBECTL_VERSION="${BASH_REMATCH[1]}" -fi - -function dind::need-source { - if [[ ! -f cluster/kubectl.sh ]]; then - echo "$0 must be called from the Kubernetes repository root directory" 1>&2 - exit 1 - fi -} - -build_tools_dir="build" -use_k8s_source=y -if [[ ! ${BUILD_KUBEADM} && ! ${BUILD_HYPERKUBE} ]]; then - use_k8s_source= -fi -if [[ ${use_k8s_source} ]]; then - dind::need-source - kubectl=cluster/kubectl.sh - if [[ ! -f ${build_tools_dir}/common.sh ]]; then - build_tools_dir="build-tools" - fi -else - if [[ ! ${LOCAL_KUBECTL_VERSION:-} ]] && ! hash kubectl 2>/dev/null; then - echo "You need kubectl binary in your PATH to use prebuilt DIND image" 1>&2 - exit 1 - fi - kubectl=kubectl -fi - -busybox_image="busybox:1.26.2" -e2e_base_image="golang:1.7.1" -sys_volume_args=() -build_volume_args=() - -function dind::set-build-volume-args { - if [ ${#build_volume_args[@]} -gt 0 ]; then - return 0 - fi - build_container_name= - if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then - build_volume_args=(-v "$PWD:/go/src/k8s.io/kubernetes") - else - build_container_name="$(KUBE_ROOT=$PWD && - . ${build_tools_dir}/common.sh && - kube::build::verify_prereqs >&2 && - echo "${KUBE_DATA_CONTAINER_NAME:-${KUBE_BUILD_DATA_CONTAINER_NAME}}")" - build_volume_args=(--volumes-from "${build_container_name}") - fi -} - -function dind::volume-exists { - local name="$1" - if docker volume inspect "${name}" >& /dev/null; then - return 0 - fi - return 1 -} - -function dind::create-volume { - local name="$1" - docker volume create --label mirantis.kubeadm_dind_cluster --name "${name}" >/dev/null -} - -# We mount /boot and /lib/modules into the container -# below to in case some of the workloads need them. -# This includes virtlet, for instance. Also this may be -# useful in future if we want DIND nodes to pass -# preflight checks. -function dind::prepare-sys-mounts { - if [[ ! ${is_moby_linux} ]]; then - sys_volume_args=() - if [[ -d /boot ]]; then - sys_volume_args+=(-v /boot:/boot) - fi - if [[ -d /lib/modules ]]; then - sys_volume_args+=(-v /lib/modules:/lib/modules) - fi - return 0 - fi - if ! dind::volume-exists kubeadm-dind-sys; then - dind::step "Saving a copy of docker host's /lib/modules" - dind::create-volume kubeadm-dind-sys - # /lib/modules into sys.tar file on kubeadm-dind-sys volume. - local nsenter="nsenter --mount=/proc/1/ns/mnt --" - docker run \ - --rm \ - --privileged \ - -v kubeadm-dind-sys:/dest \ - --pid=host \ - "${busybox_image}" \ - /bin/sh -c \ - "if ${nsenter} test -d /lib/modules; then ${nsenter} tar -C / -c lib/modules >/dest/sys.tar; fi" - fi - sys_volume_args=(-v kubeadm-dind-sys:/dind-sys) -} - -tmp_containers=() - -function dind::cleanup { - if [ ${#tmp_containers[@]} -gt 0 ]; then - for name in "${tmp_containers[@]}"; do - docker rm -vf "${name}" 2>/dev/null - done - fi -} - -trap dind::cleanup EXIT - -function dind::check-image { - local name="$1" - if docker inspect --format 'x' "${name}" >&/dev/null; then - return 0 - else - return 1 - fi -} - -function dind::filter-make-output { - # these messages make output too long and make Travis CI choke - egrep -v --line-buffered 'I[0-9][0-9][0-9][0-9] .*(parse|conversion|defaulter|deepcopy)\.go:[0-9]+\]' -} - -function dind::run-build-command { - # this is like build/run.sh, but it doesn't rsync back the binaries, - # only the generated files. - local cmd=("$@") - ( - # The following is taken from build/run.sh and build/common.sh - # of Kubernetes source tree. It differs in - # --filter='+ /_output/dockerized/bin/**' - # being removed from rsync - . ${build_tools_dir}/common.sh - kube::build::verify_prereqs - kube::build::build_image - kube::build::run_build_command "$@" - - kube::log::status "Syncing out of container" - - kube::build::start_rsyncd_container - - local rsync_extra="" - if (( ${KUBE_VERBOSE} >= 6 )); then - rsync_extra="-iv" - fi - - # The filter syntax for rsync is a little obscure. It filters on files and - # directories. If you don't go in to a directory you won't find any files - # there. Rules are evaluated in order. The last two rules are a little - # magic. '+ */' says to go in to every directory and '- /**' says to ignore - # any file or directory that isn't already specifically allowed. - # - # We are looking to copy out all of the built binaries along with various - # generated files. - kube::build::rsync \ - --filter='- /_temp/' \ - --filter='+ zz_generated.*' \ - --filter='+ generated.proto' \ - --filter='+ *.pb.go' \ - --filter='+ types.go' \ - --filter='+ */' \ - --filter='- /**' \ - "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}" - - kube::build::stop_rsyncd_container - ) -} - -function dind::make-for-linux { - local copy="$1" - shift - dind::step "Building binaries:" "$*" - if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then - dind::step "+ make WHAT=\"$*\"" - make WHAT="$*" 2>&1 | dind::filter-make-output - elif [ "${copy}" = "y" ]; then - dind::step "+ ${build_tools_dir}/run.sh make WHAT=\"$*\"" - "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output - else - dind::step "+ [using the build container] make WHAT=\"$*\"" - dind::run-build-command make WHAT="$*" 2>&1 | dind::filter-make-output - fi -} - -function dind::check-binary { - local filename="$1" - local dockerized="_output/dockerized/bin/linux/amd64/${filename}" - local plain="_output/local/bin/linux/amd64/${filename}" - dind::set-build-volume-args - # FIXME: don't hardcode amd64 arch - if [ -n "${KUBEADM_DIND_LOCAL:-${force_local:-}}" ]; then - if [ -f "${dockerized}" -o -f "${plain}" ]; then - return 0 - fi - elif docker run --rm "${build_volume_args[@]}" \ - "${busybox_image}" \ - test -f "/go/src/k8s.io/kubernetes/${dockerized}" >&/dev/null; then - return 0 - fi - return 1 -} - -function dind::ensure-downloaded-kubectl { - local kubectl_url - local kubectl_sha1 - local kubectl_sha1_linux - local kubectl_sha1_darwin - local kubectl_link - local kubectl_os - local full_kubectl_version - - case "${LOCAL_KUBECTL_VERSION}" in - v1.5) - full_kubectl_version=v1.5.4 - kubectl_sha1_linux=15d8430dc52b1f3772b88bc6a236c8fa58e07c0d - kubectl_sha1_darwin=5e671ba792567574eea48be4eddd844ba2f07c27 - ;; - v1.6) - full_kubectl_version=v1.6.6 - kubectl_sha1_linux=41153558717f3206d37f5bf34232a303ae4dade1 - kubectl_sha1_darwin=9795098e7340764b96a83e50676886d29e792033 - ;; - v1.7) - full_kubectl_version=v1.7.0 - kubectl_sha1_linux=c92ec52c02ec10a1ab54132d3cc99ad6f68c530e - kubectl_sha1_darwin=2e2708b873accafb1be8f328008e3d41a6a32c08 - ;; - "") - return 0 - ;; - *) - echo "Invalid kubectl version" >&2 - exit 1 - esac - - export PATH="${KUBECTL_DIR}:$PATH" - - if [ $(uname) = Darwin ]; then - kubectl_sha1="${kubectl_sha1_darwin}" - kubectl_os=darwin - else - kubectl_sha1="${kubectl_sha1_linux}" - kubectl_os=linux - fi - local link_target="kubectl-${full_kubectl_version}" - local link_name="${KUBECTL_DIR}"/kubectl - if [[ -h "${link_name}" && "$(readlink "${link_name}")" = "${link_target}" ]]; then - return 0 - fi - - local path="${KUBECTL_DIR}/${link_target}" - if [[ ! -f "${path}" ]]; then - mkdir -p "${KUBECTL_DIR}" - curl -sSLo "${path}" "https://storage.googleapis.com/kubernetes-release/release/${full_kubectl_version}/bin/${kubectl_os}/amd64/kubectl" - echo "${kubectl_sha1} ${path}" | sha1sum -c - chmod +x "${path}" - fi - - ln -fs "${link_target}" "${KUBECTL_DIR}/kubectl" -} - -function dind::ensure-kubectl { - if [[ ! ${use_k8s_source} ]]; then - # already checked on startup - dind::ensure-downloaded-kubectl - return 0 - fi - if [ $(uname) = Darwin ]; then - if [ ! -f _output/local/bin/darwin/amd64/kubectl ]; then - dind::step "Building kubectl" - dind::step "+ make WHAT=cmd/kubectl" - make WHAT=cmd/kubectl 2>&1 | dind::filter-make-output - fi - elif ! force_local=y dind::check-binary kubectl; then - dind::make-for-linux y cmd/kubectl - fi -} - -function dind::ensure-binaries { - local -a to_build=() - for name in "$@"; do - if ! dind::check-binary "$(basename "${name}")"; then - to_build+=("${name}") - fi - done - if [ "${#to_build[@]}" -gt 0 ]; then - dind::make-for-linux n "${to_build[@]}" - fi - return 0 -} - -function dind::ensure-network { - if ! docker network inspect kubeadm-dind-net >&/dev/null; then - docker network create --subnet="${DIND_SUBNET}/16" kubeadm-dind-net >/dev/null - fi -} - -function dind::ensure-volume { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume=1 - shift - fi - local name="$1" - if dind::volume-exists "${name}"; then - if [[ ! {reuse_volume} ]]; then - docker volume rm "${name}" >/dev/null - fi - elif [[ ${reuse_volume} ]]; then - echo "*** Failed to locate volume: ${name}" 1>&2 - return 1 - fi - dind::create-volume "${name}" -} - -function dind::run { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume="-r" - shift - fi - local container_name="${1:-}" - local ip="${2:-}" - local netshift="${3:-}" - local portforward="${4:-}" - if [[ $# -gt 4 ]]; then - shift 4 - else - shift $# - fi - local -a opts=(--ip "${ip}" "$@") - local -a args=("systemd.setenv=CNI_PLUGIN=${CNI_PLUGIN}") - - if [[ ! "${container_name}" ]]; then - echo >&2 "Must specify container name" - exit 1 - fi - - # remove any previously created containers with the same name - docker rm -vf "${container_name}" >&/dev/null || true - - if [[ "$portforward" ]]; then - opts+=(-p "$portforward") - fi - - if [[ ${CNI_PLUGIN} = bridge && ${netshift} ]]; then - args+=("systemd.setenv=CNI_BRIDGE_NETWORK_OFFSET=0.0.${netshift}.0") - fi - - opts+=(${sys_volume_args[@]+"${sys_volume_args[@]}"}) - - dind::step "Starting DIND container:" "${container_name}" - - if [[ ! ${is_moby_linux} ]]; then - opts+=(-v /boot:/boot -v /lib/modules:/lib/modules) - fi - - volume_name="kubeadm-dind-${container_name}" - dind::ensure-network - dind::ensure-volume ${reuse_volume} "${volume_name}" - - # TODO: create named volume for binaries and mount it to /k8s - # in case of the source build - - # Start the new container. - docker run \ - -d --privileged \ - --net kubeadm-dind-net \ - --name "${container_name}" \ - --hostname "${container_name}" \ - -l mirantis.kubeadm_dind_cluster \ - -v ${volume_name}:/dind \ - -v /lib/modules:/lib/modules \ - -v /sbin/modprobe:/sbin/modprobe \ - -v /dev:/dev \ - -v /sys/bus:/sys/bus \ - -v /var/run/docker.sock:/opt/outer-docker.sock \ - ${opts[@]+"${opts[@]}"} \ - "${DIND_IMAGE}" \ - ${args[@]+"${args[@]}"} -} - -function dind::kubeadm { - local container_id="$1" - shift - dind::step "Running kubeadm:" "$*" - status=0 - # See image/bare/wrapkubeadm. - # Capturing output is necessary to grab flags for 'kubeadm join' - if ! docker exec "${container_id}" wrapkubeadm "$@" 2>&1 | tee /dev/fd/2; then - echo "*** kubeadm failed" >&2 - return 1 - fi - return ${status} -} - -# function dind::bare { -# local container_name="${1:-}" -# if [[ ! "${container_name}" ]]; then -# echo >&2 "Must specify container name" -# exit 1 -# fi -# shift -# run_opts=(${@+"$@"}) -# dind::run "${container_name}" -# } - -function dind::configure-kubectl { - dind::step "Setting cluster config" - "${kubectl}" config set-cluster dind --server="http://localhost:${APISERVER_PORT}" --insecure-skip-tls-verify=true - "${kubectl}" config set-context dind --cluster=dind - "${kubectl}" config use-context dind -} - -force_make_binaries= -function dind::set-master-opts { - master_opts=() - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - # share binaries pulled from the build container between nodes - dind::ensure-volume "dind-k8s-binaries" - dind::set-build-volume-args - master_opts+=("${build_volume_args[@]}" -v dind-k8s-binaries:/k8s) - local -a bins - if [[ ${BUILD_KUBEADM} ]]; then - master_opts+=(-e KUBEADM_SOURCE=build://) - bins+=(cmd/kubeadm) - fi - if [[ ${BUILD_HYPERKUBE} ]]; then - master_opts+=(-e HYPERKUBE_SOURCE=build://) - bins+=(cmd/hyperkube) - fi - if [[ ${force_make_binaries} ]]; then - dind::make-for-linux n "${bins[@]}" - else - dind::ensure-binaries "${bins[@]}" - fi - fi -} - -cached_use_rbac= -function dind::use-rbac { - # we use rbac in case of k8s 1.6 - if [[ ${cached_use_rbac} ]]; then - [[ ${cached_use_rbac} = 1 ]] && return 0 || return 1 - fi - cached_use_rbac=0 - if "${kubectl}" version --short >& /dev/null && ! "${kubectl}" version --short | grep -q 'Server Version: v1\.5\.'; then - cached_use_rbac=1 - return 0 - fi - return 1 -} - -function dind::deploy-dashboard { - dind::step "Deploying k8s dashboard" - "${kubectl}" create -f "${DASHBOARD_URL}" - if dind::use-rbac; then - # https://kubernetes-io-vnext-staging.netlify.com/docs/admin/authorization/rbac/#service-account-permissions - # Thanks @liggitt for the hint - "${kubectl}" create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default - fi -} - -function dind::init { - local -a opts - dind::set-master-opts - local master_ip="${dind_ip_base}.2" - local container_id=$(dind::run kube-master "${master_ip}" 1 127.0.0.1:${APISERVER_PORT}:8080 ${master_opts[@]+"${master_opts[@]}"}) - # FIXME: I tried using custom tokens with 'kubeadm ex token create' but join failed with: - # 'failed to parse response as JWS object [square/go-jose: compact JWS format must have three parts]' - # So we just pick the line from 'kubeadm init' output - local kube_version_flag="" - if [[ ${BUILD_KUBEADM} ]]; then - # FIXME: this is temporary fix for kubeadm trying to get a non-existent release URL. - # It doesn't change the fact that we're deploying custom-built k8s version - kube_version_flag="--kubernetes-version=stable-1.6" - fi - kubeadm_join_flags="$(dind::kubeadm "${container_id}" init --pod-network-cidr=10.244.0.0/16 --skip-preflight-checks ${kube_version_flag} "$@" | grep '^ *kubeadm join' | sed 's/^ *kubeadm join //')" - dind::configure-kubectl - dind::deploy-dashboard -} - -function dind::create-node-container { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume="-r" - shift - fi - # if there's just one node currently, it's master, thus we need to use - # kube-node-1 hostname, if there are two nodes, we should pick - # kube-node-2 and so on - local next_node_index=${1:-$(docker ps -q --filter=label=mirantis.kubeadm_dind_cluster | wc -l | sed 's/^ *//g')} - local node_ip="${dind_ip_base}.$((next_node_index + 2))" - local -a opts - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - opts+=(-v dind-k8s-binaries:/k8s) - if [[ ${BUILD_KUBEADM} ]]; then - opts+=(-e KUBEADM_SOURCE=build://) - fi - if [[ ${BUILD_HYPERKUBE} ]]; then - opts+=(-e HYPERKUBE_SOURCE=build://) - fi - fi - dind::run ${reuse_volume} kube-node-${next_node_index} ${node_ip} $((next_node_index + 1)) "" ${opts[@]+"${opts[@]}"} -} - -function dind::join { - local container_id="$1" - shift - dind::kubeadm "${container_id}" join --skip-preflight-checks "$@" >/dev/null -} - -function dind::escape-e2e-name { - sed 's/[]\$*.^|()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n' -} - -function dind::accelerate-kube-dns { - dind::step "Patching kube-dns deployment to make it start faster" - # Could do this on the host, too, but we don't want to require jq here - # TODO: do this in wrapkubeadm - # 'kubectl version --short' is a quick check for kubectl 1.4 - # which doesn't support 'kubectl apply --force' - docker exec kube-master /bin/bash -c \ - "kubectl get deployment kube-dns -n kube-system -o json | jq '.spec.template.spec.containers[0].readinessProbe.initialDelaySeconds = 3|.spec.template.spec.containers[0].readinessProbe.periodSeconds = 3' | if kubectl version --short >&/dev/null; then kubectl apply --force -f -; else kubectl apply -f -; fi" -} - -function dind::component-ready { - local label="$1" - local out - if ! out="$("${kubectl}" get pod -l "${label}" -n kube-system \ - -o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then - return 1 - fi - if ! grep -v False <<<"${out}" | grep -q True; then - return 1 - fi - return 0 -} - -function dind::kill-failed-pods { - local pods - # workaround for https://github.com/kubernetes/kubernetes/issues/36482 - if ! pods="$(kubectl get pod -n kube-system -o jsonpath='{ .items[?(@.status.phase == "Failed")].metadata.name }' 2>/dev/null)"; then - return - fi - for name in ${pods}; do - kubectl delete pod --now -n kube-system "${name}" >&/dev/null || true - done -} - -function dind::wait-for-ready { - dind::step "Waiting for kube-proxy and the nodes" - local proxy_ready - local nodes_ready - local n=3 - while true; do - dind::kill-failed-pods - if "${kubectl}" get nodes 2>/dev/null| grep -q NotReady; then - nodes_ready= - else - nodes_ready=y - fi - if dind::component-ready k8s-app=kube-proxy; then - proxy_ready=y - else - proxy_ready= - fi - if [[ ${nodes_ready} && ${proxy_ready} ]]; then - if ((--n == 0)); then - echo "[done]" >&2 - break - fi - else - n=3 - fi - echo -n "." >&2 - sleep 1 - done - - dind::step "Bringing up kube-dns and kubernetes-dashboard" - "${kubectl}" scale deployment --replicas=1 -n kube-system kube-dns - "${kubectl}" scale deployment --replicas=1 -n kube-system kubernetes-dashboard - - while ! dind::component-ready k8s-app=kube-dns || ! dind::component-ready app=kubernetes-dashboard; do - echo -n "." >&2 - dind::kill-failed-pods - sleep 1 - done - echo "[done]" >&2 - - "${kubectl}" get nodes >&2 - dind::step "Access dashboard at:" "http://localhost:${APISERVER_PORT}/ui" -} - -function dind::up { - dind::down - dind::init - local master_ip="$(docker inspect --format="{{.NetworkSettings.IPAddress}}" kube-master)" - # pre-create node containers sequentially so they get predictable IPs - local -a node_containers - for ((n=1; n <= NUM_NODES; n++)); do - dind::step "Starting node container:" ${n} - if ! container_id="$(dind::create-node-container ${n})"; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - node_containers+=(${container_id}) - dind::step "Node container started:" ${n} - fi - done - status=0 - local -a pids - for ((n=1; n <= NUM_NODES; n++)); do - ( - dind::step "Joining node:" ${n} - container_id="${node_containers[n-1]}" - if ! dind::join ${container_id} ${kubeadm_join_flags}; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - dind::step "Node joined:" ${n} - fi - )& - pids[${n}]=$! - done - if ((NUM_NODES > 0)); then - for pid in ${pids[*]}; do - wait ${pid} - done - else - # FIXME: this may fail depending on k8s/kubeadm version - "${kubectl}" taint nodes kube-master node-role.kubernetes.io/master- || true - fi - case "${CNI_PLUGIN}" in - bridge) - ;; - flannel) - if dind::use-rbac; then - curl -sSL "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml?raw=true" | "${kubectl}" create -f - - fi - # without --validate=false this will fail on older k8s versions - curl -sSL "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true" | "${kubectl}" create --validate=false -f - - ;; - calico) - if dind::use-rbac; then - "${kubectl}" apply -f http://docs.projectcalico.org/v2.1/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml - else - "${kubectl}" apply -f http://docs.projectcalico.org/v2.0/getting-started/kubernetes/installation/hosted/kubeadm/calico.yaml - fi - ;; - weave) - if dind::use-rbac; then - "${kubectl}" apply -f "https://github.com/weaveworks/weave/blob/master/prog/weave-kube/weave-daemonset-k8s-1.6.yaml?raw=true" - else - "${kubectl}" apply -f https://git.io/weave-kube - fi - ;; - *) - echo "Unsupported CNI plugin '${CNI_PLUGIN}'" >&2 - ;; - esac - dind::accelerate-kube-dns - if [[ ${CNI_PLUGIN} != bridge ]]; then - # This is especially important in case of Calico - - # the cluster will not recover after snapshotting - # (at least not after restarting from the snapshot) - # if Calico installation is interrupted - dind::wait-for-ready - fi -} - -function dind::snapshot_container { - local container_name="$1" - docker exec -i ${container_name} /usr/local/bin/snapshot prepare - docker diff ${container_name} | docker exec -i ${container_name} /usr/local/bin/snapshot save -} - -function dind::snapshot { - dind::step "Taking snapshot of the cluster" - dind::snapshot_container kube-master - for ((n=1; n <= NUM_NODES; n++)); do - dind::snapshot_container "kube-node-${n}" - done - dind::wait-for-ready -} - -restore_cmd=restore -function dind::restore_container { - local container_id="$1" - docker exec ${container_id} /usr/local/bin/snapshot "${restore_cmd}" -} - -function dind::restore { - local master_ip="${dind_ip_base}.2" - dind::down - dind::step "Restoring master container" - dind::set-master-opts - for ((n=0; n <= NUM_NODES; n++)); do - ( - if [[ n -eq 0 ]]; then - dind::step "Restoring master container" - dind::restore_container "$(dind::run -r kube-master "${master_ip}" 1 127.0.0.1:${APISERVER_PORT}:8080 ${master_opts[@]+"${master_opts[@]}"})" - dind::step "Master container restored" - else - dind::step "Restoring node container:" ${n} - if ! container_id="$(dind::create-node-container -r ${n})"; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - dind::restore_container "${container_id}" - dind::step "Node container restored:" ${n} - fi - fi - )& - pids[${n}]=$! - done - for pid in ${pids[*]}; do - wait ${pid} - done - # Recheck kubectl config. It's possible that the cluster was started - # on this docker from different host - dind::configure-kubectl - dind::wait-for-ready -} - -function dind::down { - docker ps -a -q --filter=label=mirantis.kubeadm_dind_cluster | while read container_id; do - dind::step "Removing container:" "${container_id}" - docker rm -fv "${container_id}" - done -} - -function dind::remove-volumes { - # docker 1.13+: docker volume ls -q -f label=mirantis.kubeadm_dind_cluster - docker volume ls -q | (grep '^kubeadm-dind' || true) | while read volume_id; do - dind::step "Removing volume:" "${volume_id}" - docker volume rm "${volume_id}" - done -} - -function dind::check-for-snapshot { - if ! dind::volume-exists "kubeadm-dind-kube-master"; then - return 1 - fi - for ((n=1; n <= NUM_NODES; n++)); do - if ! dind::volume-exists "kubeadm-dind-kube-node-${n}"; then - return 1 - fi - done -} - -function dind::do-run-e2e { - local parallel="${1:-}" - local focus="${2:-}" - local skip="${3:-}" - dind::need-source - local test_args="--host=http://localhost:${APISERVER_PORT}" - local -a e2e_volume_opts=() - local term= - if [[ ${focus} ]]; then - test_args="--ginkgo.focus=${focus} ${test_args}" - fi - if [[ ${skip} ]]; then - test_args="--ginkgo.skip=${skip} ${test_args}" - fi - if [[ ${E2E_REPORT_DIR} ]]; then - test_args="--report-dir=/report ${test_args}" - e2e_volume_opts=(-v "${E2E_REPORT_DIR}:/report") - fi - dind::make-for-linux n cmd/kubectl test/e2e/e2e.test - dind::step "Running e2e tests with args:" "${test_args}" - dind::set-build-volume-args - if [ -t 1 ] ; then - term="-it" - test_args="--ginkgo.noColor ${test_args}" - fi - docker run \ - --rm ${term} \ - --net=host \ - "${build_volume_args[@]}" \ - -e KUBERNETES_PROVIDER=dind \ - -e KUBE_MASTER_IP=http://localhost:${APISERVER_PORT} \ - -e KUBE_MASTER=local \ - -e KUBERNETES_CONFORMANCE_TEST=y \ - -e GINKGO_PARALLEL=${parallel} \ - ${e2e_volume_opts[@]+"${e2e_volume_opts[@]}"} \ - -w /go/src/k8s.io/kubernetes \ - "${e2e_base_image}" \ - bash -c "cluster/kubectl.sh config set-cluster dind --server='http://localhost:${APISERVER_PORT}' --insecure-skip-tls-verify=true && - cluster/kubectl.sh config set-context dind --cluster=dind && - cluster/kubectl.sh config use-context dind && - go run hack/e2e.go --v --test -check_version_skew=false --test_args='${test_args}'" -} - -function dind::clean { - dind::down - # dind::remove-images - dind::remove-volumes - if docker network inspect kubeadm-dind-net >&/dev/null; then - docker network rm kubeadm-dind-net - fi -} - -function dind::run-e2e { - local focus="${1:-}" - local skip="${2:-\[Serial\]}" - if [[ "$focus" ]]; then - focus="$(dind::escape-e2e-name "${focus}")" - else - focus="\[Conformance\]" - fi - dind::do-run-e2e y "${focus}" "${skip}" -} - -function dind::run-e2e-serial { - local focus="${1:-}" - local skip="${2:-}" - dind::need-source - if [[ "$focus" ]]; then - focus="$(dind::escape-e2e-name "${focus}")" - else - focus="\[Serial\].*\[Conformance\]" - fi - dind::do-run-e2e n "${focus}" "${skip}" - # TBD: specify filter -} - -function dind::step { - local OPTS="" - if [ "$1" = "-n" ]; then - shift - OPTS+="-n" - fi - GREEN="$1" - shift - if [ -t 2 ] ; then - echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2 - else - echo ${OPTS} "* ${GREEN} $*" 1>&2 - fi -} - -case "${1:-}" in - up) - if [[ ! ( ${DIND_IMAGE} =~ local ) ]]; then - dind::step "Making sure DIND image is up to date" - docker pull "${DIND_IMAGE}" >&2 - fi - - dind::prepare-sys-mounts - dind::ensure-kubectl - if [[ ${SKIP_SNAPSHOT} ]]; then - force_make_binaries=y dind::up - dind::wait-for-ready - elif ! dind::check-for-snapshot; then - force_make_binaries=y dind::up - dind::snapshot - else - dind::restore - fi - ;; - reup) - dind::prepare-sys-mounts - dind::ensure-kubectl - if [[ ${SKIP_SNAPSHOT} ]]; then - force_make_binaries=y dind::up - dind::wait-for-ready - elif ! dind::check-for-snapshot; then - force_make_binaries=y dind::up - dind::snapshot - else - force_make_binaries=y - restore_cmd=update_and_restore - dind::restore - fi - ;; - down) - dind::down - ;; - init) - shift - dind::prepare-sys-mounts - dind::ensure-kubectl - dind::init "$@" - ;; - join) - shift - dind::prepare-sys-mounts - dind::ensure-kubectl - dind::join "$(dind::create-node-container)" "$@" - ;; - # bare) - # shift - # dind::bare "$@" - # ;; - snapshot) - shift - dind::snapshot - ;; - restore) - shift - dind::restore - ;; - clean) - dind::clean - ;; - e2e) - shift - dind::run-e2e "$@" - ;; - e2e-serial) - shift - dind::run-e2e-serial "$@" - ;; - *) - echo "usage:" >&2 - echo " $0 up" >&2 - echo " $0 reup" >&2 - echo " $0 down" >&2 - echo " $0 init kubeadm-args..." >&2 - echo " $0 join kubeadm-args..." >&2 - # echo " $0 bare container_name [docker_options...]" - echo " $0 clean" - echo " $0 e2e [test-name-substring]" >&2 - echo " $0 e2e-serial [test-name-substring]" >&2 - exit 1 - ;; -esac diff --git a/tests/scripts/gen_release_notes.sh b/tests/scripts/gen_release_notes.sh index ac4ad99ac..061435cae 100755 --- a/tests/scripts/gen_release_notes.sh +++ b/tests/scripts/gen_release_notes.sh @@ -16,27 +16,27 @@ if [ -z "${GITHUB_USER}" ] || [ -z "${GITHUB_TOKEN}" ]; then help fi -pr_list=$(git log --pretty="%s" --merges --left-only ${FROM_BRANCH}...${TO_TAG} | grep pull | awk '/Merge pull request/ {print $4}' | cut -c 2-) +pr_list=$(git log --pretty="%s" --merges --left-only "${FROM_BRANCH}"..."${TO_TAG}" | grep pull | awk '/Merge pull request/ {print $4}' | cut -c 2-) # for releases notes function release_notes() { for pr in $pr_list; do # get PR title - backport_pr=$(curl -s -u ${GITHUB_USER}:${GITHUB_TOKEN} https://api.github.com/repos/rook/rook/pulls/${pr} | jq '.title') + backport_pr=$(curl -s -u "${GITHUB_USER}":"${GITHUB_TOKEN}" "https://api.github.com/repos/rook/rook/pulls/${pr}" | jq '.title') # with upstream/release-1.6 v1.6.8, it was giving extra PR numbers, so removing after PR for changing tag is merged. - if [[ "$backport_pr" =~ "build: Update build version to $TO_TAG for the release" ]]; then + if [[ "$backport_pr" =~ ./*"build: Update build version to $TO_TAG"* ]]; then break fi # check if it is manual backport PR or not, for mergify backport PR it will contain "(backport" if [[ "$backport_pr" =~ .*"(backport".* ]]; then # find the PR number after the # - original_pr=$(echo $backport_pr | sed -n -e 's/^.*#//p' | grep -E0o '[0-9]' | tr -d '\n') + original_pr=$(echo "$backport_pr" | sed -n -e 's/^.*#//p' | grep -E0o '[0-9]' | tr -d '\n') else # in manual backport PR, we'll directly fetch the owner and title from the PR number original_pr=$pr fi # get the PR title and PR owner in required format - title_with_user=$(curl -s -u ${GITHUB_USER}:${GITHUB_TOKEN} https://api.github.com/repos/rook/rook/pulls/${original_pr} | jq '.title+ " (#, @"+.user.login+")"') + title_with_user=$(curl -s -u "${GITHUB_USER}":"${GITHUB_TOKEN}" "https://api.github.com/repos/rook/rook/pulls/${original_pr}" | jq '.title+ " (#, @"+.user.login+")"') # add PR number after "#" result=$(echo "$title_with_user" | sed "s/(#/(#$original_pr/" |tail -c +2) # remove last `"` diff --git a/tests/scripts/generate-tls-config.sh b/tests/scripts/generate-tls-config.sh old mode 100644 new mode 100755 diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 555700c48..0269f2e05 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -121,7 +121,7 @@ function build_rook() { tests/scripts/validate_modified_files.sh build docker images if [[ "$build_type" == "build" ]]; then - docker tag $(docker images | awk '/build-/ {print $1}') rook/ceph:master + docker tag "$(docker images | awk '/build-/ {print $1}')" rook/ceph:master fi } @@ -133,7 +133,10 @@ function validate_yaml() { cd cluster/examples/kubernetes/ceph kubectl create -f crds.yaml -f common.yaml # skipping folders and some yamls that are only for openshift. - kubectl create $(ls -I scc.yaml -I "*-openshift.yaml" -I "*.sh" -I "*.py" -p | grep -v / | awk ' { print " -f " $1 } ') --dry-run + manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc*')" + with_f_arg="$(echo "$manifests" | awk '{printf " -f %s",$1}')" # don't add newline + # shellcheck disable=SC2086 # '-f manifest1.yaml -f manifest2.yaml etc.' should not be quoted + kubectl create ${with_f_arg} --dry-run } function create_cluster_prerequisites() { @@ -156,9 +159,18 @@ function deploy_cluster() { } function wait_for_prepare_pod() { - timeout 180 sh -c 'until kubectl -n rook-ceph logs -f job/$(kubectl -n rook-ceph get job -l app=rook-ceph-osd-prepare -o jsonpath='{.items[0].metadata.name}'); do sleep 5; done' || true - timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done' || true - kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}') || true + timeout 180 bash <<-'EOF' + until kubectl -n rook-ceph logs --follow job/$(kubectl -n rook-ceph get job -l app=rook-ceph-osd-prepare -o jsonpath='.items[0].metadata.name}') || true; do + sleep 5 + done +EOF + timeout 60 bash <<-'EOF' + until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers || true; do + echo "waiting for osd container" + sleep 1 + done +EOF + kubectl -n rook-ceph describe job/"$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')" || true kubectl -n rook-ceph describe deploy/rook-ceph-osd-0 || true } @@ -166,7 +178,7 @@ function wait_for_ceph_to_be_ready() { DAEMONS=$1 OSD_COUNT=$2 mkdir test - tests/scripts/validate_cluster.sh $DAEMONS $OSD_COUNT + tests/scripts/validate_cluster.sh "$DAEMONS" "$OSD_COUNT" kubectl -n rook-ceph get pods } @@ -195,12 +207,20 @@ function deploy_first_rook_cluster() { kubectl create -f crds.yaml -f common.yaml -f operator.yaml yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false - yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}1 + yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"1 kubectl create -f cluster-test.yaml -f toolbox.yaml } function wait_for_rgw_pods() { - for i in {1..120}; do kubectl -n $1 get pod -l app=rook-ceph-rgw -o jsonpath='{.items[0].metadata.name}' && echo -e "\nrgw pods found" && break || echo -e "\nwaiting for rgw pods"; sleep 5; done + for _ in {1..120}; do + if kubectl -n "$1" get pod -l app=rook-ceph-rgw -o jsonpath='{.items[0].metadata.name}'; then + echo "rgw pods found" + break + fi + echo "waiting for rgw pods" + sleep 5; + done + } function deploy_second_rook_cluster() { @@ -208,7 +228,7 @@ function deploy_second_rook_cluster() { cd cluster/examples/kubernetes/ceph/ NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f - sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' cluster-test.yaml - yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}2 + yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"2 yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external" yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary kubectl create -f cluster-test.yaml -f toolbox.yaml @@ -224,28 +244,19 @@ function write_object_to_cluster1_read_from_cluster2() { CLUSTER_1_IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") - ACCESS_KEY=$(echo "${BASE64_ACCESS_KEY}" | base64 --decode) - SECRET_KEY=$(echo "${BASE64_SECRET_KEY}" | base64 --decode) - s3cmd --config=s3cfg --access_key="${ACCESS_KEY}" --secret_key="${SECRET_KEY}" --host="${CLUSTER_1_IP_ADDR}" mb s3://bkt - s3cmd --config=s3cfg --access_key="${ACCESS_KEY}" --secret_key="${SECRET_KEY}" --host="${CLUSTER_1_IP_ADDR}" put ./1M.dat s3://bkt + ACCESS_KEY=$(echo ${BASE64_ACCESS_KEY} | base64 --decode) + SECRET_KEY=$(echo ${BASE64_SECRET_KEY} | base64 --decode) + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_1_IP_ADDR} mb s3://bkt + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_1_IP_ADDR} put ./1M.dat s3://bkt CLUSTER_2_IP_ADDR=$(kubectl -n rook-ceph-secondary get svc rook-ceph-rgw-zone-b-multisite-store -o jsonpath="{.spec.clusterIP}") - s3cmd --config=s3cfg --access_key="${ACCESS_KEY}" --secret_key="${SECRET_KEY}" --host="${CLUSTER_2_IP_ADDR}" get s3://bkt/1M.dat 1M-get.dat --force + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_2_IP_ADDR} get s3://bkt/1M.dat 1M-get.dat --force diff 1M.dat 1M-get.dat } -selected_function="$1" -if [ "$selected_function" = "generate_tls_config" ]; then - scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - bash "${scriptdir}"/generate-tls-config.sh "$2" "$3" "$4" "$5" -elif [ "$selected_function" = "wait_for_ceph_to_be_ready" ]; then - $selected_function "$2" "$3" -elif [ "$selected_function" = "wait_for_rgw_pods" ]; then - $selected_function "$2" -else - $selected_function -fi - -if [ $? -ne 0 ]; then - echo "Function call to '$selected_function' was not successful" >&2 +FUNCTION="$1" +shift # remove function arg now that we've recorded it +# call the function with the remainder of the user-provided args +if ! $FUNCTION "$@"; then + echo "Call to $FUNCTION was not successful" >&2 exit 1 fi diff --git a/tests/scripts/helm.sh b/tests/scripts/helm.sh index c2f85c01c..0c70a010e 100755 --- a/tests/scripts/helm.sh +++ b/tests/scripts/helm.sh @@ -1,9 +1,7 @@ #!/bin/bash +e -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" temp="/tmp/rook-tests-scripts-helm" -HELM="${temp}/helm" helm_version="${HELM_VERSION:-"v3.6.2"}" arch="${ARCH:-}" @@ -29,16 +27,12 @@ detectArch() { install() { # Download and unpack helm - if [ -x "${TEST_HELM_PATH}" ]; then - HELM="${TEST_HELM_PATH}" - else - local dist - dist="$(uname -s)" - dist=$(echo "${dist}" | tr "[:upper:]" "[:lower:]") - mkdir -p "${temp}" - wget "https://get.helm.sh/helm-${helm_version}-${dist}-${arch}.tar.gz" -O "${temp}/helm.tar.gz" - tar -C "${temp}" -xvf "${temp}/helm.tar.gz" --strip-components 1 - fi + local dist + dist="$(uname -s)" + dist=$(echo "${dist}" | tr "[:upper:]" "[:lower:]") + mkdir -p "${temp}" + wget "https://get.helm.sh/helm-${helm_version}-${dist}-${arch}.tar.gz" -O "${temp}/helm.tar.gz" + tar -C "${temp}" -xvf "${temp}/helm.tar.gz" --strip-components 1 } if [ -z "${arch}" ]; then diff --git a/tests/scripts/k8s-vagrant-multi-node.sh b/tests/scripts/k8s-vagrant-multi-node.sh deleted file mode 100755 index e51d0f924..000000000 --- a/tests/scripts/k8s-vagrant-multi-node.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -e - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -REPO_DIR="${REPO_DIR:-${scriptdir}/../../.cache/k8s-vagrant-multi-node/}" -# shellcheck disable=SC1090 -source "${scriptdir}/../../build/common.sh" - -function init() { - if [ ! -d "${REPO_DIR}" ]; then - echo "k8s-vagrant-multi-node not found in rook cache dir. Cloning.." - mkdir -p "${REPO_DIR}" - git clone https://github.com/galexrt/k8s-vagrant-multi-node.git "${REPO_DIR}" - # checkout latest tag of the repo initially after clone - git -C "${REPO_DIR}" checkout "$(git -C "${REPO_DIR}" describe --tags `git rev-list --tags --max-count=1`)" - else - git -C "${REPO_DIR}" pull || { echo "git pull failed with exit code $?. continuing as the repo is already there ..."; } - fi -} - -# Deletes pods with 'rook-' prefix. Namespace is expected as the first argument -function delete_rook_pods() { - for P in $(kubectl get pods -n "$1" | awk "/$2/ {print \$1}"); do - kubectl delete pod "$P" -n "$1" - done -} - -# current kubectl context == minikube, returns boolean -function check_context() { - if [ "$(kubectl config view 2>/dev/null | awk '/current-context/ {print $NF}')" = "minikube" ]; then - return 0 - fi - - return 1 -} - -function copy_image_to_cluster() { - local build_image=$1 - local final_image=$2 - make load-image IMG="${build_image}" TAG="${final_image}" -} - -function copy_images() { - if [[ "$1" == "" || "$1" == "ceph" ]]; then - echo "copying ceph images" - copy_image_to_cluster "${BUILD_REGISTRY}/ceph-amd64" rook/ceph:master - fi - - if [[ "$1" == "" || "$1" == "cassandra" ]]; then - echo "copying cassandra image" - copy_image_to_cluster "${BUILD_REGISTRY}/cassandra-amd64" rook/cassandra:master - fi - - if [[ "$1" == "" || "$1" == "nfs" ]]; then - echo "copying nfs image" - copy_image_to_cluster "${BUILD_REGISTRY}/nfs-amd64" rook/nfs:master - fi -} - -init - -cd "${REPO_DIR}" || { echo "failed to access k8s-vagrant-multi-node dir ${REPO_DIR}. exiting."; exit 1; } - -case "${1:-}" in - status) - make status - ;; - up) - make up - copy_images "${2}" - ;; - update) - copy_images "${2}" - ;; - restart) - if check_context; then - [ "$2" ] && regex=$2 || regex="^rook-" - echo "Restarting Rook pods matching the regex \"$regex\" in the following namespaces.." - for ns in $(kubectl get ns -o name | grep '^rook-'); do - echo "-> $ns" - delete_rook_pods "$ns" $regex - done - else - echo "To prevent accidental data loss acting only on 'minikube' context. No action is taken." - fi - ;; - helm) - echo " copying rook image for helm" - helm_tag="$(cat _output/version)" - copy_image_to_cluster "${BUILD_REGISTRY}/ceph-amd64" "rook/ceph:${helm_tag}" - ;; - clean) - make clean - ;; - shell) - "${SHELL}" - ;; - *) - echo "usage:" >&2 - echo " $0 status" >&2 - echo " $0 up [ceph | cassandra | nfs]" >&2 - echo " $0 update" >&2 - echo " $0 restart" >&2 - echo " $0 helm" >&2 - echo " $0 clean" >&2 - echo " $0 shell - Open a '${SHELL}' shell in the cloned k8s-vagrant-multi-node project." >&2 - echo " $0 help" >&2 - ;; -esac diff --git a/tests/scripts/kubeadm-dind.sh b/tests/scripts/kubeadm-dind.sh deleted file mode 100755 index 259dfd55e..000000000 --- a/tests/scripts/kubeadm-dind.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash +e - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "${scriptdir}/../../build/common.sh" - -tarname=image.tar -tarfile=${WORK_DIR}/tests/${tarname} - -copy_image_to_cluster() { - local final_image=$1 - - mkdir -p ${WORK_DIR}/tests - docker save -o ${tarfile} ${final_image} - for c in kube-master kube-node-1 kube-node-2; do - docker cp ${tarfile} ${c}:/ - docker exec ${c} /bin/bash -c "docker load -i /${tarname}" - done -} - -copy_rbd() { - for c in kube-master kube-node-1 kube-node-2; do - docker cp ${scriptdir}/dind-cluster-rbd ${c}:/bin/rbd - docker exec ${c} /bin/bash -c "chmod +x /bin/rbd" - # hack for Azure, after vm is started first docker pull command fails intermittently - local maxRetry=3 - local cur=1 - while [ $cur -le $maxRetry ]; do - docker exec ${c} /bin/bash -c "docker pull ceph/base" - if [ $? -eq 0 ]; then - break - fi - sleep 1 - ((++cur)) - done - done -} - -# configure dind-cluster -export EMBEDDED_CONFIG=1 -export KUBECTL_DIR=${KUBEADM_DIND_DIR} -export DIND_SUBNET=10.192.0.0 -export APISERVER_PORT=${APISERVER_PORT:-8080} -export NUM_NODES=${NUM_NODES:-2} -export KUBE_VERSION=${KUBE_VERSION:-"v1.6"} -export DIND_IMAGE="${DIND_IMAGE:-mirantis/kubeadm-dind-cluster:${KUBE_VERSION}}" -export CNI_PLUGIN="${CNI_PLUGIN:-bridge}" - -case "${1:-}" in - up) - ${scriptdir}/dind-cluster.sh reup - ${scriptdir}/makeTestImages.sh save amd64 || true - copy_image_to_cluster rook/ceph:master - set +e - copy_image_to_cluster ceph/base ceph/base:latest - set -e - copy_rbd - ;; - down) - ${scriptdir}/dind-cluster.sh down - ;; - clean) - ${scriptdir}/dind-cluster.sh clean - ;; - update) - copy_image_to_cluster rook/ceph:master - ;; - wordpress) - copy_image_to_cluster mysql:5.6 - copy_image_to_cluster wordpress:4.6.1-apache - ;; - *) - echo "usage:" >&2 - echo " $0 up" >&2 - echo " $0 down" >&2 - echo " $0 clean" >&2 - echo " $0 update" >&2 - echo " $0 wordpress" >&2 -esac diff --git a/tests/scripts/kubeadm-install.sh b/tests/scripts/kubeadm-install.sh index 629b480ef..aa9b73eca 100755 --- a/tests/scripts/kubeadm-install.sh +++ b/tests/scripts/kubeadm-install.sh @@ -1,7 +1,6 @@ #!/bin/bash +e KUBE_VERSION=${1:-"v1.15.12"} -ARCH=$(dpkg --print-architecture) null_str= KUBE_INSTALL_VERSION="${KUBE_VERSION/v/$null_str}"-00 @@ -17,7 +16,7 @@ wait_for_dpkg_unlock() { retryInterval=10 until [ ${retry} -ge ${maxRetries} ] do - if [[ `sudo lsof /var/lib/dpkg/lock|wc -l` -le 0 ]]; then + if [[ $(sudo lsof /var/lib/dpkg/lock|wc -l) -le 0 ]]; then break fi ((++retry)) @@ -25,7 +24,7 @@ wait_for_dpkg_unlock() { sleep ${retryInterval} done - if [ ${retry} -ge ${maxRetries} ]; then + if [ "${retry}" -ge ${maxRetries} ]; then echo "Failed after ${maxRetries} attempts! - cannot install kubeadm" exit 1 fi @@ -39,9 +38,7 @@ wait_for_dpkg_unlock sudo apt-get install -y apt-transport-https sudo curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - -sudo cat </etc/apt/sources.list.d/kubernetes.list -deb http://apt.kubernetes.io/ kubernetes-xenial main -EOF +echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list sudo apt-get update wait_for_dpkg_unlock diff --git a/tests/scripts/kubeadm.sh b/tests/scripts/kubeadm.sh index 4203f150a..6e6d4693e 100755 --- a/tests/scripts/kubeadm.sh +++ b/tests/scripts/kubeadm.sh @@ -3,7 +3,7 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" tarname=image.tar -tarfile="${WORK_DIR}/tests/${tarname}" +export tarfile="${WORK_DIR}/tests/${tarname}" export KUBE_VERSION=${KUBE_VERSION:-"v1.15.12"} @@ -38,10 +38,10 @@ usage(){ #install k8s master node install_master(){ - sudo kubeadm init $skippreflightcheck --kubernetes-version ${KUBE_VERSION} + sudo kubeadm init $skippreflightcheck --kubernetes-version "${KUBE_VERSION}" - sudo cp /etc/kubernetes/admin.conf $HOME/ - sudo chown $(id -u):$(id -g) $HOME/admin.conf + sudo cp /etc/kubernetes/admin.conf "$HOME"/ + sudo chown "$(id -u)":"$(id -g)" "$HOME"/admin.conf export KUBECONFIG=$HOME/admin.conf kubectl taint nodes --all node-role.kubernetes.io/master- @@ -50,7 +50,7 @@ install_master(){ echo "wait for K8s master node to be Ready" INC=0 while [[ $INC -lt 20 ]]; do - kube_ready=$(kubectl get node -o jsonpath='{.items['$count'].status.conditions[?(@.reason == "KubeletReady")].status}') + kube_ready=$(kubectl get node -o jsonpath='{.items['"$count"'].status.conditions[?(@.reason == "KubeletReady")].status}') if [ "${kube_ready}" == "True" ]; then break fi @@ -71,7 +71,7 @@ install_master(){ install_node(){ echo "inside install node function" echo "kubeadm join ${1} ${2} ${3} ${4} ${5} $skippreflightcheck" - sudo kubeadm join ${1} ${2} ${3} ${4} ${5} $skippreflightcheck || true + sudo kubeadm join "${1}" "${2}" "${3}" "${4}" "${5}" $skippreflightcheck || true } #wait for all nodes in the cluster to be ready status @@ -79,8 +79,8 @@ wait_for_ready(){ #expect 3 node cluster by default local numberOfNode=${1:-3} local count=0 - sudo cp /etc/kubernetes/admin.conf $HOME/ - sudo chown $(id -u):$(id -g) $HOME/admin.conf + sudo cp /etc/kubernetes/admin.conf "$HOME"/ + sudo chown "$(id -u)":"$(id -g)" "$HOME"/admin.conf export KUBECONFIG=$HOME/admin.conf until [[ $count -eq $numberOfNode ]]; do @@ -113,8 +113,8 @@ wait_for_ready(){ kubeadm_reset() { kubectl delete -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" sudo kubeadm reset --force $skippreflightcheck - rm $HOME/admin.conf - rm -rf $HOME/.kube + rm "$HOME"/admin.conf + rm -rf "$HOME"/.kube sudo apt-get -y remove kubelet sudo apt-get -y remove kubeadm sudo swapon -a @@ -125,7 +125,7 @@ case "${1:-}" in up) sudo sh -c "${scriptdir}/kubeadm-install.sh ${KUBE_VERSION}" install_master - ${scriptdir}/makeTestImages.sh tag ${arch} || true + "${scriptdir}"/makeTestImages.sh tag ${arch} || true ;; clean) kubeadm_reset @@ -143,7 +143,7 @@ case "${1:-}" in ;; node) if [ "$#" -eq 5 ] || [ "$#" -eq 7 ]; then - install_node $3 $4 $5 $6 $7 + install_node "$3" "$4" "$5" "$6" "$7" else echo "invalid arguments for install node" usage @@ -159,7 +159,7 @@ case "${1:-}" in ;; wait) if [ "$#" -eq 2 ]; then - wait_for_ready $2 + wait_for_ready "$2" else echo "invalid number of arguments for wait" usage diff --git a/tests/scripts/localPathPV.sh b/tests/scripts/localPathPV.sh index d370eb8a9..8af20dad0 100755 --- a/tests/scripts/localPathPV.sh +++ b/tests/scripts/localPathPV.sh @@ -37,7 +37,7 @@ sudo test ! -b "${test_scratch_device}" && echo "invalid scratch device, not a b function prepare_node() { sudo rm -rf /var/lib/rook/rook-integration-test sudo mkdir -p /var/lib/rook/rook-integration-test/mon1 /var/lib/rook/rook-integration-test/mon2 /var/lib/rook/rook-integration-test/mon3 - node_name=$(kubectl get nodes -o jsonpath={.items[*].metadata.name}) + node_name=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') kubectl label nodes "${node_name}" rook.io/has-disk=true kubectl delete pv -l type=local } @@ -124,14 +124,14 @@ eof function create_osd_pvc() { local osd_count=$1 local storage=6Gi - + for osd in $(seq 1 "$osd_count"); do path=${test_scratch_device}${osd} if [ "$osd_count" -eq 1 ]; then path=$test_scratch_device storage=10Gi fi - + cat <&2 @@ -36,7 +36,7 @@ case "${1:-}" in load) case "${2:-}" in arm|arm64|amd64) - echo "Loading archived images to docker: $(ls | grep tar)" + echo "Loading archived images to docker: $(ls ./*tar*)" docker load -i "ceph-$2.tar" docker load -i "cassandra-$2.tar" diff --git a/tests/scripts/minikube.sh b/tests/scripts/minikube.sh index cb4723585..361c61843 100755 --- a/tests/scripts/minikube.sh +++ b/tests/scripts/minikube.sh @@ -1,7 +1,7 @@ #!/bin/bash -e scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# shellcheck disable=SC1090 +# source=build/common.sh source "${scriptdir}/../../build/common.sh" function wait_for_ssh() { @@ -22,7 +22,7 @@ function copy_image_to_cluster() { local final_image=$2 local docker_env_tag="${DOCKERCMD}-env" ${DOCKERCMD} save "${build_image}" | \ - (eval "$(minikube ${docker_env_tag} --shell bash)" && \ + (eval "$(minikube "${docker_env_tag}" --shell bash)" && \ ${DOCKERCMD} load && \ ${DOCKERCMD} tag "${build_image}" "${final_image}") } diff --git a/tests/scripts/minishift.sh b/tests/scripts/minishift.sh index ff3a4cb44..9db28c61b 100755 --- a/tests/scripts/minishift.sh +++ b/tests/scripts/minishift.sh @@ -1,15 +1,16 @@ #!/bin/bash -e scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# source=build/common.sh source "${scriptdir}/../../build/common.sh" function wait_for_ssh() { local tries=100 - while (( ${tries} > 0 )) ; do - if `minishift ssh echo connected &> /dev/null` ; then + while (( tries > 0 )) ; do + if minishift ssh echo connected &> /dev/null ; then return 0 fi - tries=$(( ${tries} - 1 )) + (( tries-- )) sleep 0.1 done echo ERROR: ssh did not come up >&2 @@ -19,7 +20,7 @@ function wait_for_ssh() { function copy_image_to_cluster() { local build_image=$1 local final_image=$2 - docker save ${build_image} | (eval $(minishift docker-env --shell bash) && docker load && docker tag ${build_image} ${final_image}) + docker save "${build_image}" | (eval "$(minishift docker-env --shell bash)" && docker load && docker tag "${build_image}" "${final_image}") } # current kubectl context == minishift, returns boolean @@ -37,12 +38,12 @@ MEMORY=${MEMORY:-"3000"} case "${1:-}" in up) echo "starting minishift" - minishift start --memory=${MEMORY} --vm-driver=virtualbox --iso-url centos + minishift start --memory="${MEMORY}" --vm-driver=virtualbox --iso-url centos wait_for_ssh # create a link so the default dataDirHostPath will work for this environment #minishift ssh "sudo mkdir /mnt/sda1/var/lib/rook;sudo ln -s /mnt/sda1/var/lib/rook /var/lib/rook" - copy_image_to_cluster ${BUILD_REGISTRY}/ceph-amd64 rook/ceph:master + copy_image_to_cluster "${BUILD_REGISTRY}"/ceph-amd64 rook/ceph:master ;; down) minishift delete -f @@ -53,7 +54,7 @@ case "${1:-}" in ;; update) echo "updating the rook images" - copy_image_to_cluster ${BUILD_REGISTRY}/ceph-amd64 rook/ceph:master + copy_image_to_cluster "${BUILD_REGISTRY}"/ceph-amd64 rook/ceph:master ;; restart) if check_context; then @@ -73,8 +74,8 @@ case "${1:-}" in ;; helm) echo " copying rook image for helm" - helm_tag="`cat _output/version`" - copy_image_to_cluster ${BUILD_REGISTRY}/rook-amd64 rook/rook:${helm_tag} + helm_tag="cat _output/version" + copy_image_to_cluster "${BUILD_REGISTRY}"/rook-amd64 rook/rook:"${helm_tag}" ;; clean) minishift delete -f diff --git a/tests/scripts/webhook-patch-ca-bundle.sh b/tests/scripts/webhook-patch-ca-bundle.sh index a41388ec1..4618410e4 100755 --- a/tests/scripts/webhook-patch-ca-bundle.sh +++ b/tests/scripts/webhook-patch-ca-bundle.sh @@ -5,7 +5,8 @@ set -o errexit set -o nounset set -o pipefail -export CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') +CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') +export CA_BUNDLE if command -v envsubst >/dev/null 2>&1; then envsubst