diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml new file mode 100644 index 000000000..8e188f70a --- /dev/null +++ b/.github/workflows/shellcheck.yaml @@ -0,0 +1,28 @@ +name: ShellCheck +on: + push: + tags: + - v* + branches: + - master + - release-* + pull_request: + branches: + - master + - release-* + +jobs: + shellcheck: + name: Shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + with: + severity: warning + check_together: 'yes' + disable_matcher: false + additional_files: build/run build/reset build/sed-in-place + ignore: olm + format: gcc diff --git a/build/common.sh b/build/common.sh index 3e9069f32..c5534fcc9 100644 --- a/build/common.sh +++ b/build/common.sh @@ -1,5 +1,5 @@ #!/bin/bash -e - +set -u # Copyright 2016 The Rook Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,31 +18,41 @@ BUILD_HOST=$(hostname) BUILD_REPO=github.com/rook/rook BUILD_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd -P) SHA256CMD=${SHA256CMD:-shasum -a 256} -BUILD_REGISTRY=build-$(echo ${BUILD_HOST}-${BUILD_ROOT} | ${SHA256CMD} | cut -c1-8) +BUILD_REGISTRY=build-$(echo "${BUILD_HOST}"-"${BUILD_ROOT}" | ${SHA256CMD} | cut -c1-8) DOCKERCMD=${DOCKERCMD:-docker} -OUTPUT_DIR=${BUILD_ROOT}/_output -WORK_DIR=${BUILD_ROOT}/.work +export scriptdir +scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export OUTPUT_DIR=${BUILD_ROOT}/_output +export WORK_DIR=${BUILD_ROOT}/.work CACHE_DIR=${BUILD_ROOT}/.cache -KUBEADM_DIND_DIR=${CACHE_DIR}/kubeadm-dind +export KUBEADM_DIND_DIR=${CACHE_DIR}/kubeadm-dind CROSS_IMAGE=${BUILD_REGISTRY}/cross-amd64 CROSS_IMAGE_VOLUME=cross-volume CROSS_RSYNC_PORT=10873 function ver() { - printf "%d%03d%03d%03d" $(echo "$1" | tr '.' ' ') + local full_ver maj min bug build + full_ver="$1" # functions should name input params for easier understanding + maj="$(echo "${full_ver}" | cut -f1 -d'.')" # when splitting a param, name the components for easier understanding + min="$(echo "${full_ver}" | cut -f2 -d'.')" + bug="$(echo "${full_ver}" | cut -f3 -d'.')" + build="$(echo "${full_ver}" | cut -f4 -d'.')" + printf "%d%03d%03d%03d" "${maj}" "${min}" "${bug}" "${build}" } function check_git() { # git version 2.6.6+ through 2.8.3 had a bug with submodules. this makes it hard # to share a cloned directory between host and container # see https://github.com/git/git/blob/master/Documentation/RelNotes/2.8.3.txt#L33 - local gitversion=$(git --version | cut -d" " -f3) - if (( $(ver ${gitversion}) > $(ver 2.6.6) && $(ver ${gitversion}) < $(ver 2.8.3) )); then - echo WARN: your running git version ${gitversion} which has a bug related to relative + local gitversion + gitversion=$(git --version | cut -d" " -f3) + + if (( $(ver "${gitversion}") > $(ver 2.6.6) && $(ver "${gitversion}") < $(ver 2.8.3) )); then + echo WARN: you are running git version "${gitversion}" which has a bug related to relative echo WARN: submodule paths. Please consider upgrading to 2.8.3 or later fi } @@ -56,18 +66,18 @@ function start_rsync_container() { -p ${CROSS_RSYNC_PORT}:873 \ -v ${CROSS_IMAGE_VOLUME}:/volume \ --entrypoint "/tini" \ - ${CROSS_IMAGE} \ + "${CROSS_IMAGE}" \ -- /build/rsyncd.sh } function wait_for_rsync() { # wait for rsync to come up local tries=100 - while (( ${tries} > 0 )) ; do + while (( tries > 0 )); do if rsync "rsync://localhost:${CROSS_RSYNC_PORT}/" &> /dev/null ; then return 0 fi - tries=$(( ${tries} - 1 )) + (( tries-- )) sleep 0.1 done echo ERROR: rsyncd did not come up >&2 @@ -77,8 +87,8 @@ function wait_for_rsync() { function stop_rsync_container() { local id=$1 - ${DOCKERCMD} stop ${id} &> /dev/null || true - ${DOCKERCMD} rm ${id} &> /dev/null || true + ${DOCKERCMD} stop "${id}" &> /dev/null || true + ${DOCKERCMD} rm "${id}" &> /dev/null || true } function run_rsync() { @@ -90,10 +100,11 @@ function run_rsync() { # run the container as an rsyncd daemon so that we can copy the # source tree to the container volume. - local id=$(start_rsync_container) + local id + id=$(start_rsync_container) # wait for rsync to come up - wait_for_rsync || stop_rsync_container ${id} + wait_for_rsync || { stop_rsync_container "${id}"; return 1; } # NOTE: add --progress to show files being syncd rsync \ @@ -101,15 +112,15 @@ function run_rsync() { --delete \ --prune-empty-dirs \ "$@" \ - $src $dst || { stop_rsync_container ${id}; return 1; } + "$src" "$dst" || { stop_rsync_container "${id}"; return 1; } - stop_rsync_container ${id} + stop_rsync_container "${id}" } function rsync_host_to_container() { - run_rsync ${scriptdir}/.. rsync://localhost:${CROSS_RSYNC_PORT}/volume/go/src/${BUILD_REPO} "$@" + run_rsync "${scriptdir}"/.. rsync://localhost:${CROSS_RSYNC_PORT}/volume/go/src/${BUILD_REPO} "$@" } function rsync_container_to_host() { - run_rsync rsync://localhost:${CROSS_RSYNC_PORT}/volume/go/src/${BUILD_REPO}/ ${scriptdir}/.. "$@" + run_rsync rsync://localhost:${CROSS_RSYNC_PORT}/volume/go/src/${BUILD_REPO}/ "${scriptdir}"/.. "$@" } diff --git a/images/cross/run.sh b/images/cross/run.sh index 038d9013c..da0997406 100755 --- a/images/cross/run.sh +++ b/images/cross/run.sh @@ -14,9 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARGS="$@" +ARGS=( "$@" ) if [ $# -eq 0 ]; then - ARGS=/bin/bash + ARGS=( /bin/bash ) fi BUILDER_USER=${BUILDER_USER:-rook} @@ -24,11 +24,11 @@ BUILDER_GROUP=${BUILDER_GROUP:-rook} BUILDER_UID=${BUILDER_UID:-1000} BUILDER_GID=${BUILDER_GID:-1000} -groupadd -o -g $BUILDER_GID $BUILDER_GROUP 2> /dev/null -useradd -o -m -g $BUILDER_GID -u $BUILDER_UID $BUILDER_USER 2> /dev/null +groupadd -o -g "$BUILDER_GID" "$BUILDER_GROUP" 2> /dev/null +useradd -o -m -g "$BUILDER_GID" -u "$BUILDER_UID" "$BUILDER_USER" 2> /dev/null echo "$BUILDER_USER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers export HOME=/home/${BUILDER_USER} echo "127.0.0.1 $(cat /etc/hostname)" >> /etc/hosts [[ -S /var/run/docker.sock ]] && chmod 666 /var/run/docker.sock -chown -R $BUILDER_UID:$BUILDER_GID $HOME -exec chpst -u :$BUILDER_UID:$BUILDER_GID ${ARGS} +chown -R "$BUILDER_UID":"$BUILDER_GID" "$HOME" +exec chpst -u :"$BUILDER_UID":"$BUILDER_GID" "${ARGS[0]}" diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index cb54a0165..5b968564a 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -445,8 +445,8 @@ func generateRgwTlsCertSecret(s suite.Suite, helper *clients.TestClient, k8sh *u tlscertdir, err := ioutil.TempDir(root, "tlscertdir") require.NoError(s.T(), err, "failed to create directory for TLS certs") defer os.RemoveAll(tlscertdir) - cmdArgs := utils.CommandArgs{Command: filepath.Join(root, "tests/scripts/github-action-helper.sh"), - CmdArgs: []string{"generate_tls_config", tlscertdir, rgwServiceName, namespace}} + cmdArgs := utils.CommandArgs{Command: filepath.Join(root, "tests/scripts/generate-tls-config.sh"), + CmdArgs: []string{tlscertdir, rgwServiceName, namespace}} cmdOut := utils.ExecuteCommand(cmdArgs) require.NoError(s.T(), cmdOut.Err) tlsKeyIn, err := ioutil.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".key")) diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index dea3cad9b..3e54399e6 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -36,7 +36,7 @@ function create_secret_generic { --from-file=vault.key="${TMPDIR}"/vault.key \ --from-file=vault.crt="${TMPDIR}"/vault.crt \ --from-file=vault.ca="${TMPDIR}"/vault.ca - + # for rook kubectl create secret generic vault-ca-cert --namespace ${ROOK_NAMESPACE} --from-file=cert="${TMPDIR}"/vault.ca kubectl create secret generic vault-client-cert --namespace ${ROOK_NAMESPACE} --from-file=cert="${TMPDIR}"/vault.crt @@ -44,7 +44,7 @@ function create_secret_generic { } function vault_helm_tls { - + cat <"${TMPDIR}/"custom-values.yaml global: enabled: true @@ -73,7 +73,7 @@ server: path = "/vault/data" } EOF - + } function deploy_vault { @@ -82,12 +82,12 @@ function deploy_vault { bash "${scriptdir}"/generate-tls-config.sh "${TMPDIR}" ${SERVICE} ${NAMESPACE} create_secret_generic vault_helm_tls - + # Install Vault with Helm helm repo add hashicorp https://helm.releases.hashicorp.com helm install vault hashicorp/vault --values "${TMPDIR}/"custom-values.yaml timeout 120 sh -c 'until kubectl get pods -l app.kubernetes.io/name=vault --field-selector=status.phase=Running|grep vault-0; do sleep 5; done' - + # Unseal Vault VAULT_INIT_TEMP_DIR=$(mktemp) kubectl exec -ti vault-0 -- vault operator init -format "json" -ca-cert /vault/userconfig/vault-server-tls/vault.crt | tee -a "$VAULT_INIT_TEMP_DIR" @@ -95,10 +95,10 @@ function deploy_vault { kubectl exec -ti vault-0 -- vault operator unseal -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$(jq -r ".unseal_keys_b64[$i]" "$VAULT_INIT_TEMP_DIR")" done kubectl get pods -l app.kubernetes.io/name=vault - + # Wait for vault to be ready once unsealed while [[ $(kubectl get pods -l app.kubernetes.io/name=vault -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "waiting vault to be ready" && sleep 1; done - + # Configure Vault ROOT_TOKEN=$(jq -r '.root_token' "$VAULT_INIT_TEMP_DIR") kubectl exec -it vault-0 -- vault login -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$ROOT_TOKEN" @@ -107,7 +107,7 @@ function deploy_vault { kubectl exec -ti vault-0 -- vault secrets enable -ca-cert /vault/userconfig/vault-server-tls/vault.crt -path=rook/ver2 kv-v2 kubectl exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver1 || true # failure is expected kubectl exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver2 || true # failure is expected - + # Configure Vault Policy for Rook echo ' path "rook/*" { @@ -116,10 +116,10 @@ function deploy_vault { path "sys/mounts" { capabilities = ["read"] }'| kubectl exec -i vault-0 -- vault policy write -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook - - + # Create a token for Rook ROOK_TOKEN=$(kubectl exec vault-0 -- vault token create -policy=rook -format json -ca-cert /vault/userconfig/vault-server-tls/vault.crt|jq -r '.auth.client_token'|base64) - + # Configure cluster sed -i "s|ROOK_TOKEN|${ROOK_TOKEN//[$'\t\r\n']}|" tests/manifests/test-kms-vault.yaml } @@ -132,7 +132,7 @@ function validate_rgw_token { RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=) VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=) VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE) - + #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then @@ -152,7 +152,7 @@ function validate_rgw_deployment { function validate_osd_secret { NB_OSD_PVC=$(kubectl -n rook-ceph get pvc|grep -c set1) NB_VAULT_SECRET=$(kubectl -n default exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver1|grep -c set1) - + if [ "$NB_OSD_PVC" -ne "$NB_VAULT_SECRET" ]; then echo "number of osd pvc is $NB_OSD_PVC and number of vault secret is $NB_VAULT_SECRET, mismatch" exit 1 diff --git a/tests/scripts/deploy_admission_controller.sh b/tests/scripts/deploy_admission_controller.sh index c1f4ae644..10970d074 100755 --- a/tests/scripts/deploy_admission_controller.sh +++ b/tests/scripts/deploy_admission_controller.sh @@ -5,10 +5,10 @@ set -eEo pipefail function cleanup() { set +e - kubectl -n rook-ceph delete validatingwebhookconfigurations $WEBHOOK_CONFIG_NAME + kubectl -n rook-ceph delete validatingwebhookconfigurations "$WEBHOOK_CONFIG_NAME" kubectl -n rook-ceph delete certificate rook-admission-controller-cert kubectl -n rook-ceph delete issuers selfsigned-issuer - kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/$CERT_VERSION/cert-manager.yaml + kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/"$CERT_VERSION"/cert-manager.yaml set -e } @@ -17,23 +17,23 @@ function error_log() { kubectl -n rook-ceph get issuer kubectl -n rook-ceph get certificate kubectl -n rook-ceph get secret | grep rook-ceph-admission-controller - kubectl -n rook-ceph get validatingwebhookconfigurations.admissionregistration.k8s.io - kubectl describe validatingwebhookconfigurations.admissionregistration.k8s.io cert-manager-webhook - kubectl describe validatingwebhookconfigurations.admissionregistration.k8s.io rook-ceph-webhook + kubectl -n rook-ceph get validatingwebhookconfigurations.admissionregistration.k8s.io + kubectl describe validatingwebhookconfigurations.admissionregistration.k8s.io cert-manager-webhook + kubectl describe validatingwebhookconfigurations.admissionregistration.k8s.io rook-ceph-webhook kubectl -n cert-manager logs deploy/cert-manager-webhook --tail=10 kubectl -n cert-manager logs deploy/cert-manager-cainjector --tail=10 set -e +x cleanup } -trap cleanup SIGINT +trap cleanup SIGINT trap error_log ERR # Minimum 1.16.0 kubernetes version is required to start the admission controller SERVER_VERSION=$(kubectl version --short | awk -F "." '/Server Version/ {print $2}') MINIMUM_VERSION=16 -if [ ${SERVER_VERSION} -lt ${MINIMUM_VERSION} ]; then +if [ "${SERVER_VERSION}" -lt ${MINIMUM_VERSION} ]; then echo "required minimum kubernetes version 1.$MINIMUM_VERSION.0" exit fi @@ -51,10 +51,33 @@ echo "$BASE_DIR" echo "Deploying cert-manager" kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/$CERT_VERSION/cert-manager.yaml -timeout 150 sh -c 'until [ $(kubectl -n cert-manager get pods --field-selector=status.phase=Running|grep -c ^cert-) -eq 3 ]; do sleep 1 && echo "waiting for cert-manager pods to be in running state"; done' -timeout 20 sh -c 'until [ $(kubectl -n cert-manager get pods -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 3 ]; do sleep 1 && echo "waiting for the pods to be in ready state"; done' -timeout 25 sh -c 'until [ $(kubectl get validatingwebhookconfigurations cert-manager-webhook -o jsonpath='{.webhooks[*].clientConfig.caBundle}' | wc -c) -gt 1 ]; do sleep 1 && echo "waiting for caInjector to inject in caBundle for cert-manager validating webhook"; done' -timeout 25 sh -c 'until [ $(kubectl get mutatingwebhookconfigurations cert-manager-webhook -o jsonpath='{.webhooks[*].clientConfig.caBundle}' | wc -c) -gt 1 ]; do sleep 1 && echo "waiting for caInjector to inject in caBundle for cert-managers mutating webhook"; done' +timeout 150 bash <<-'EOF' + until [ $(kubectl -n cert-manager get pods --field-selector=status.phase=Running | grep -c ^cert-) -eq 3 ]; do + echo "waiting for cert-manager pods to be in running state" + sleep 1 + done +EOF + +timeout 20 bash <<-'EOF' + until [ $(kubectl -n cert-manager get pods -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 3 ]; do + echo "waiting for the pods to be in ready state" + sleep 1 + done +EOF + +timeout 25 bash <<-'EOF' + until [ $(kubectl get validatingwebhookconfigurations cert-manager-webhook -o jsonpath='{.webhooks[*].clientConfig.caBundle}' | wc -c) -gt 1 ]; do + echo "waiting for caInjector to inject in caBundle for cert-manager validating webhook" + sleep 1 + done +EOF + +timeout 25 bash <<-'EOF' + until [ $(kubectl get mutatingwebhookconfigurations cert-manager-webhook -o jsonpath='{.webhooks[*].clientConfig.caBundle}' | wc -c) -gt 1 ]; do + echo "waiting for caInjector to inject in caBundle for cert-managers mutating webhook" + sleep 1 + done +EOF echo "Successfully deployed cert-manager" @@ -87,7 +110,7 @@ EOF echo "Successfully created Issuer and Certificate" echo "Deploying webhook config" -cat ${BASE_DIR}/webhook-config.yaml | \ +< "${BASE_DIR}"/webhook-config.yaml \ "${BASE_DIR}"/webhook-patch-ca-bundle.sh | \ sed -e "s|\${NAMESPACE}|${NAMESPACE}|g" | \ sed -e "s|\${WEBHOOK_CONFIG_NAME}|${WEBHOOK_CONFIG_NAME}|g" | \ diff --git a/tests/scripts/deploy_admission_controller_test.sh b/tests/scripts/deploy_admission_controller_test.sh index 3ba1d0bb1..cd7145c1c 100755 --- a/tests/scripts/deploy_admission_controller_test.sh +++ b/tests/scripts/deploy_admission_controller_test.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash -BASE_DIR=$(cd "$(dirname "$0")"; pwd) +BASE_DIR=$(cd "$(dirname "$0")" && pwd) if [ -z "$KUBECONFIG" ];then if [ -f "$HOME/.kube/config" ]; then export KUBECONFIG="$HOME/.kube/config" else - sudo cp /etc/kubernetes/admin.conf $HOME/ - sudo chown $(id -u):$(id -g) $HOME/admin.conf + sudo cp /etc/kubernetes/admin.conf "$HOME"/ + sudo chown "$(id -u)":"$(id -g)" "$HOME"/admin.conf export KUBECONFIG=$HOME/admin.conf fi fi @@ -15,5 +15,5 @@ fi if [ -f "${BASE_DIR}"/deploy_admission_controller.sh ];then bash "${BASE_DIR}"/deploy_admission_controller.sh else - echo ""${BASE_DIR}"/deploy_admission_controller.sh not found !" -fi \ No newline at end of file + echo "${BASE_DIR}/deploy_admission_controller.sh not found !" +fi diff --git a/tests/scripts/dind-cluster-rbd b/tests/scripts/dind-cluster-rbd deleted file mode 100644 index e0d3b0ab6..000000000 --- a/tests/scripts/dind-cluster-rbd +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -DOCKER_HOST=unix:///opt/outer-docker.sock /usr/bin/docker run --rm -v /sys:/sys --net=host --privileged=true ceph/base rbd "$@" diff --git a/tests/scripts/dind-cluster.sh b/tests/scripts/dind-cluster.sh deleted file mode 100755 index e6eb0080e..000000000 --- a/tests/scripts/dind-cluster.sh +++ /dev/null @@ -1,990 +0,0 @@ -#!/bin/bash -# Copyright 2017 Mirantis -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -if [ $(uname) = Darwin ]; then - readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} -else - readlinkf(){ readlink -f "$1"; } -fi -DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" - -RUN_ON_BTRFS_ANYWAY="${RUN_ON_BTRFS_ANYWAY:-}" -if [[ ! ${RUN_ON_BTRFS_ANYWAY} ]] && docker info| grep -q '^Storage Driver: btrfs'; then - echo "ERROR: Docker is using btrfs storage driver which is unsupported by kubeadm-dind-cluster" >&2 - echo "Please refer to the documentation for more info." >&2 - echo "Set RUN_ON_BTRFS_ANYWAY to non-empty string to continue anyway." >&2 - exit 1 -fi - -# In case of moby linux, -v will not work so we can't -# mount /lib/modules and /boot -is_moby_linux= -if docker info|grep -s '^Kernel Version: .*-moby$' > /dev/null 2>&1; then - is_moby_linux=1 -fi - -#%CONFIG% - -if [[ ! ${EMBEDDED_CONFIG:-} ]]; then - source "${DIND_ROOT}/config.sh" -fi - -CNI_PLUGIN="${CNI_PLUGIN:-bridge}" -DIND_SUBNET="${DIND_SUBNET:-10.192.0.0}" -dind_ip_base="$(echo "${DIND_SUBNET}" | sed 's/\.0$//')" -DIND_IMAGE="${DIND_IMAGE:-}" -BUILD_KUBEADM="${BUILD_KUBEADM:-}" -BUILD_HYPERKUBE="${BUILD_HYPERKUBE:-}" -APISERVER_PORT=${APISERVER_PORT:-8080} -NUM_NODES=${NUM_NODES:-2} -LOCAL_KUBECTL_VERSION=${LOCAL_KUBECTL_VERSION:-} -KUBECTL_DIR="${KUBECTL_DIR:-${HOME}/.kubeadm-dind-cluster}" -DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml}" -SKIP_SNAPSHOT="${SKIP_SNAPSHOT:-}" -E2E_REPORT_DIR="${E2E_REPORT_DIR:-}" - -if [[ ! ${LOCAL_KUBECTL_VERSION:-} && ${DIND_IMAGE:-} =~ :(v[0-9]+\.[0-9]+)$ ]]; then - LOCAL_KUBECTL_VERSION="${BASH_REMATCH[1]}" -fi - -function dind::need-source { - if [[ ! -f cluster/kubectl.sh ]]; then - echo "$0 must be called from the Kubernetes repository root directory" 1>&2 - exit 1 - fi -} - -build_tools_dir="build" -use_k8s_source=y -if [[ ! ${BUILD_KUBEADM} && ! ${BUILD_HYPERKUBE} ]]; then - use_k8s_source= -fi -if [[ ${use_k8s_source} ]]; then - dind::need-source - kubectl=cluster/kubectl.sh - if [[ ! -f ${build_tools_dir}/common.sh ]]; then - build_tools_dir="build-tools" - fi -else - if [[ ! ${LOCAL_KUBECTL_VERSION:-} ]] && ! hash kubectl 2>/dev/null; then - echo "You need kubectl binary in your PATH to use prebuilt DIND image" 1>&2 - exit 1 - fi - kubectl=kubectl -fi - -busybox_image="busybox:1.26.2" -e2e_base_image="golang:1.7.1" -sys_volume_args=() -build_volume_args=() - -function dind::set-build-volume-args { - if [ ${#build_volume_args[@]} -gt 0 ]; then - return 0 - fi - build_container_name= - if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then - build_volume_args=(-v "$PWD:/go/src/k8s.io/kubernetes") - else - build_container_name="$(KUBE_ROOT=$PWD && - . ${build_tools_dir}/common.sh && - kube::build::verify_prereqs >&2 && - echo "${KUBE_DATA_CONTAINER_NAME:-${KUBE_BUILD_DATA_CONTAINER_NAME}}")" - build_volume_args=(--volumes-from "${build_container_name}") - fi -} - -function dind::volume-exists { - local name="$1" - if docker volume inspect "${name}" >& /dev/null; then - return 0 - fi - return 1 -} - -function dind::create-volume { - local name="$1" - docker volume create --label mirantis.kubeadm_dind_cluster --name "${name}" >/dev/null -} - -# We mount /boot and /lib/modules into the container -# below to in case some of the workloads need them. -# This includes virtlet, for instance. Also this may be -# useful in future if we want DIND nodes to pass -# preflight checks. -function dind::prepare-sys-mounts { - if [[ ! ${is_moby_linux} ]]; then - sys_volume_args=() - if [[ -d /boot ]]; then - sys_volume_args+=(-v /boot:/boot) - fi - if [[ -d /lib/modules ]]; then - sys_volume_args+=(-v /lib/modules:/lib/modules) - fi - return 0 - fi - if ! dind::volume-exists kubeadm-dind-sys; then - dind::step "Saving a copy of docker host's /lib/modules" - dind::create-volume kubeadm-dind-sys - # /lib/modules into sys.tar file on kubeadm-dind-sys volume. - local nsenter="nsenter --mount=/proc/1/ns/mnt --" - docker run \ - --rm \ - --privileged \ - -v kubeadm-dind-sys:/dest \ - --pid=host \ - "${busybox_image}" \ - /bin/sh -c \ - "if ${nsenter} test -d /lib/modules; then ${nsenter} tar -C / -c lib/modules >/dest/sys.tar; fi" - fi - sys_volume_args=(-v kubeadm-dind-sys:/dind-sys) -} - -tmp_containers=() - -function dind::cleanup { - if [ ${#tmp_containers[@]} -gt 0 ]; then - for name in "${tmp_containers[@]}"; do - docker rm -vf "${name}" 2>/dev/null - done - fi -} - -trap dind::cleanup EXIT - -function dind::check-image { - local name="$1" - if docker inspect --format 'x' "${name}" >&/dev/null; then - return 0 - else - return 1 - fi -} - -function dind::filter-make-output { - # these messages make output too long and make Travis CI choke - egrep -v --line-buffered 'I[0-9][0-9][0-9][0-9] .*(parse|conversion|defaulter|deepcopy)\.go:[0-9]+\]' -} - -function dind::run-build-command { - # this is like build/run.sh, but it doesn't rsync back the binaries, - # only the generated files. - local cmd=("$@") - ( - # The following is taken from build/run.sh and build/common.sh - # of Kubernetes source tree. It differs in - # --filter='+ /_output/dockerized/bin/**' - # being removed from rsync - . ${build_tools_dir}/common.sh - kube::build::verify_prereqs - kube::build::build_image - kube::build::run_build_command "$@" - - kube::log::status "Syncing out of container" - - kube::build::start_rsyncd_container - - local rsync_extra="" - if (( ${KUBE_VERBOSE} >= 6 )); then - rsync_extra="-iv" - fi - - # The filter syntax for rsync is a little obscure. It filters on files and - # directories. If you don't go in to a directory you won't find any files - # there. Rules are evaluated in order. The last two rules are a little - # magic. '+ */' says to go in to every directory and '- /**' says to ignore - # any file or directory that isn't already specifically allowed. - # - # We are looking to copy out all of the built binaries along with various - # generated files. - kube::build::rsync \ - --filter='- /_temp/' \ - --filter='+ zz_generated.*' \ - --filter='+ generated.proto' \ - --filter='+ *.pb.go' \ - --filter='+ types.go' \ - --filter='+ */' \ - --filter='- /**' \ - "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}" - - kube::build::stop_rsyncd_container - ) -} - -function dind::make-for-linux { - local copy="$1" - shift - dind::step "Building binaries:" "$*" - if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then - dind::step "+ make WHAT=\"$*\"" - make WHAT="$*" 2>&1 | dind::filter-make-output - elif [ "${copy}" = "y" ]; then - dind::step "+ ${build_tools_dir}/run.sh make WHAT=\"$*\"" - "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output - else - dind::step "+ [using the build container] make WHAT=\"$*\"" - dind::run-build-command make WHAT="$*" 2>&1 | dind::filter-make-output - fi -} - -function dind::check-binary { - local filename="$1" - local dockerized="_output/dockerized/bin/linux/amd64/${filename}" - local plain="_output/local/bin/linux/amd64/${filename}" - dind::set-build-volume-args - # FIXME: don't hardcode amd64 arch - if [ -n "${KUBEADM_DIND_LOCAL:-${force_local:-}}" ]; then - if [ -f "${dockerized}" -o -f "${plain}" ]; then - return 0 - fi - elif docker run --rm "${build_volume_args[@]}" \ - "${busybox_image}" \ - test -f "/go/src/k8s.io/kubernetes/${dockerized}" >&/dev/null; then - return 0 - fi - return 1 -} - -function dind::ensure-downloaded-kubectl { - local kubectl_url - local kubectl_sha1 - local kubectl_sha1_linux - local kubectl_sha1_darwin - local kubectl_link - local kubectl_os - local full_kubectl_version - - case "${LOCAL_KUBECTL_VERSION}" in - v1.5) - full_kubectl_version=v1.5.4 - kubectl_sha1_linux=15d8430dc52b1f3772b88bc6a236c8fa58e07c0d - kubectl_sha1_darwin=5e671ba792567574eea48be4eddd844ba2f07c27 - ;; - v1.6) - full_kubectl_version=v1.6.6 - kubectl_sha1_linux=41153558717f3206d37f5bf34232a303ae4dade1 - kubectl_sha1_darwin=9795098e7340764b96a83e50676886d29e792033 - ;; - v1.7) - full_kubectl_version=v1.7.0 - kubectl_sha1_linux=c92ec52c02ec10a1ab54132d3cc99ad6f68c530e - kubectl_sha1_darwin=2e2708b873accafb1be8f328008e3d41a6a32c08 - ;; - "") - return 0 - ;; - *) - echo "Invalid kubectl version" >&2 - exit 1 - esac - - export PATH="${KUBECTL_DIR}:$PATH" - - if [ $(uname) = Darwin ]; then - kubectl_sha1="${kubectl_sha1_darwin}" - kubectl_os=darwin - else - kubectl_sha1="${kubectl_sha1_linux}" - kubectl_os=linux - fi - local link_target="kubectl-${full_kubectl_version}" - local link_name="${KUBECTL_DIR}"/kubectl - if [[ -h "${link_name}" && "$(readlink "${link_name}")" = "${link_target}" ]]; then - return 0 - fi - - local path="${KUBECTL_DIR}/${link_target}" - if [[ ! -f "${path}" ]]; then - mkdir -p "${KUBECTL_DIR}" - curl -sSLo "${path}" "https://storage.googleapis.com/kubernetes-release/release/${full_kubectl_version}/bin/${kubectl_os}/amd64/kubectl" - echo "${kubectl_sha1} ${path}" | sha1sum -c - chmod +x "${path}" - fi - - ln -fs "${link_target}" "${KUBECTL_DIR}/kubectl" -} - -function dind::ensure-kubectl { - if [[ ! ${use_k8s_source} ]]; then - # already checked on startup - dind::ensure-downloaded-kubectl - return 0 - fi - if [ $(uname) = Darwin ]; then - if [ ! -f _output/local/bin/darwin/amd64/kubectl ]; then - dind::step "Building kubectl" - dind::step "+ make WHAT=cmd/kubectl" - make WHAT=cmd/kubectl 2>&1 | dind::filter-make-output - fi - elif ! force_local=y dind::check-binary kubectl; then - dind::make-for-linux y cmd/kubectl - fi -} - -function dind::ensure-binaries { - local -a to_build=() - for name in "$@"; do - if ! dind::check-binary "$(basename "${name}")"; then - to_build+=("${name}") - fi - done - if [ "${#to_build[@]}" -gt 0 ]; then - dind::make-for-linux n "${to_build[@]}" - fi - return 0 -} - -function dind::ensure-network { - if ! docker network inspect kubeadm-dind-net >&/dev/null; then - docker network create --subnet="${DIND_SUBNET}/16" kubeadm-dind-net >/dev/null - fi -} - -function dind::ensure-volume { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume=1 - shift - fi - local name="$1" - if dind::volume-exists "${name}"; then - if [[ ! {reuse_volume} ]]; then - docker volume rm "${name}" >/dev/null - fi - elif [[ ${reuse_volume} ]]; then - echo "*** Failed to locate volume: ${name}" 1>&2 - return 1 - fi - dind::create-volume "${name}" -} - -function dind::run { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume="-r" - shift - fi - local container_name="${1:-}" - local ip="${2:-}" - local netshift="${3:-}" - local portforward="${4:-}" - if [[ $# -gt 4 ]]; then - shift 4 - else - shift $# - fi - local -a opts=(--ip "${ip}" "$@") - local -a args=("systemd.setenv=CNI_PLUGIN=${CNI_PLUGIN}") - - if [[ ! "${container_name}" ]]; then - echo >&2 "Must specify container name" - exit 1 - fi - - # remove any previously created containers with the same name - docker rm -vf "${container_name}" >&/dev/null || true - - if [[ "$portforward" ]]; then - opts+=(-p "$portforward") - fi - - if [[ ${CNI_PLUGIN} = bridge && ${netshift} ]]; then - args+=("systemd.setenv=CNI_BRIDGE_NETWORK_OFFSET=0.0.${netshift}.0") - fi - - opts+=(${sys_volume_args[@]+"${sys_volume_args[@]}"}) - - dind::step "Starting DIND container:" "${container_name}" - - if [[ ! ${is_moby_linux} ]]; then - opts+=(-v /boot:/boot -v /lib/modules:/lib/modules) - fi - - volume_name="kubeadm-dind-${container_name}" - dind::ensure-network - dind::ensure-volume ${reuse_volume} "${volume_name}" - - # TODO: create named volume for binaries and mount it to /k8s - # in case of the source build - - # Start the new container. - docker run \ - -d --privileged \ - --net kubeadm-dind-net \ - --name "${container_name}" \ - --hostname "${container_name}" \ - -l mirantis.kubeadm_dind_cluster \ - -v ${volume_name}:/dind \ - -v /lib/modules:/lib/modules \ - -v /sbin/modprobe:/sbin/modprobe \ - -v /dev:/dev \ - -v /sys/bus:/sys/bus \ - -v /var/run/docker.sock:/opt/outer-docker.sock \ - ${opts[@]+"${opts[@]}"} \ - "${DIND_IMAGE}" \ - ${args[@]+"${args[@]}"} -} - -function dind::kubeadm { - local container_id="$1" - shift - dind::step "Running kubeadm:" "$*" - status=0 - # See image/bare/wrapkubeadm. - # Capturing output is necessary to grab flags for 'kubeadm join' - if ! docker exec "${container_id}" wrapkubeadm "$@" 2>&1 | tee /dev/fd/2; then - echo "*** kubeadm failed" >&2 - return 1 - fi - return ${status} -} - -# function dind::bare { -# local container_name="${1:-}" -# if [[ ! "${container_name}" ]]; then -# echo >&2 "Must specify container name" -# exit 1 -# fi -# shift -# run_opts=(${@+"$@"}) -# dind::run "${container_name}" -# } - -function dind::configure-kubectl { - dind::step "Setting cluster config" - "${kubectl}" config set-cluster dind --server="http://localhost:${APISERVER_PORT}" --insecure-skip-tls-verify=true - "${kubectl}" config set-context dind --cluster=dind - "${kubectl}" config use-context dind -} - -force_make_binaries= -function dind::set-master-opts { - master_opts=() - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - # share binaries pulled from the build container between nodes - dind::ensure-volume "dind-k8s-binaries" - dind::set-build-volume-args - master_opts+=("${build_volume_args[@]}" -v dind-k8s-binaries:/k8s) - local -a bins - if [[ ${BUILD_KUBEADM} ]]; then - master_opts+=(-e KUBEADM_SOURCE=build://) - bins+=(cmd/kubeadm) - fi - if [[ ${BUILD_HYPERKUBE} ]]; then - master_opts+=(-e HYPERKUBE_SOURCE=build://) - bins+=(cmd/hyperkube) - fi - if [[ ${force_make_binaries} ]]; then - dind::make-for-linux n "${bins[@]}" - else - dind::ensure-binaries "${bins[@]}" - fi - fi -} - -cached_use_rbac= -function dind::use-rbac { - # we use rbac in case of k8s 1.6 - if [[ ${cached_use_rbac} ]]; then - [[ ${cached_use_rbac} = 1 ]] && return 0 || return 1 - fi - cached_use_rbac=0 - if "${kubectl}" version --short >& /dev/null && ! "${kubectl}" version --short | grep -q 'Server Version: v1\.5\.'; then - cached_use_rbac=1 - return 0 - fi - return 1 -} - -function dind::deploy-dashboard { - dind::step "Deploying k8s dashboard" - "${kubectl}" create -f "${DASHBOARD_URL}" - if dind::use-rbac; then - # https://kubernetes-io-vnext-staging.netlify.com/docs/admin/authorization/rbac/#service-account-permissions - # Thanks @liggitt for the hint - "${kubectl}" create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default - fi -} - -function dind::init { - local -a opts - dind::set-master-opts - local master_ip="${dind_ip_base}.2" - local container_id=$(dind::run kube-master "${master_ip}" 1 127.0.0.1:${APISERVER_PORT}:8080 ${master_opts[@]+"${master_opts[@]}"}) - # FIXME: I tried using custom tokens with 'kubeadm ex token create' but join failed with: - # 'failed to parse response as JWS object [square/go-jose: compact JWS format must have three parts]' - # So we just pick the line from 'kubeadm init' output - local kube_version_flag="" - if [[ ${BUILD_KUBEADM} ]]; then - # FIXME: this is temporary fix for kubeadm trying to get a non-existent release URL. - # It doesn't change the fact that we're deploying custom-built k8s version - kube_version_flag="--kubernetes-version=stable-1.6" - fi - kubeadm_join_flags="$(dind::kubeadm "${container_id}" init --pod-network-cidr=10.244.0.0/16 --skip-preflight-checks ${kube_version_flag} "$@" | grep '^ *kubeadm join' | sed 's/^ *kubeadm join //')" - dind::configure-kubectl - dind::deploy-dashboard -} - -function dind::create-node-container { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume="-r" - shift - fi - # if there's just one node currently, it's master, thus we need to use - # kube-node-1 hostname, if there are two nodes, we should pick - # kube-node-2 and so on - local next_node_index=${1:-$(docker ps -q --filter=label=mirantis.kubeadm_dind_cluster | wc -l | sed 's/^ *//g')} - local node_ip="${dind_ip_base}.$((next_node_index + 2))" - local -a opts - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - opts+=(-v dind-k8s-binaries:/k8s) - if [[ ${BUILD_KUBEADM} ]]; then - opts+=(-e KUBEADM_SOURCE=build://) - fi - if [[ ${BUILD_HYPERKUBE} ]]; then - opts+=(-e HYPERKUBE_SOURCE=build://) - fi - fi - dind::run ${reuse_volume} kube-node-${next_node_index} ${node_ip} $((next_node_index + 1)) "" ${opts[@]+"${opts[@]}"} -} - -function dind::join { - local container_id="$1" - shift - dind::kubeadm "${container_id}" join --skip-preflight-checks "$@" >/dev/null -} - -function dind::escape-e2e-name { - sed 's/[]\$*.^|()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n' -} - -function dind::accelerate-kube-dns { - dind::step "Patching kube-dns deployment to make it start faster" - # Could do this on the host, too, but we don't want to require jq here - # TODO: do this in wrapkubeadm - # 'kubectl version --short' is a quick check for kubectl 1.4 - # which doesn't support 'kubectl apply --force' - docker exec kube-master /bin/bash -c \ - "kubectl get deployment kube-dns -n kube-system -o json | jq '.spec.template.spec.containers[0].readinessProbe.initialDelaySeconds = 3|.spec.template.spec.containers[0].readinessProbe.periodSeconds = 3' | if kubectl version --short >&/dev/null; then kubectl apply --force -f -; else kubectl apply -f -; fi" -} - -function dind::component-ready { - local label="$1" - local out - if ! out="$("${kubectl}" get pod -l "${label}" -n kube-system \ - -o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then - return 1 - fi - if ! grep -v False <<<"${out}" | grep -q True; then - return 1 - fi - return 0 -} - -function dind::kill-failed-pods { - local pods - # workaround for https://github.com/kubernetes/kubernetes/issues/36482 - if ! pods="$(kubectl get pod -n kube-system -o jsonpath='{ .items[?(@.status.phase == "Failed")].metadata.name }' 2>/dev/null)"; then - return - fi - for name in ${pods}; do - kubectl delete pod --now -n kube-system "${name}" >&/dev/null || true - done -} - -function dind::wait-for-ready { - dind::step "Waiting for kube-proxy and the nodes" - local proxy_ready - local nodes_ready - local n=3 - while true; do - dind::kill-failed-pods - if "${kubectl}" get nodes 2>/dev/null| grep -q NotReady; then - nodes_ready= - else - nodes_ready=y - fi - if dind::component-ready k8s-app=kube-proxy; then - proxy_ready=y - else - proxy_ready= - fi - if [[ ${nodes_ready} && ${proxy_ready} ]]; then - if ((--n == 0)); then - echo "[done]" >&2 - break - fi - else - n=3 - fi - echo -n "." >&2 - sleep 1 - done - - dind::step "Bringing up kube-dns and kubernetes-dashboard" - "${kubectl}" scale deployment --replicas=1 -n kube-system kube-dns - "${kubectl}" scale deployment --replicas=1 -n kube-system kubernetes-dashboard - - while ! dind::component-ready k8s-app=kube-dns || ! dind::component-ready app=kubernetes-dashboard; do - echo -n "." >&2 - dind::kill-failed-pods - sleep 1 - done - echo "[done]" >&2 - - "${kubectl}" get nodes >&2 - dind::step "Access dashboard at:" "http://localhost:${APISERVER_PORT}/ui" -} - -function dind::up { - dind::down - dind::init - local master_ip="$(docker inspect --format="{{.NetworkSettings.IPAddress}}" kube-master)" - # pre-create node containers sequentially so they get predictable IPs - local -a node_containers - for ((n=1; n <= NUM_NODES; n++)); do - dind::step "Starting node container:" ${n} - if ! container_id="$(dind::create-node-container ${n})"; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - node_containers+=(${container_id}) - dind::step "Node container started:" ${n} - fi - done - status=0 - local -a pids - for ((n=1; n <= NUM_NODES; n++)); do - ( - dind::step "Joining node:" ${n} - container_id="${node_containers[n-1]}" - if ! dind::join ${container_id} ${kubeadm_join_flags}; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - dind::step "Node joined:" ${n} - fi - )& - pids[${n}]=$! - done - if ((NUM_NODES > 0)); then - for pid in ${pids[*]}; do - wait ${pid} - done - else - # FIXME: this may fail depending on k8s/kubeadm version - "${kubectl}" taint nodes kube-master node-role.kubernetes.io/master- || true - fi - case "${CNI_PLUGIN}" in - bridge) - ;; - flannel) - if dind::use-rbac; then - curl -sSL "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml?raw=true" | "${kubectl}" create -f - - fi - # without --validate=false this will fail on older k8s versions - curl -sSL "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true" | "${kubectl}" create --validate=false -f - - ;; - calico) - if dind::use-rbac; then - "${kubectl}" apply -f http://docs.projectcalico.org/v2.1/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml - else - "${kubectl}" apply -f http://docs.projectcalico.org/v2.0/getting-started/kubernetes/installation/hosted/kubeadm/calico.yaml - fi - ;; - weave) - if dind::use-rbac; then - "${kubectl}" apply -f "https://github.com/weaveworks/weave/blob/master/prog/weave-kube/weave-daemonset-k8s-1.6.yaml?raw=true" - else - "${kubectl}" apply -f https://git.io/weave-kube - fi - ;; - *) - echo "Unsupported CNI plugin '${CNI_PLUGIN}'" >&2 - ;; - esac - dind::accelerate-kube-dns - if [[ ${CNI_PLUGIN} != bridge ]]; then - # This is especially important in case of Calico - - # the cluster will not recover after snapshotting - # (at least not after restarting from the snapshot) - # if Calico installation is interrupted - dind::wait-for-ready - fi -} - -function dind::snapshot_container { - local container_name="$1" - docker exec -i ${container_name} /usr/local/bin/snapshot prepare - docker diff ${container_name} | docker exec -i ${container_name} /usr/local/bin/snapshot save -} - -function dind::snapshot { - dind::step "Taking snapshot of the cluster" - dind::snapshot_container kube-master - for ((n=1; n <= NUM_NODES; n++)); do - dind::snapshot_container "kube-node-${n}" - done - dind::wait-for-ready -} - -restore_cmd=restore -function dind::restore_container { - local container_id="$1" - docker exec ${container_id} /usr/local/bin/snapshot "${restore_cmd}" -} - -function dind::restore { - local master_ip="${dind_ip_base}.2" - dind::down - dind::step "Restoring master container" - dind::set-master-opts - for ((n=0; n <= NUM_NODES; n++)); do - ( - if [[ n -eq 0 ]]; then - dind::step "Restoring master container" - dind::restore_container "$(dind::run -r kube-master "${master_ip}" 1 127.0.0.1:${APISERVER_PORT}:8080 ${master_opts[@]+"${master_opts[@]}"})" - dind::step "Master container restored" - else - dind::step "Restoring node container:" ${n} - if ! container_id="$(dind::create-node-container -r ${n})"; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - dind::restore_container "${container_id}" - dind::step "Node container restored:" ${n} - fi - fi - )& - pids[${n}]=$! - done - for pid in ${pids[*]}; do - wait ${pid} - done - # Recheck kubectl config. It's possible that the cluster was started - # on this docker from different host - dind::configure-kubectl - dind::wait-for-ready -} - -function dind::down { - docker ps -a -q --filter=label=mirantis.kubeadm_dind_cluster | while read container_id; do - dind::step "Removing container:" "${container_id}" - docker rm -fv "${container_id}" - done -} - -function dind::remove-volumes { - # docker 1.13+: docker volume ls -q -f label=mirantis.kubeadm_dind_cluster - docker volume ls -q | (grep '^kubeadm-dind' || true) | while read volume_id; do - dind::step "Removing volume:" "${volume_id}" - docker volume rm "${volume_id}" - done -} - -function dind::check-for-snapshot { - if ! dind::volume-exists "kubeadm-dind-kube-master"; then - return 1 - fi - for ((n=1; n <= NUM_NODES; n++)); do - if ! dind::volume-exists "kubeadm-dind-kube-node-${n}"; then - return 1 - fi - done -} - -function dind::do-run-e2e { - local parallel="${1:-}" - local focus="${2:-}" - local skip="${3:-}" - dind::need-source - local test_args="--host=http://localhost:${APISERVER_PORT}" - local -a e2e_volume_opts=() - local term= - if [[ ${focus} ]]; then - test_args="--ginkgo.focus=${focus} ${test_args}" - fi - if [[ ${skip} ]]; then - test_args="--ginkgo.skip=${skip} ${test_args}" - fi - if [[ ${E2E_REPORT_DIR} ]]; then - test_args="--report-dir=/report ${test_args}" - e2e_volume_opts=(-v "${E2E_REPORT_DIR}:/report") - fi - dind::make-for-linux n cmd/kubectl test/e2e/e2e.test - dind::step "Running e2e tests with args:" "${test_args}" - dind::set-build-volume-args - if [ -t 1 ] ; then - term="-it" - test_args="--ginkgo.noColor ${test_args}" - fi - docker run \ - --rm ${term} \ - --net=host \ - "${build_volume_args[@]}" \ - -e KUBERNETES_PROVIDER=dind \ - -e KUBE_MASTER_IP=http://localhost:${APISERVER_PORT} \ - -e KUBE_MASTER=local \ - -e KUBERNETES_CONFORMANCE_TEST=y \ - -e GINKGO_PARALLEL=${parallel} \ - ${e2e_volume_opts[@]+"${e2e_volume_opts[@]}"} \ - -w /go/src/k8s.io/kubernetes \ - "${e2e_base_image}" \ - bash -c "cluster/kubectl.sh config set-cluster dind --server='http://localhost:${APISERVER_PORT}' --insecure-skip-tls-verify=true && - cluster/kubectl.sh config set-context dind --cluster=dind && - cluster/kubectl.sh config use-context dind && - go run hack/e2e.go --v --test -check_version_skew=false --test_args='${test_args}'" -} - -function dind::clean { - dind::down - # dind::remove-images - dind::remove-volumes - if docker network inspect kubeadm-dind-net >&/dev/null; then - docker network rm kubeadm-dind-net - fi -} - -function dind::run-e2e { - local focus="${1:-}" - local skip="${2:-\[Serial\]}" - if [[ "$focus" ]]; then - focus="$(dind::escape-e2e-name "${focus}")" - else - focus="\[Conformance\]" - fi - dind::do-run-e2e y "${focus}" "${skip}" -} - -function dind::run-e2e-serial { - local focus="${1:-}" - local skip="${2:-}" - dind::need-source - if [[ "$focus" ]]; then - focus="$(dind::escape-e2e-name "${focus}")" - else - focus="\[Serial\].*\[Conformance\]" - fi - dind::do-run-e2e n "${focus}" "${skip}" - # TBD: specify filter -} - -function dind::step { - local OPTS="" - if [ "$1" = "-n" ]; then - shift - OPTS+="-n" - fi - GREEN="$1" - shift - if [ -t 2 ] ; then - echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2 - else - echo ${OPTS} "* ${GREEN} $*" 1>&2 - fi -} - -case "${1:-}" in - up) - if [[ ! ( ${DIND_IMAGE} =~ local ) ]]; then - dind::step "Making sure DIND image is up to date" - docker pull "${DIND_IMAGE}" >&2 - fi - - dind::prepare-sys-mounts - dind::ensure-kubectl - if [[ ${SKIP_SNAPSHOT} ]]; then - force_make_binaries=y dind::up - dind::wait-for-ready - elif ! dind::check-for-snapshot; then - force_make_binaries=y dind::up - dind::snapshot - else - dind::restore - fi - ;; - reup) - dind::prepare-sys-mounts - dind::ensure-kubectl - if [[ ${SKIP_SNAPSHOT} ]]; then - force_make_binaries=y dind::up - dind::wait-for-ready - elif ! dind::check-for-snapshot; then - force_make_binaries=y dind::up - dind::snapshot - else - force_make_binaries=y - restore_cmd=update_and_restore - dind::restore - fi - ;; - down) - dind::down - ;; - init) - shift - dind::prepare-sys-mounts - dind::ensure-kubectl - dind::init "$@" - ;; - join) - shift - dind::prepare-sys-mounts - dind::ensure-kubectl - dind::join "$(dind::create-node-container)" "$@" - ;; - # bare) - # shift - # dind::bare "$@" - # ;; - snapshot) - shift - dind::snapshot - ;; - restore) - shift - dind::restore - ;; - clean) - dind::clean - ;; - e2e) - shift - dind::run-e2e "$@" - ;; - e2e-serial) - shift - dind::run-e2e-serial "$@" - ;; - *) - echo "usage:" >&2 - echo " $0 up" >&2 - echo " $0 reup" >&2 - echo " $0 down" >&2 - echo " $0 init kubeadm-args..." >&2 - echo " $0 join kubeadm-args..." >&2 - # echo " $0 bare container_name [docker_options...]" - echo " $0 clean" - echo " $0 e2e [test-name-substring]" >&2 - echo " $0 e2e-serial [test-name-substring]" >&2 - exit 1 - ;; -esac diff --git a/tests/scripts/gen_release_notes.sh b/tests/scripts/gen_release_notes.sh index ac4ad99ac..061435cae 100755 --- a/tests/scripts/gen_release_notes.sh +++ b/tests/scripts/gen_release_notes.sh @@ -16,27 +16,27 @@ if [ -z "${GITHUB_USER}" ] || [ -z "${GITHUB_TOKEN}" ]; then help fi -pr_list=$(git log --pretty="%s" --merges --left-only ${FROM_BRANCH}...${TO_TAG} | grep pull | awk '/Merge pull request/ {print $4}' | cut -c 2-) +pr_list=$(git log --pretty="%s" --merges --left-only "${FROM_BRANCH}"..."${TO_TAG}" | grep pull | awk '/Merge pull request/ {print $4}' | cut -c 2-) # for releases notes function release_notes() { for pr in $pr_list; do # get PR title - backport_pr=$(curl -s -u ${GITHUB_USER}:${GITHUB_TOKEN} https://api.github.com/repos/rook/rook/pulls/${pr} | jq '.title') + backport_pr=$(curl -s -u "${GITHUB_USER}":"${GITHUB_TOKEN}" "https://api.github.com/repos/rook/rook/pulls/${pr}" | jq '.title') # with upstream/release-1.6 v1.6.8, it was giving extra PR numbers, so removing after PR for changing tag is merged. - if [[ "$backport_pr" =~ "build: Update build version to $TO_TAG for the release" ]]; then + if [[ "$backport_pr" =~ ./*"build: Update build version to $TO_TAG"* ]]; then break fi # check if it is manual backport PR or not, for mergify backport PR it will contain "(backport" if [[ "$backport_pr" =~ .*"(backport".* ]]; then # find the PR number after the # - original_pr=$(echo $backport_pr | sed -n -e 's/^.*#//p' | grep -E0o '[0-9]' | tr -d '\n') + original_pr=$(echo "$backport_pr" | sed -n -e 's/^.*#//p' | grep -E0o '[0-9]' | tr -d '\n') else # in manual backport PR, we'll directly fetch the owner and title from the PR number original_pr=$pr fi # get the PR title and PR owner in required format - title_with_user=$(curl -s -u ${GITHUB_USER}:${GITHUB_TOKEN} https://api.github.com/repos/rook/rook/pulls/${original_pr} | jq '.title+ " (#, @"+.user.login+")"') + title_with_user=$(curl -s -u "${GITHUB_USER}":"${GITHUB_TOKEN}" "https://api.github.com/repos/rook/rook/pulls/${original_pr}" | jq '.title+ " (#, @"+.user.login+")"') # add PR number after "#" result=$(echo "$title_with_user" | sed "s/(#/(#$original_pr/" |tail -c +2) # remove last `"` diff --git a/tests/scripts/generate-tls-config.sh b/tests/scripts/generate-tls-config.sh old mode 100644 new mode 100755 diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 555700c48..0269f2e05 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -121,7 +121,7 @@ function build_rook() { tests/scripts/validate_modified_files.sh build docker images if [[ "$build_type" == "build" ]]; then - docker tag $(docker images | awk '/build-/ {print $1}') rook/ceph:master + docker tag "$(docker images | awk '/build-/ {print $1}')" rook/ceph:master fi } @@ -133,7 +133,10 @@ function validate_yaml() { cd cluster/examples/kubernetes/ceph kubectl create -f crds.yaml -f common.yaml # skipping folders and some yamls that are only for openshift. - kubectl create $(ls -I scc.yaml -I "*-openshift.yaml" -I "*.sh" -I "*.py" -p | grep -v / | awk ' { print " -f " $1 } ') --dry-run + manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc*')" + with_f_arg="$(echo "$manifests" | awk '{printf " -f %s",$1}')" # don't add newline + # shellcheck disable=SC2086 # '-f manifest1.yaml -f manifest2.yaml etc.' should not be quoted + kubectl create ${with_f_arg} --dry-run } function create_cluster_prerequisites() { @@ -156,9 +159,18 @@ function deploy_cluster() { } function wait_for_prepare_pod() { - timeout 180 sh -c 'until kubectl -n rook-ceph logs -f job/$(kubectl -n rook-ceph get job -l app=rook-ceph-osd-prepare -o jsonpath='{.items[0].metadata.name}'); do sleep 5; done' || true - timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done' || true - kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}') || true + timeout 180 bash <<-'EOF' + until kubectl -n rook-ceph logs --follow job/$(kubectl -n rook-ceph get job -l app=rook-ceph-osd-prepare -o jsonpath='.items[0].metadata.name}') || true; do + sleep 5 + done +EOF + timeout 60 bash <<-'EOF' + until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers || true; do + echo "waiting for osd container" + sleep 1 + done +EOF + kubectl -n rook-ceph describe job/"$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')" || true kubectl -n rook-ceph describe deploy/rook-ceph-osd-0 || true } @@ -166,7 +178,7 @@ function wait_for_ceph_to_be_ready() { DAEMONS=$1 OSD_COUNT=$2 mkdir test - tests/scripts/validate_cluster.sh $DAEMONS $OSD_COUNT + tests/scripts/validate_cluster.sh "$DAEMONS" "$OSD_COUNT" kubectl -n rook-ceph get pods } @@ -195,12 +207,20 @@ function deploy_first_rook_cluster() { kubectl create -f crds.yaml -f common.yaml -f operator.yaml yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false - yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}1 + yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"1 kubectl create -f cluster-test.yaml -f toolbox.yaml } function wait_for_rgw_pods() { - for i in {1..120}; do kubectl -n $1 get pod -l app=rook-ceph-rgw -o jsonpath='{.items[0].metadata.name}' && echo -e "\nrgw pods found" && break || echo -e "\nwaiting for rgw pods"; sleep 5; done + for _ in {1..120}; do + if kubectl -n "$1" get pod -l app=rook-ceph-rgw -o jsonpath='{.items[0].metadata.name}'; then + echo "rgw pods found" + break + fi + echo "waiting for rgw pods" + sleep 5; + done + } function deploy_second_rook_cluster() { @@ -208,7 +228,7 @@ function deploy_second_rook_cluster() { cd cluster/examples/kubernetes/ceph/ NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f - sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' cluster-test.yaml - yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}2 + yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"2 yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external" yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary kubectl create -f cluster-test.yaml -f toolbox.yaml @@ -224,28 +244,19 @@ function write_object_to_cluster1_read_from_cluster2() { CLUSTER_1_IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") - ACCESS_KEY=$(echo "${BASE64_ACCESS_KEY}" | base64 --decode) - SECRET_KEY=$(echo "${BASE64_SECRET_KEY}" | base64 --decode) - s3cmd --config=s3cfg --access_key="${ACCESS_KEY}" --secret_key="${SECRET_KEY}" --host="${CLUSTER_1_IP_ADDR}" mb s3://bkt - s3cmd --config=s3cfg --access_key="${ACCESS_KEY}" --secret_key="${SECRET_KEY}" --host="${CLUSTER_1_IP_ADDR}" put ./1M.dat s3://bkt + ACCESS_KEY=$(echo ${BASE64_ACCESS_KEY} | base64 --decode) + SECRET_KEY=$(echo ${BASE64_SECRET_KEY} | base64 --decode) + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_1_IP_ADDR} mb s3://bkt + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_1_IP_ADDR} put ./1M.dat s3://bkt CLUSTER_2_IP_ADDR=$(kubectl -n rook-ceph-secondary get svc rook-ceph-rgw-zone-b-multisite-store -o jsonpath="{.spec.clusterIP}") - s3cmd --config=s3cfg --access_key="${ACCESS_KEY}" --secret_key="${SECRET_KEY}" --host="${CLUSTER_2_IP_ADDR}" get s3://bkt/1M.dat 1M-get.dat --force + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_2_IP_ADDR} get s3://bkt/1M.dat 1M-get.dat --force diff 1M.dat 1M-get.dat } -selected_function="$1" -if [ "$selected_function" = "generate_tls_config" ]; then - scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - bash "${scriptdir}"/generate-tls-config.sh "$2" "$3" "$4" "$5" -elif [ "$selected_function" = "wait_for_ceph_to_be_ready" ]; then - $selected_function "$2" "$3" -elif [ "$selected_function" = "wait_for_rgw_pods" ]; then - $selected_function "$2" -else - $selected_function -fi - -if [ $? -ne 0 ]; then - echo "Function call to '$selected_function' was not successful" >&2 +FUNCTION="$1" +shift # remove function arg now that we've recorded it +# call the function with the remainder of the user-provided args +if ! $FUNCTION "$@"; then + echo "Call to $FUNCTION was not successful" >&2 exit 1 fi diff --git a/tests/scripts/helm.sh b/tests/scripts/helm.sh index c2f85c01c..0c70a010e 100755 --- a/tests/scripts/helm.sh +++ b/tests/scripts/helm.sh @@ -1,9 +1,7 @@ #!/bin/bash +e -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" temp="/tmp/rook-tests-scripts-helm" -HELM="${temp}/helm" helm_version="${HELM_VERSION:-"v3.6.2"}" arch="${ARCH:-}" @@ -29,16 +27,12 @@ detectArch() { install() { # Download and unpack helm - if [ -x "${TEST_HELM_PATH}" ]; then - HELM="${TEST_HELM_PATH}" - else - local dist - dist="$(uname -s)" - dist=$(echo "${dist}" | tr "[:upper:]" "[:lower:]") - mkdir -p "${temp}" - wget "https://get.helm.sh/helm-${helm_version}-${dist}-${arch}.tar.gz" -O "${temp}/helm.tar.gz" - tar -C "${temp}" -xvf "${temp}/helm.tar.gz" --strip-components 1 - fi + local dist + dist="$(uname -s)" + dist=$(echo "${dist}" | tr "[:upper:]" "[:lower:]") + mkdir -p "${temp}" + wget "https://get.helm.sh/helm-${helm_version}-${dist}-${arch}.tar.gz" -O "${temp}/helm.tar.gz" + tar -C "${temp}" -xvf "${temp}/helm.tar.gz" --strip-components 1 } if [ -z "${arch}" ]; then diff --git a/tests/scripts/k8s-vagrant-multi-node.sh b/tests/scripts/k8s-vagrant-multi-node.sh deleted file mode 100755 index e51d0f924..000000000 --- a/tests/scripts/k8s-vagrant-multi-node.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -e - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -REPO_DIR="${REPO_DIR:-${scriptdir}/../../.cache/k8s-vagrant-multi-node/}" -# shellcheck disable=SC1090 -source "${scriptdir}/../../build/common.sh" - -function init() { - if [ ! -d "${REPO_DIR}" ]; then - echo "k8s-vagrant-multi-node not found in rook cache dir. Cloning.." - mkdir -p "${REPO_DIR}" - git clone https://github.com/galexrt/k8s-vagrant-multi-node.git "${REPO_DIR}" - # checkout latest tag of the repo initially after clone - git -C "${REPO_DIR}" checkout "$(git -C "${REPO_DIR}" describe --tags `git rev-list --tags --max-count=1`)" - else - git -C "${REPO_DIR}" pull || { echo "git pull failed with exit code $?. continuing as the repo is already there ..."; } - fi -} - -# Deletes pods with 'rook-' prefix. Namespace is expected as the first argument -function delete_rook_pods() { - for P in $(kubectl get pods -n "$1" | awk "/$2/ {print \$1}"); do - kubectl delete pod "$P" -n "$1" - done -} - -# current kubectl context == minikube, returns boolean -function check_context() { - if [ "$(kubectl config view 2>/dev/null | awk '/current-context/ {print $NF}')" = "minikube" ]; then - return 0 - fi - - return 1 -} - -function copy_image_to_cluster() { - local build_image=$1 - local final_image=$2 - make load-image IMG="${build_image}" TAG="${final_image}" -} - -function copy_images() { - if [[ "$1" == "" || "$1" == "ceph" ]]; then - echo "copying ceph images" - copy_image_to_cluster "${BUILD_REGISTRY}/ceph-amd64" rook/ceph:master - fi - - if [[ "$1" == "" || "$1" == "cassandra" ]]; then - echo "copying cassandra image" - copy_image_to_cluster "${BUILD_REGISTRY}/cassandra-amd64" rook/cassandra:master - fi - - if [[ "$1" == "" || "$1" == "nfs" ]]; then - echo "copying nfs image" - copy_image_to_cluster "${BUILD_REGISTRY}/nfs-amd64" rook/nfs:master - fi -} - -init - -cd "${REPO_DIR}" || { echo "failed to access k8s-vagrant-multi-node dir ${REPO_DIR}. exiting."; exit 1; } - -case "${1:-}" in - status) - make status - ;; - up) - make up - copy_images "${2}" - ;; - update) - copy_images "${2}" - ;; - restart) - if check_context; then - [ "$2" ] && regex=$2 || regex="^rook-" - echo "Restarting Rook pods matching the regex \"$regex\" in the following namespaces.." - for ns in $(kubectl get ns -o name | grep '^rook-'); do - echo "-> $ns" - delete_rook_pods "$ns" $regex - done - else - echo "To prevent accidental data loss acting only on 'minikube' context. No action is taken." - fi - ;; - helm) - echo " copying rook image for helm" - helm_tag="$(cat _output/version)" - copy_image_to_cluster "${BUILD_REGISTRY}/ceph-amd64" "rook/ceph:${helm_tag}" - ;; - clean) - make clean - ;; - shell) - "${SHELL}" - ;; - *) - echo "usage:" >&2 - echo " $0 status" >&2 - echo " $0 up [ceph | cassandra | nfs]" >&2 - echo " $0 update" >&2 - echo " $0 restart" >&2 - echo " $0 helm" >&2 - echo " $0 clean" >&2 - echo " $0 shell - Open a '${SHELL}' shell in the cloned k8s-vagrant-multi-node project." >&2 - echo " $0 help" >&2 - ;; -esac diff --git a/tests/scripts/kubeadm-dind.sh b/tests/scripts/kubeadm-dind.sh deleted file mode 100755 index 259dfd55e..000000000 --- a/tests/scripts/kubeadm-dind.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash +e - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "${scriptdir}/../../build/common.sh" - -tarname=image.tar -tarfile=${WORK_DIR}/tests/${tarname} - -copy_image_to_cluster() { - local final_image=$1 - - mkdir -p ${WORK_DIR}/tests - docker save -o ${tarfile} ${final_image} - for c in kube-master kube-node-1 kube-node-2; do - docker cp ${tarfile} ${c}:/ - docker exec ${c} /bin/bash -c "docker load -i /${tarname}" - done -} - -copy_rbd() { - for c in kube-master kube-node-1 kube-node-2; do - docker cp ${scriptdir}/dind-cluster-rbd ${c}:/bin/rbd - docker exec ${c} /bin/bash -c "chmod +x /bin/rbd" - # hack for Azure, after vm is started first docker pull command fails intermittently - local maxRetry=3 - local cur=1 - while [ $cur -le $maxRetry ]; do - docker exec ${c} /bin/bash -c "docker pull ceph/base" - if [ $? -eq 0 ]; then - break - fi - sleep 1 - ((++cur)) - done - done -} - -# configure dind-cluster -export EMBEDDED_CONFIG=1 -export KUBECTL_DIR=${KUBEADM_DIND_DIR} -export DIND_SUBNET=10.192.0.0 -export APISERVER_PORT=${APISERVER_PORT:-8080} -export NUM_NODES=${NUM_NODES:-2} -export KUBE_VERSION=${KUBE_VERSION:-"v1.6"} -export DIND_IMAGE="${DIND_IMAGE:-mirantis/kubeadm-dind-cluster:${KUBE_VERSION}}" -export CNI_PLUGIN="${CNI_PLUGIN:-bridge}" - -case "${1:-}" in - up) - ${scriptdir}/dind-cluster.sh reup - ${scriptdir}/makeTestImages.sh save amd64 || true - copy_image_to_cluster rook/ceph:master - set +e - copy_image_to_cluster ceph/base ceph/base:latest - set -e - copy_rbd - ;; - down) - ${scriptdir}/dind-cluster.sh down - ;; - clean) - ${scriptdir}/dind-cluster.sh clean - ;; - update) - copy_image_to_cluster rook/ceph:master - ;; - wordpress) - copy_image_to_cluster mysql:5.6 - copy_image_to_cluster wordpress:4.6.1-apache - ;; - *) - echo "usage:" >&2 - echo " $0 up" >&2 - echo " $0 down" >&2 - echo " $0 clean" >&2 - echo " $0 update" >&2 - echo " $0 wordpress" >&2 -esac diff --git a/tests/scripts/kubeadm-install.sh b/tests/scripts/kubeadm-install.sh index 629b480ef..aa9b73eca 100755 --- a/tests/scripts/kubeadm-install.sh +++ b/tests/scripts/kubeadm-install.sh @@ -1,7 +1,6 @@ #!/bin/bash +e KUBE_VERSION=${1:-"v1.15.12"} -ARCH=$(dpkg --print-architecture) null_str= KUBE_INSTALL_VERSION="${KUBE_VERSION/v/$null_str}"-00 @@ -17,7 +16,7 @@ wait_for_dpkg_unlock() { retryInterval=10 until [ ${retry} -ge ${maxRetries} ] do - if [[ `sudo lsof /var/lib/dpkg/lock|wc -l` -le 0 ]]; then + if [[ $(sudo lsof /var/lib/dpkg/lock|wc -l) -le 0 ]]; then break fi ((++retry)) @@ -25,7 +24,7 @@ wait_for_dpkg_unlock() { sleep ${retryInterval} done - if [ ${retry} -ge ${maxRetries} ]; then + if [ "${retry}" -ge ${maxRetries} ]; then echo "Failed after ${maxRetries} attempts! - cannot install kubeadm" exit 1 fi @@ -39,9 +38,7 @@ wait_for_dpkg_unlock sudo apt-get install -y apt-transport-https sudo curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - -sudo cat </etc/apt/sources.list.d/kubernetes.list -deb http://apt.kubernetes.io/ kubernetes-xenial main -EOF +echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list sudo apt-get update wait_for_dpkg_unlock diff --git a/tests/scripts/kubeadm.sh b/tests/scripts/kubeadm.sh index 4203f150a..6e6d4693e 100755 --- a/tests/scripts/kubeadm.sh +++ b/tests/scripts/kubeadm.sh @@ -3,7 +3,7 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" tarname=image.tar -tarfile="${WORK_DIR}/tests/${tarname}" +export tarfile="${WORK_DIR}/tests/${tarname}" export KUBE_VERSION=${KUBE_VERSION:-"v1.15.12"} @@ -38,10 +38,10 @@ usage(){ #install k8s master node install_master(){ - sudo kubeadm init $skippreflightcheck --kubernetes-version ${KUBE_VERSION} + sudo kubeadm init $skippreflightcheck --kubernetes-version "${KUBE_VERSION}" - sudo cp /etc/kubernetes/admin.conf $HOME/ - sudo chown $(id -u):$(id -g) $HOME/admin.conf + sudo cp /etc/kubernetes/admin.conf "$HOME"/ + sudo chown "$(id -u)":"$(id -g)" "$HOME"/admin.conf export KUBECONFIG=$HOME/admin.conf kubectl taint nodes --all node-role.kubernetes.io/master- @@ -50,7 +50,7 @@ install_master(){ echo "wait for K8s master node to be Ready" INC=0 while [[ $INC -lt 20 ]]; do - kube_ready=$(kubectl get node -o jsonpath='{.items['$count'].status.conditions[?(@.reason == "KubeletReady")].status}') + kube_ready=$(kubectl get node -o jsonpath='{.items['"$count"'].status.conditions[?(@.reason == "KubeletReady")].status}') if [ "${kube_ready}" == "True" ]; then break fi @@ -71,7 +71,7 @@ install_master(){ install_node(){ echo "inside install node function" echo "kubeadm join ${1} ${2} ${3} ${4} ${5} $skippreflightcheck" - sudo kubeadm join ${1} ${2} ${3} ${4} ${5} $skippreflightcheck || true + sudo kubeadm join "${1}" "${2}" "${3}" "${4}" "${5}" $skippreflightcheck || true } #wait for all nodes in the cluster to be ready status @@ -79,8 +79,8 @@ wait_for_ready(){ #expect 3 node cluster by default local numberOfNode=${1:-3} local count=0 - sudo cp /etc/kubernetes/admin.conf $HOME/ - sudo chown $(id -u):$(id -g) $HOME/admin.conf + sudo cp /etc/kubernetes/admin.conf "$HOME"/ + sudo chown "$(id -u)":"$(id -g)" "$HOME"/admin.conf export KUBECONFIG=$HOME/admin.conf until [[ $count -eq $numberOfNode ]]; do @@ -113,8 +113,8 @@ wait_for_ready(){ kubeadm_reset() { kubectl delete -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" sudo kubeadm reset --force $skippreflightcheck - rm $HOME/admin.conf - rm -rf $HOME/.kube + rm "$HOME"/admin.conf + rm -rf "$HOME"/.kube sudo apt-get -y remove kubelet sudo apt-get -y remove kubeadm sudo swapon -a @@ -125,7 +125,7 @@ case "${1:-}" in up) sudo sh -c "${scriptdir}/kubeadm-install.sh ${KUBE_VERSION}" install_master - ${scriptdir}/makeTestImages.sh tag ${arch} || true + "${scriptdir}"/makeTestImages.sh tag ${arch} || true ;; clean) kubeadm_reset @@ -143,7 +143,7 @@ case "${1:-}" in ;; node) if [ "$#" -eq 5 ] || [ "$#" -eq 7 ]; then - install_node $3 $4 $5 $6 $7 + install_node "$3" "$4" "$5" "$6" "$7" else echo "invalid arguments for install node" usage @@ -159,7 +159,7 @@ case "${1:-}" in ;; wait) if [ "$#" -eq 2 ]; then - wait_for_ready $2 + wait_for_ready "$2" else echo "invalid number of arguments for wait" usage diff --git a/tests/scripts/localPathPV.sh b/tests/scripts/localPathPV.sh index d370eb8a9..8af20dad0 100755 --- a/tests/scripts/localPathPV.sh +++ b/tests/scripts/localPathPV.sh @@ -37,7 +37,7 @@ sudo test ! -b "${test_scratch_device}" && echo "invalid scratch device, not a b function prepare_node() { sudo rm -rf /var/lib/rook/rook-integration-test sudo mkdir -p /var/lib/rook/rook-integration-test/mon1 /var/lib/rook/rook-integration-test/mon2 /var/lib/rook/rook-integration-test/mon3 - node_name=$(kubectl get nodes -o jsonpath={.items[*].metadata.name}) + node_name=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') kubectl label nodes "${node_name}" rook.io/has-disk=true kubectl delete pv -l type=local } @@ -124,14 +124,14 @@ eof function create_osd_pvc() { local osd_count=$1 local storage=6Gi - + for osd in $(seq 1 "$osd_count"); do path=${test_scratch_device}${osd} if [ "$osd_count" -eq 1 ]; then path=$test_scratch_device storage=10Gi fi - + cat <&2 @@ -36,7 +36,7 @@ case "${1:-}" in load) case "${2:-}" in arm|arm64|amd64) - echo "Loading archived images to docker: $(ls | grep tar)" + echo "Loading archived images to docker: $(ls ./*tar*)" docker load -i "ceph-$2.tar" docker load -i "cassandra-$2.tar" diff --git a/tests/scripts/minikube.sh b/tests/scripts/minikube.sh index cb4723585..361c61843 100755 --- a/tests/scripts/minikube.sh +++ b/tests/scripts/minikube.sh @@ -1,7 +1,7 @@ #!/bin/bash -e scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# shellcheck disable=SC1090 +# source=build/common.sh source "${scriptdir}/../../build/common.sh" function wait_for_ssh() { @@ -22,7 +22,7 @@ function copy_image_to_cluster() { local final_image=$2 local docker_env_tag="${DOCKERCMD}-env" ${DOCKERCMD} save "${build_image}" | \ - (eval "$(minikube ${docker_env_tag} --shell bash)" && \ + (eval "$(minikube "${docker_env_tag}" --shell bash)" && \ ${DOCKERCMD} load && \ ${DOCKERCMD} tag "${build_image}" "${final_image}") } diff --git a/tests/scripts/minishift.sh b/tests/scripts/minishift.sh index ff3a4cb44..9db28c61b 100755 --- a/tests/scripts/minishift.sh +++ b/tests/scripts/minishift.sh @@ -1,15 +1,16 @@ #!/bin/bash -e scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# source=build/common.sh source "${scriptdir}/../../build/common.sh" function wait_for_ssh() { local tries=100 - while (( ${tries} > 0 )) ; do - if `minishift ssh echo connected &> /dev/null` ; then + while (( tries > 0 )) ; do + if minishift ssh echo connected &> /dev/null ; then return 0 fi - tries=$(( ${tries} - 1 )) + (( tries-- )) sleep 0.1 done echo ERROR: ssh did not come up >&2 @@ -19,7 +20,7 @@ function wait_for_ssh() { function copy_image_to_cluster() { local build_image=$1 local final_image=$2 - docker save ${build_image} | (eval $(minishift docker-env --shell bash) && docker load && docker tag ${build_image} ${final_image}) + docker save "${build_image}" | (eval "$(minishift docker-env --shell bash)" && docker load && docker tag "${build_image}" "${final_image}") } # current kubectl context == minishift, returns boolean @@ -37,12 +38,12 @@ MEMORY=${MEMORY:-"3000"} case "${1:-}" in up) echo "starting minishift" - minishift start --memory=${MEMORY} --vm-driver=virtualbox --iso-url centos + minishift start --memory="${MEMORY}" --vm-driver=virtualbox --iso-url centos wait_for_ssh # create a link so the default dataDirHostPath will work for this environment #minishift ssh "sudo mkdir /mnt/sda1/var/lib/rook;sudo ln -s /mnt/sda1/var/lib/rook /var/lib/rook" - copy_image_to_cluster ${BUILD_REGISTRY}/ceph-amd64 rook/ceph:master + copy_image_to_cluster "${BUILD_REGISTRY}"/ceph-amd64 rook/ceph:master ;; down) minishift delete -f @@ -53,7 +54,7 @@ case "${1:-}" in ;; update) echo "updating the rook images" - copy_image_to_cluster ${BUILD_REGISTRY}/ceph-amd64 rook/ceph:master + copy_image_to_cluster "${BUILD_REGISTRY}"/ceph-amd64 rook/ceph:master ;; restart) if check_context; then @@ -73,8 +74,8 @@ case "${1:-}" in ;; helm) echo " copying rook image for helm" - helm_tag="`cat _output/version`" - copy_image_to_cluster ${BUILD_REGISTRY}/rook-amd64 rook/rook:${helm_tag} + helm_tag="cat _output/version" + copy_image_to_cluster "${BUILD_REGISTRY}"/rook-amd64 rook/rook:"${helm_tag}" ;; clean) minishift delete -f diff --git a/tests/scripts/webhook-patch-ca-bundle.sh b/tests/scripts/webhook-patch-ca-bundle.sh index a41388ec1..4618410e4 100755 --- a/tests/scripts/webhook-patch-ca-bundle.sh +++ b/tests/scripts/webhook-patch-ca-bundle.sh @@ -5,7 +5,8 @@ set -o errexit set -o nounset set -o pipefail -export CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') +CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') +export CA_BUNDLE if command -v envsubst >/dev/null 2>&1; then envsubst