From 1b06cbe3ef35d5ea3a940ca6c55285959c134577 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 28 Sep 2020 14:27:10 +0800 Subject: [PATCH] `utils.sh`: add function info, warning, error. Turn `retry_failed.sh` into a function includes `utils.sh` everywhere --- tools/ci/config/assign-test.yml | 2 +- tools/ci/config/build.yml | 16 ++++---- tools/ci/config/deploy.yml | 2 +- tools/ci/config/post_check.yml | 5 ++- tools/ci/config/pre_check.yml | 2 +- tools/ci/config/target-test.yml | 6 +-- tools/ci/executable-list.txt | 1 - tools/ci/retry_failed.sh | 45 ---------------------- tools/ci/utils.sh | 66 ++++++++++++++++++++++++++++++++- 9 files changed, 83 insertions(+), 62 deletions(-) delete mode 100755 tools/ci/retry_failed.sh diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index bdbf78c508..51556e6171 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -70,7 +70,7 @@ update_test_cases: PYTHON_VER: 3.7.7 script: - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8) - - ./tools/ci/retry_failed.sh git clone $TEST_MANAGEMENT_REPO + - retry_failed git clone $TEST_MANAGEMENT_REPO - python $CHECKOUT_REF_SCRIPT test-management test-management - cd test-management - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE} diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index bbfe201a2e..ec638eb9e0 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -26,7 +26,7 @@ script: # Set the variable for 'esp-idf-template' testing - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} - - ./tools/ci/retry_failed.sh git clone ${ESP_IDF_TEMPLATE_GIT} + - retry_failed git clone ${ESP_IDF_TEMPLATE_GIT} # Try to use the same branch name for esp-idf-template that we're # using on esp-idf. If it doesn't exist then just stick to the default branch - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template @@ -68,7 +68,7 @@ check_docs_gh_links: - SSC/ssc_bin expire_in: 1 week script: - - ./tools/ci/retry_failed.sh git clone $SSC_REPOSITORY + - retry_failed git clone $SSC_REPOSITORY - python $CHECKOUT_REF_SCRIPT SSC SSC - cd SSC - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME @@ -310,6 +310,7 @@ test_build_system_cmake_macos: build_docker: extends: + - .before_script_slim - .rules:master-release-schedule stage: build image: espressif/docker-builder:1 @@ -317,7 +318,6 @@ build_docker: - build_docker_amd64_brno variables: DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" - before_script: [] script: - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi @@ -331,12 +331,13 @@ build_docker: - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build .test-on-windows: - extends: .rules:master-release-schedule + extends: + - .before_script_slim + - .rules:master-release-schedule stage: build image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross tags: - build - before_script: [] script: - cd $TEST_DIR - mkdir build @@ -363,7 +364,9 @@ build_cmdlinerunner: TEST_DIR: tools/windows/tool_setup/cmdlinerunner build_installer: - extends: .rules:master-release-schedule + extends: + - .before_script_slim + - .rules:master-release-schedule # using a different stage here to be able to use artifacts from build_cmdlinerunner job stage: host_test image: $CI_DOCKER_REGISTRY/wine-innosetup:1 @@ -371,7 +374,6 @@ build_installer: - build needs: - build_cmdlinerunner - before_script: [] script: - cd tools/windows/tool_setup/ - ./build_installer.sh diff --git a/tools/ci/config/deploy.yml b/tools/ci/config/deploy.yml index 2f8c5d7a9b..8211e314cb 100644 --- a/tools/ci/config/deploy.yml +++ b/tools/ci/config/deploy.yml @@ -129,7 +129,7 @@ deploy_test_result: # artifacts of job update_test_cases creates test-management folder # we need to remove it so we can clone test-management folder again - rm -rf test-management - - ./tools/ci/retry_failed.sh git clone $TEST_MANAGEMENT_REPO + - retry_failed git clone $TEST_MANAGEMENT_REPO - python3 $CHECKOUT_REF_SCRIPT test-management test-management - cd test-management - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE} diff --git a/tools/ci/config/post_check.yml b/tools/ci/config/post_check.yml index 16eb8e9b58..7d4716509b 100644 --- a/tools/ci/config/post_check.yml +++ b/tools/ci/config/post_check.yml @@ -20,7 +20,9 @@ - .before_script_lesser check_submodule_sync: - extends: .post_check_job_template + extends: + - .before_script_slim + - .post_check_job_template tags: - github_sync retry: 2 @@ -28,7 +30,6 @@ check_submodule_sync: GIT_STRATEGY: clone SUBMODULES_TO_FETCH: "none" PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" - before_script: [] script: - git submodule deinit --force . # setting the default remote URL to the public one, to resolve relative location URLs diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index b4dc23e7ba..ff5f51c6ca 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -163,7 +163,7 @@ check_readme_links: - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/* expire_in: 1 day script: - - ./tools/ci/retry_failed.sh git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils + - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils # Setup parameters of triggered/regular job - export TRIGGERED_RELATIVE=${BOT_LABEL_STATIC_ANALYSIS-} && export TRIGGERED_ABSOLUTE=${BOT_LABEL_STATIC_ANALYSIS_ALL-} && export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-} - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 928d02a3f6..0eb6f30b5c 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -554,12 +554,12 @@ nvs_compatible_test: # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 # clone local test env configs - - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO + - retry_failed git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench # can not retry if downing git lfs files failed, so using empty_branch first. - - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} + - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch + - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script # prepare nvs bins - cd auto_test_script diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 5814dd0530..be0a9b49ff 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -58,7 +58,6 @@ tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh tools/ci/normalize_clangtidy_path.py tools/ci/push_to_github.sh -tools/ci/retry_failed.sh tools/ci/test_build_system.sh tools/ci/test_build_system_cmake.sh tools/ci/test_configure_ci_environment.sh diff --git a/tools/ci/retry_failed.sh b/tools/ci/retry_failed.sh deleted file mode 100755 index 2da480b471..0000000000 --- a/tools/ci/retry_failed.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# -# Retries a command RETRY_ATTEMPTS times in case of failure -# -# Inspired by https://stackoverflow.com/a/8351489 -# - -max_attempts=${RETRY_ATTEMPTS-3} -RETRY_TIMEWAIT=${RETRY_TIMEWAIT-1} -attempt=1 -exitCode=0 -whole_start=$(date +%s) -attempt_start=whole_start - -while true; do - if "$@" ; then - exitCode=0 - break - else - exitCode=$? - fi - - if (( $attempt >= $max_attempts )) ; then - break - fi - - echo "Failed! ("$@") Spent time $(( $(date '+%s') - ${attempt_start} )) sec. Retrying in ${RETRY_TIMEWAIT}..." 1>&2 - sleep $RETRY_TIMEWAIT - attempt=$(( attempt + 1 )) - RETRY_TIMEWAIT=$(( RETRY_TIMEWAIT * 2 )) - attempt_start=$(date +%s) -done - -if [[ $exitCode != 0 ]] ; then - echo -n "Totally failed! ("$@")" 1>&2 -else - echo -n "Done ("$@")" 1>&2 -fi - -echo " Spent time $(( $(date '+%s') - ${whole_start} )) sec in total" 1>&2 - -exit $exitCode diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index f7baff5195..febc39116a 100755 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -44,7 +44,7 @@ function fetch_submodules() { } function get_all_submodules() { - echo "$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g')" + git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g' } function set_component_ut_vars() { @@ -53,3 +53,67 @@ function set_component_ut_vars() { export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs) echo "COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES written into export" } + +function error() { + printf "\033[0;31m%s\n\033[0m" "${1}" >&2 +} + +function info() { + printf "\033[0;32m%s\n\033[0m" "${1}" >&2 +} + +function warning() { + printf "\033[0;33m%s\n\033[0m" "${1}" >&2 +} + +function run_cmd() { + local start=$(date +%s) + eval "$@" + local ret=$? + local end=$(date +%s) + local duration=$((end - start)) + + if [[ $ret -eq 0 ]]; then + info "(\$ $*) succeeded in ${duration} seconds." + return 0 + else + error "(\$ $*) failed in ${duration} seconds." + return $ret + fi +} + +# Retries a command RETRY_ATTEMPTS times in case of failure +# Inspired by https://stackoverflow.com/a/8351489 +function retry_failed() { + local max_attempts=${RETRY_ATTEMPTS-3} + local timeout=${RETRY_TIMEWAIT-1} + local attempt=1 + local exitCode=0 + + whole_start=$(date +%s) + while true; do + if run_cmd "$@"; then + exitCode=0 + break + else + exitCode=$? + fi + + if ((attempt >= max_attempts)); then + break + fi + + error "Retrying in ${timeout} seconds..." + sleep $timeout + attempt=$((attempt + 1)) + timeout=$((timeout * 2)) + done + + local duration=$(($(date '+%s') - whole_start)) + if [[ $exitCode != 0 ]]; then + error "Totally failed! Spent $duration sec in total" + else + info "Done! Spent $duration sec in total" + fi + return $exitCode +}