utils.sh: add function info, warning, error. Turn retry_failed.sh into a function

includes `utils.sh` everywhere

tools/ci/config/assign-test.yml

@@ -70,7 +70,7 @@ update_test_cases:
     PYTHON_VER: 3.7.7
   script:
     - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
-    - ./tools/ci/retry_failed.sh git clone $TEST_MANAGEMENT_REPO
+    - retry_failed git clone $TEST_MANAGEMENT_REPO
     - python $CHECKOUT_REF_SCRIPT test-management test-management
     - cd test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}

tools/ci/config/build.yml

@@ -26,7 +26,7 @@
   script:
     # Set the variable for 'esp-idf-template' testing
     - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
-    - ./tools/ci/retry_failed.sh git clone ${ESP_IDF_TEMPLATE_GIT}
+    - retry_failed git clone ${ESP_IDF_TEMPLATE_GIT}
     # Try to use the same branch name for esp-idf-template that we're
     # using on esp-idf. If it doesn't exist then just stick to the default branch
     - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
@@ -68,7 +68,7 @@ check_docs_gh_links:
       - SSC/ssc_bin
     expire_in: 1 week
   script:
-    - ./tools/ci/retry_failed.sh git clone $SSC_REPOSITORY
+    - retry_failed git clone $SSC_REPOSITORY
     - python $CHECKOUT_REF_SCRIPT SSC SSC
     - cd SSC
     - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME
@@ -310,6 +310,7 @@ test_build_system_cmake_macos:
 
 build_docker:
   extends:
+    - .before_script_slim
     - .rules:master-release-schedule
   stage: build
   image: espressif/docker-builder:1
@@ -317,7 +318,6 @@ build_docker:
     - build_docker_amd64_brno
   variables:
     DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
-  before_script: []
   script:
     - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL
     - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi
@@ -331,12 +331,13 @@ build_docker:
     - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
 
 .test-on-windows:
-  extends: .rules:master-release-schedule
+  extends:
+    - .before_script_slim
+    - .rules:master-release-schedule
   stage: build
   image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
   tags:
     - build
-  before_script: []
   script:
     - cd $TEST_DIR
     - mkdir build
@@ -363,7 +364,9 @@ build_cmdlinerunner:
     TEST_DIR: tools/windows/tool_setup/cmdlinerunner
 
 build_installer:
-  extends: .rules:master-release-schedule
+  extends:
+    - .before_script_slim
+    - .rules:master-release-schedule
   # using a different stage here to be able to use artifacts from build_cmdlinerunner job
   stage: host_test
   image: $CI_DOCKER_REGISTRY/wine-innosetup:1
@@ -371,7 +374,6 @@ build_installer:
     - build
   needs:
     - build_cmdlinerunner
-  before_script: []
   script:
     - cd tools/windows/tool_setup/
     - ./build_installer.sh

tools/ci/config/deploy.yml

@@ -129,7 +129,7 @@ deploy_test_result:
     # artifacts of job update_test_cases creates test-management folder
     # we need to remove it so we can clone test-management folder again
     - rm -rf test-management
-    - ./tools/ci/retry_failed.sh git clone $TEST_MANAGEMENT_REPO
+    - retry_failed git clone $TEST_MANAGEMENT_REPO
     - python3 $CHECKOUT_REF_SCRIPT test-management test-management
     - cd test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}

tools/ci/config/post_check.yml

@@ -20,7 +20,9 @@
     - .before_script_lesser
 
 check_submodule_sync:
-  extends: .post_check_job_template
+  extends:
+    - .before_script_slim
+    - .post_check_job_template
   tags:
     - github_sync
   retry: 2
@@ -28,7 +30,6 @@ check_submodule_sync:
     GIT_STRATEGY: clone
     SUBMODULES_TO_FETCH: "none"
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
-  before_script: []
   script:
     - git submodule deinit --force .
     # setting the default remote URL to the public one, to resolve relative location URLs

tools/ci/config/pre_check.yml

@@ -163,7 +163,7 @@ check_readme_links:
       - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
     expire_in: 1 day
   script:
-    - ./tools/ci/retry_failed.sh git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
+    - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
     # Setup parameters of triggered/regular job
     - export TRIGGERED_RELATIVE=${BOT_LABEL_STATIC_ANALYSIS-} && export TRIGGERED_ABSOLUTE=${BOT_LABEL_STATIC_ANALYSIS_ALL-} && export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
     - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml

tools/ci/config/target-test.yml

@@ -554,12 +554,12 @@ nvs_compatible_test:
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
     # clone local test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # clone test bench
     # can not retry if downing git lfs files failed, so using empty_branch first.
-    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
     # prepare nvs bins
     - cd auto_test_script

tools/ci/executable-list.txt

@@ -58,7 +58,6 @@ tools/ci/mirror-submodule-update.sh
 tools/ci/multirun_with_pyenv.sh
 tools/ci/normalize_clangtidy_path.py
 tools/ci/push_to_github.sh
-tools/ci/retry_failed.sh
 tools/ci/test_build_system.sh
 tools/ci/test_build_system_cmake.sh
 tools/ci/test_configure_ci_environment.sh

tools/ci/retry_failed.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-#
-# Retries a command RETRY_ATTEMPTS times in case of failure
-#
-# Inspired by https://stackoverflow.com/a/8351489
-#
-
-max_attempts=${RETRY_ATTEMPTS-3}
-RETRY_TIMEWAIT=${RETRY_TIMEWAIT-1}
-attempt=1
-exitCode=0
-whole_start=$(date +%s)
-attempt_start=whole_start
-
-while true; do
-  if "$@" ; then
-    exitCode=0
-    break
-  else
-    exitCode=$?
-  fi
-
-  if (( $attempt >= $max_attempts )) ; then
-    break
-  fi
-
-  echo "Failed! ("$@") Spent time $(( $(date '+%s') - ${attempt_start} )) sec. Retrying in ${RETRY_TIMEWAIT}..." 1>&2
-  sleep $RETRY_TIMEWAIT
-  attempt=$(( attempt + 1 ))
-  RETRY_TIMEWAIT=$(( RETRY_TIMEWAIT * 2 ))
-  attempt_start=$(date +%s)
-done
-
-if [[ $exitCode != 0 ]] ; then
-  echo -n "Totally failed! ("$@")" 1>&2
-else
-  echo -n "Done ("$@")" 1>&2
-fi
-
-echo " Spent time $(( $(date '+%s') - ${whole_start} )) sec in total" 1>&2
-
-exit $exitCode

tools/ci/utils.sh

@@ -44,7 +44,7 @@ function fetch_submodules() {
 }
 
 function get_all_submodules() {
-  echo "$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g')"
+  git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g'
 }
 
 function set_component_ut_vars() {
@@ -53,3 +53,67 @@ function set_component_ut_vars() {
   export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs)
   echo "COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES written into export"
 }
+
+function error() {
+  printf "\033[0;31m%s\n\033[0m" "${1}" >&2
+}
+
+function info() {
+  printf "\033[0;32m%s\n\033[0m" "${1}" >&2
+}
+
+function warning() {
+  printf "\033[0;33m%s\n\033[0m" "${1}" >&2
+}
+
+function run_cmd() {
+  local start=$(date +%s)
+  eval "$@"
+  local ret=$?
+  local end=$(date +%s)
+  local duration=$((end - start))
+
+  if [[ $ret -eq 0 ]]; then
+    info "(\$ $*) succeeded in ${duration} seconds."
+    return 0
+  else
+    error "(\$ $*) failed in ${duration} seconds."
+    return $ret
+  fi
+}
+
+# Retries a command RETRY_ATTEMPTS times in case of failure
+# Inspired by https://stackoverflow.com/a/8351489
+function retry_failed() {
+  local max_attempts=${RETRY_ATTEMPTS-3}
+  local timeout=${RETRY_TIMEWAIT-1}
+  local attempt=1
+  local exitCode=0
+
+  whole_start=$(date +%s)
+  while true; do
+    if run_cmd "$@"; then
+      exitCode=0
+      break
+    else
+      exitCode=$?
+    fi
+
+    if ((attempt >= max_attempts)); then
+      break
+    fi
+
+    error "Retrying in ${timeout} seconds..."
+    sleep $timeout
+    attempt=$((attempt + 1))
+    timeout=$((timeout * 2))
+  done
+
+  local duration=$(($(date '+%s') - whole_start))
+  if [[ $exitCode != 0 ]]; then
+    error "Totally failed! Spent $duration sec in total"
+  else
+    info "Done! Spent $duration sec in total"
+  fi
+  return $exitCode
+}