Merge branch 'ci/refactor_ci_yaml' into 'master'

CI: Refactor CI rules into separate yaml file: rules.yml See merge request espressif/esp-idf!10634
2025-08-01 19:54:32 +02:00 · 2020-10-28 11:09:01 +08:00
parent b6e763ac86 75ab9223f9
commit b1d98e519d
16 changed files with 920 additions and 825 deletions
--- a/.editorconfig
+++ b/.editorconfig
@@ -37,3 +37,7 @@ max_line_length = 119
 indent_style = space
 indent_size = 4
 max_line_length = 120
+
+[{*.sh,*.yml}]
+indent_style = space
+indent_size = 2
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,13 +49,14 @@ variables:
  IDF_PATH: "$CI_PROJECT_DIR"
  BATCH_BUILD: "1"
  V: "0"
-  APPLY_BOT_FILTER_SCRIPT: "$CI_PROJECT_DIR/tools/ci/apply_bot_filter.py"
  CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"

  # Docker images
  BOT_DOCKER_IMAGE_TAG: ":latest"
+
  # target test config file, used by assign test job
  CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
+
  # target test repo parameters
  TEST_ENV_CONFIG_REPO: "${GITLAB_SSH_SERVER}/qa/ci-test-runner-configs.git"
  CI_AUTO_TEST_SCRIPT_REPO_URL: "${GITLAB_SSH_SERVER}/qa/auto_test_script.git"
@@ -64,11 +65,6 @@ variables:
  # Versioned esp-idf-doc env image to use for all document building jobs
  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v7"

-
-# before each job, we need to check if this job is filtered by bot stage/job filter
-.apply_bot_filter: &apply_bot_filter
-  python $APPLY_BOT_FILTER_SCRIPT || exit 0
-
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
  export IDF_MIRROR_PREFIX_MAP=
@@ -77,92 +73,41 @@ variables:
  tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
  fi

-.fetch_submodules: &fetch_submodules |
-  python $SUBMODULE_FETCH_TOOL -s $SUBMODULES_TO_FETCH
-
-.add_ssh_keys: &add_ssh_keys |
-  mkdir -p ~/.ssh
-  chmod 700 ~/.ssh
-  echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
-  base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-  chmod 600 ~/.ssh/id_rsa
-  echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
-
 before_script:
+  - source tools/ci/utils.sh
  - source tools/ci/setup_python.sh
-  # apply bot filter in before script
-  - *apply_bot_filter
-  # add gitlab ssh key
-  - *add_ssh_keys
-  # Set some options and environment for CI
+  - apply_bot_filter
+  - add_gitlab_ssh_keys
  - source tools/ci/configure_ci_environment.sh
  - *setup_tools_unless_target_test
-  - *fetch_submodules
-
-# used for component-based unit test apps
-.before_script_for_component_ut:
-  variables:
-    COMPONENT_UT_EXCLUDE_LIST_FP: ${CI_PROJECT_DIR}/tools/ci/component_ut_excludes.txt
-  before_script:
-    - source tools/ci/setup_python.sh
-    - *apply_bot_filter
-    - *add_ssh_keys
-    - source tools/ci/configure_ci_environment.sh
-    - *setup_tools_unless_target_test
-    - *fetch_submodules
-    - export COMPONENT_UT_DIRS=`find components/ -name test_apps -type d`
-    - export COMPONENT_UT_EXCLUDES=`[ -r $COMPONENT_UT_EXCLUDE_LIST_FP ] && cat $COMPONENT_UT_EXCLUDE_LIST_FP | xargs`
+  - fetch_submodules

 # used for check scripts which we want to run unconditionally
 .before_script_lesser_nofilter:
-  variables:
-    GIT_SUBMODULE_STRATEGY: none
  before_script:
    - echo "Not setting up GitLab key, not fetching submodules, not applying bot filter"
+    - source tools/ci/utils.sh
    - source tools/ci/setup_python.sh
    - source tools/ci/configure_ci_environment.sh

 # used for everything else where we want to do no prep, except for bot filter
 .before_script_lesser:
-  variables:
-    GIT_SUBMODULE_STRATEGY: none
  before_script:
    - echo "Not setting up GitLab key, not fetching submodules"
+    - source tools/ci/utils.sh
    - source tools/ci/setup_python.sh
-    # apply bot filter in before script
-    - *apply_bot_filter
+    - apply_bot_filter
    - source tools/ci/configure_ci_environment.sh

-.check_job_template:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - host_test
-  dependencies: []
-  extends: .before_script_lesser_nofilter
-
-.check_job_template_with_filter:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - host_test
-  dependencies: []
-  extends: .before_script_lesser
-
-.python_lint_template:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - host_test
-  dependencies: []
-
-.macos_build_template:
-  stage: build
-  tags:
-    - macos_shell
-  dependencies: []
+.before_script_slim:
  before_script:
-    - *apply_bot_filter
+    - echo "Only load utils.sh inside"
+    - source tools/ci/utils.sh
+
+.before_script_macos:
+  before_script:
+    - source tools/ci/utils.sh
+    - apply_bot_filter
    - $IDF_PATH/tools/idf_tools.py install-python-env
    # On macOS, these tools need to be installed
    - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
@@ -173,51 +118,10 @@ before_script:
    - source tools/ci/configure_ci_environment.sh
    # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script.
    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-    - *fetch_submodules
-
-.build_template_app_template:
-  stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - build
-  variables:
-    BATCH_BUILD: "1"
-    LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
-    BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
-    SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
-    BUILD_DIR: "@t/@w"
-    BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt"
-    BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
-    BUILD_COMMAND_ARGS: ""
-  artifacts:
-    when: always
-    paths:
-      - log_template_app/*
-      - size_info.txt
-      - build_template_app/**/size.json
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-  script:
-    # Set the variable for 'esp-idf-template' testing
-    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
-    - ./tools/ci/retry_failed.sh git clone ${ESP_IDF_TEMPLATE_GIT}
-    # Try to use the same branch name for esp-idf-template that we're
-    # using on esp-idf. If it doesn't exist then just stick to the default branch
-    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
-    - export PATH="$IDF_PATH/tools:$PATH"
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-    # Only do the default cmake build for each target, remaining part are done in the build_template_app job
-    - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
-    # Check if there are any stray printf/ets_printf references in WiFi libs
-    - cd components/esp_wifi/lib
-    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
-    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
+    - fetch_submodules

 include:
+  - '/tools/ci/config/rules.yml'
  - '/tools/ci/config/pre_check.yml'
  - '/tools/ci/config/build.yml'
  - '/tools/ci/config/assign-test.yml'
--- a/tools/ci/config/README.md
+++ b/tools/ci/config/README.md
@@ -0,0 +1,135 @@
+# Rules for `rules.yml`
+
+- [Rules for `rules.yml`](#rules-for-rulesyml)
+  - [How to Develop With `rules.yml`?](#how-to-develop-with-rulesyml)
+    - [How to Add a New `Job`?](#how-to-add-a-new-job)
+    - [How to Add a New `Rules` Template?](#how-to-add-a-new-rules-template)
+    - [How to Add a New `if` Anchor?](#how-to-add-a-new-if-anchor)
+  - [Naming Rules](#naming-rules)
+    - [Common Naming Rules](#common-naming-rules)
+    - [`if` Anchors Naming Rules](#if-anchors-naming-rules)
+      - [Common Phrases/Abbreviations](#common-phrasesabbreviations)
+    - [`rules` Template Naming Rules](#rules-template-naming-rules)
+  - [Reusable Shell Script `tools/ci/utils.sh`](#reusable-shell-script-toolsciutilssh)
+    - [Functions](#functions)
+      - [CI Job Related](#ci-job-related)
+      - [Shell Script Related](#shell-script-related)
+
+## How to Develop With `rules.yml`?
+
+### How to Add a New `Job`?
+
+check if there's a suitable `.rules:<rules-you-need>` template
+
+1. if there is, put this in the job `extends`. All done, now you can close this window. (`extends` could be array or string)
+2. if there isn't
+   1. check [How to Add a New `Rules` Template?](#how-to-add-a-new-rules-template), create a suitable one
+   2. follow step 1
+
+### How to Add a New `Rules` Template?
+
+check if there's a suitable `.if-<if-anchor-you-need>` anchor
+
+ 1. if there is, create a rule following [`rules` Template Naming Rules](#rules-template-naming-rules).For detail information, please refer to [GitLab Documentation `rules-if`](https://docs.gitlab.com/ee/ci/yaml/README.html#rulesif). Here's an example.
+
+```yaml
+.rules:dev:
+rules:
+  - <<: *if-trigger
+  - <<: *if-dev-push
+```
+
+2. if there isn't
+
+   1. check [How to Add a New `if` Anchor?](#how-to-add-a-new-if-anchor), create a suitable one
+   2. follow step 1
+
+### How to Add a New `if` Anchor?
+
+Create an `if` anchor following [`if` Anchors Naming Rules](#if-anchors-naming-rules). For detail information about how to write the condition clause, please refer to [GitLab Documentation `only/except (advanced)](https://docs.gitlab.com/ee/ci/yaml/README.html#onlyexcept-advanced). Here's an example.
+
+```yaml
+.if-schedule: &if-schedule:
+  if: '$CI_PIPELINE_SOURCE == "schedule"'
+```
+
+## Naming Rules
+
+### Common Naming Rules
+
+if a phrase has multi words, use `_` to concat them.
+
+> e.g. `regular_test`
+
+if a name have multi phrase, use `-` to concat them.
+
+> e.g. `regular_test-example_test`
+
+### `if` Anchors Naming Rules
+
+- if it's a label: `.if-label-<label_name>`
+- if it's a ref: `.if-ref-<ref_name>`
+- if it's a branch: `.if-branch-<branch_name>`
+- if it's a tag: `.if-tag-<tag_name>`
+- if it's a operating system: `.if-os-mac`
+- if it's multi-type combination: `.if-ref-<release_name>-branch-<branch_name>`
+
+#### Common Phrases/Abbreviations
+
+- `no_label`
+
+   `$BOT_TRIGGER_WITH_LABEL == null`
+
+- `protected`
+
+  `($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)`
+
+-  `target_test`
+
+   `example_test` or `custom_test` or `unit_test-all_targets`
+
+### `rules` Template Naming Rules
+
+- if it's os related: `.rules:os:<os_name>`
+- if it's tag related: `.rules:tag:<tag_1>-<tag_2>`
+- if it's label related: `.rules:labels:<label_1>-<label_2>`
+
+    By default, all `.rules:labels` should include both `if-label-regular_test` and `if-protected-no-label` implicitly, unless they have special postfixes:
+
+    - slim: `regular_test` not included
+    - only: only have the phrases listed
+
+- if it's target test related: `.rules:tests:<test_type_1>-<test_type_2>`
+
+    By default, all `.rules:tests` should include `if-protected-no_label` implicitly, unless they have special postfixes (same as above)
+
+- if it needs to build at first, then do some target test: `.rules:build_tests:<test_type_1>-<test_type_2>`
+
+    By default, all `.rules:build_tests` should include `if-protected-no-label`, `if-label-build`, `if-label-regular-test` implictly, unless they have special postfixes (same as above)
+
+- if a job supports all targets, use `-all_targets` as postfix
+
+## Reusable Shell Script `tools/ci/utils.sh`
+
+It is used to put all the reusable shell script as small functions. If you want to set `before_script: []` for you job, now you can set `extends: .before_script_slim` instead. it will only run `source tools/ci/utils.sh`
+
+If you're developing CI shell scripts, you can use these functions without source. They're already included in all `before_script`
+
+To run these commands in shell script locally, place `source tools/ci/utils.sh` at the very beginning.
+
+### Functions
+
+#### CI Job Related
+- `apply_bot_filter`
+- `add_gitlab_ssh_keys`
+- `add_github_ssh_keys`
+- `add_doc_server_ssh_keys`
+- `fetch_submodules`
+- `get_all_submodules`
+
+#### Shell Script Related
+- `error`: log in red color
+- `warning`: log in orange color
+- `info`: log in green color
+- `run_cmd`: run the command with duration seconds info
+- `retry_failed`: run the command with duration seconds info, retry when failed
--- a/tools/ci/config/assign-test.yml
+++ b/tools/ci/config/assign-test.yml
@@ -1,21 +1,23 @@
 assign_test:
-  extends: .before_script_for_component_ut
+  extends:
+    - .rules:tests:target_test-integration_test-weekend_test
  tags:
    - assign_test
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
  stage: assign_test
  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
  # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
-  dependencies:
+  dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here.
    - build_ssc_esp32
    - build_esp_idf_tests_cmake_esp32
    - build_esp_idf_tests_cmake_esp32s2
  variables:
    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    EXAMPLE_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
-    TEST_APP_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
-    COMPONENT_UT_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
-    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
+    EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples"
+    CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps"
+    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
+    # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh`
+    COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut"
    INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
    INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
    ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
@@ -32,42 +34,27 @@ assign_test:
      - build_component_ut/artifact_index.json
      - tools/unit-test-app/builds/artifact_index.json
    expire_in: 1 week
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2
-      - $BOT_LABEL_INTEGRATION_TEST
-      - $BOT_LABEL_EXAMPLE_TEST
-      - $BOT_LABEL_CUSTOM_TEST
-      - $BOT_LABEL_WEEKEND_TEST
  script:
-    # assign example tests
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $IDF_PATH/examples -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_CONFIG_OUTPUT_PATH
-    # assign test apps
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $IDF_PATH/tools/test_apps -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APP_CONFIG_OUTPUT_PATH
-    # assign component ut
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_CONFIG_OUTPUT_PATH
-    # assign unit test cases
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_CASE_FILE -c $CI_TARGET_TEST_CONFIG_FILE -o $IDF_PATH/components/idf_test/unit_test/CIConfigs
+    - set_component_ut_vars
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
    # clone test script to assign tests
    # can not retry if downing git lfs files failed, so using empty_branch first.
-    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    # assign integration test cases
    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH

 update_test_cases:
+  extends: .rules:ref:master-schedule
  stage: assign_test
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env
  tags:
    - deploy_test
-  only:
-    refs:
-      - master
-      - schedules
-  dependencies:
+  needs:
    - build_esp_idf_tests_cmake_esp32
    - build_esp_idf_tests_cmake_esp32s2
  artifacts:
@@ -77,18 +64,18 @@ update_test_cases:
    expire_in: 1 week
  variables:
    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    UNIT_TEST_CASE_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
+    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
    PYTHON_VER: 3.7.7
  script:
    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
-    - ./tools/ci/retry_failed.sh git clone $TEST_MANAGEMENT_REPO
+    - retry_failed git clone $TEST_MANAGEMENT_REPO
    - python $CHECKOUT_REF_SCRIPT test-management test-management
    - cd test-management
    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
    # update unit test cases
-    - export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_CASE_DIR -maxdepth 1 -name "*.yml" | xargs)
+    - export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_DIR -maxdepth 1 -name "*.yml" | xargs)
    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA
    # update example test cases
    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA
--- a/tools/ci/config/build.yml
+++ b/tools/ci/config/build.yml
@@ -4,33 +4,78 @@
  tags:
    - build
  variables:
-    BATCH_BUILD: "1"
-    V: "0"
    SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"

+.build_template_app_template:
+  extends:
+    - .build_template
+    - .rules:labels:build
+  variables:
+    LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
+    BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
+    BUILD_DIR: "@t/@w"
+    BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt"
+    BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
+    BUILD_COMMAND_ARGS: ""
+  artifacts:
+    when: always
+    paths:
+      - log_template_app/*
+      - size_info.txt
+      - build_template_app/**/size.json
+  script:
+    # Set the variable for 'esp-idf-template' testing
+    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
+    - retry_failed git clone ${ESP_IDF_TEMPLATE_GIT}
+    # Try to use the same branch name for esp-idf-template that we're
+    # using on esp-idf. If it doesn't exist then just stick to the default branch
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
+    - export PATH="$IDF_PATH/tools:$PATH"
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+    # Only do the default cmake build for each target, remaining part are done in the build_template_app job
+    - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
+    # Check if there are any stray printf/ets_printf references in WiFi libs
+    - cd components/esp_wifi/lib
+    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
+    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
+
+# build-related-pre-check-jobs ------------------------------------------------
+# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
+fast_template_app:
+  extends: .build_template_app_template
+  stage: pre_check
+  variables:
+    BUILD_COMMAND_ARGS: "-p"
+
+check_docs_gh_links:
+  extends: .build_docs_template
+  stage: pre_check
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+  script:
+    - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
+#------------------------------------------------------------------------------
+
 .build_ssc_template:
-  extends: .build_template
-  parallel: 3
+  extends:
+    - .build_template
+    - .rules:build_tests:integration_test
  artifacts:
    paths:
      - SSC/ssc_bin
    expire_in: 1 week
-  variables:
-    TARGET_NAME: "ESP32"
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_INTEGRATION_TEST
-      - $BOT_LABEL_REGULAR_TEST
  script:
-    - ./tools/ci/retry_failed.sh git clone $SSC_REPOSITORY
+    - retry_failed git clone $SSC_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT SSC SSC
    - cd SSC
    - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME

 build_ssc_esp32:
  extends: .build_ssc_template
+  parallel: 3
  variables:
    TARGET_NAME: "ESP32"

@@ -41,7 +86,9 @@ build_ssc_esp32s2:
    TARGET_NAME: "ESP32S2"

 .build_esp_idf_tests_cmake:
-  extends: .build_template
+  extends:
+    - .build_template
+    - .rules:build_tests:unit_test
  artifacts:
    paths:
      - tools/unit-test-app/output/${IDF_TARGET}
@@ -52,13 +99,6 @@ build_ssc_esp32s2:
      - $SIZE_INFO_LOCATION
    when: always
    expire_in: 4 days
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2
-      - $BOT_LABEL_REGULAR_TEST
  variables:
    LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake"
    BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
@@ -89,17 +129,9 @@ build_esp_idf_tests_cmake_esp32s3:
    IDF_TARGET: esp32s3

 .build_examples_template:
-  extends: .build_template
-  artifacts:
-    when: always
-    expire_in: 4 days
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_EXAMPLE_TEST
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_WEEKEND_TEST
+  extends:
+    - .build_template
+    - .rules:build_tests:example_test-weekend_test
  variables:
    TEST_PREFIX: examples
    TEST_RELATIVE_DIR: examples
@@ -116,7 +148,8 @@ build_esp_idf_tests_cmake_esp32s3:
    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh

 build_examples_make:
-  extends: .build_examples_template
+  extends:
+    - .build_examples_template
  # This is a workaround for a rarely encountered issue with building examples in CI.
  # Probably related to building of Kconfig in 'make clean' stage
  retry: 1
@@ -126,23 +159,16 @@ build_examples_make:
      - $LOG_PATH
      - build_${TEST_PREFIX}/*/*/*/build/size.json
      - $SIZE_INFO_LOCATION
+    when: always
+    expire_in: 4 days
  variables:
    BUILD_SYSTEM: make
    IDF_TARGET: esp32  # currently we only support esp32
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-      - pipelines
-      - web

 # same as above, but for CMake
-.build_examples_cmake: &build_examples_cmake
+.build_examples_cmake:
  extends: .build_examples_template
-  dependencies:
+  needs:
    - scan_tests
  artifacts:
    paths:
@@ -174,18 +200,13 @@ build_examples_cmake_esp32s2:
    IDF_TARGET: esp32s2

 .build_test_apps:
-  extends: .build_examples_cmake
+  extends:
+    - .build_examples_cmake
+    - .rules:build_tests:custom_test-weekend_test
  variables:
    TEST_PREFIX: test_apps
    TEST_RELATIVE_DIR: tools/test_apps
    TEST_TYPE: custom_test
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_CUSTOM_TEST
-      - $BOT_LABEL_WEEKEND_TEST
  script:
    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh

@@ -208,17 +229,12 @@ build_test_apps_esp32s3:
    IDF_TARGET: esp32s3

 .build_component_ut:
-  extends: .build_test_apps
+  extends:
+    - .build_test_apps
+    - .rules:build_tests:unit_test
  variables:
    TEST_PREFIX: component_ut
    TEST_RELATIVE_DIR: component_ut
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2

 build_component_ut_esp32:
  extends: .build_component_ut
@@ -230,128 +246,84 @@ build_component_ut_esp32s2:
  variables:
    IDF_TARGET: esp32s2

-# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
-
-.build_docs_template: &build_docs_template
+.build_docs_template:
  stage: build
  image: $ESP_IDF_DOC_ENV_IMAGE
  tags:
    - build_docs
+  script:
+    - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs $DOC_BUILDERS -l $DOCLANG -t $DOCTGT build
+  parallel:
+    matrix:
+      - DOCLANG: [ "en", "zh_CN" ]
+        DOCTGT: [ "esp32", "esp32s2" ]
+
+build_docs_html:
+  extends:
+    - .build_docs_template
+    - .rules:labels:build_docs
  artifacts:
    when: always
    paths:
      - docs/_build/*/*/*.txt
      - docs/_build/*/*/html/*
    expire_in: 4 days
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_BUILD_DOCS
-      - $BOT_LABEL_REGULAR_TEST
-  dependencies: []
-  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs html -l $DOCLANG -t $DOCTGT build
-
-build_docs_html:
-  extends: .build_docs_template
-  parallel:
-    matrix:
-      - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2"]
+  variables:
+    DOC_BUILDERS: "html"

 build_docs_pdf:
-  stage: build
-  image: $ESP_IDF_DOC_ENV_IMAGE
-  tags:
-    - build_docs
+  extends:
+    - .build_docs_template
+    - .rules:labels:build_docs-slim
  artifacts:
    when: always
    paths:
      - docs/_build/*/*/latex/*
    expire_in: 4 days
-  rules:
-    - if: '$CI_COMMIT_REF_NAME == "master"'
-    - if: '$CI_COMMIT_REF_NAME =~ "/^release\/v/"'
-    - if: '$CI_COMMIT_REF_NAME =~ "/^v\d+\.\d+(\.\d+)?($|-)/"'
-    - if: $BOT_LABEL_BUILD_DOCS
-  dependencies: []
-  parallel:
-    matrix:
-      - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2"]
+  variables:
+    DOC_BUILDERS: "latex"
+
+.test_build_system_template:
+  extends:
+    - .build_template
+    - .rules:build_tests:weekend_test
  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs latex -l $DOCLANG -t $DOCTGT build
+    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
+    - rm -rf test_build_system
+    - mkdir test_build_system
+    - cd test_build_system
+    - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT}

 test_build_system:
-  extends: .build_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_WEEKEND_TEST
-  script:
-    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
-    - rm -rf test_build_system
-    - mkdir test_build_system
-    - cd test_build_system
-    - ${IDF_PATH}/tools/ci/test_build_system.sh
+  extends: .test_build_system_template
+  variables:
+    SHELL_TEST_SCRIPT: test_build_system.sh

 test_build_system_cmake:
-  extends: .build_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_WEEKEND_TEST
-  script:
-    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
-    - rm -rf test_build_system
-    - mkdir test_build_system
-    - cd test_build_system
-    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh
+  extends: .test_build_system_template
+  variables:
+    SHELL_TEST_SCRIPT: test_build_system_cmake.sh

 test_build_system_cmake_macos:
-  extends: .macos_build_template
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
-      - triggers
-      - pipelines
-      - web
-    variables:
-      - $CI_PIPELINE_SOURCE != "push" && $BOT_LABEL_MACOS_TEST != null
-      - $CI_PIPELINE_SOURCE == "push"
-  script:
-    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
-    - rm -rf test_build_system
-    - mkdir test_build_system
-    - cd test_build_system
-    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh
+  extends:
+    - .test_build_system_template
+    - .before_script_macos
+    - .rules:os:mac_os
+  tags:
+    - macos_shell

 build_docker:
+  extends:
+    - .before_script_slim
+    - .rules:protected-schedule
  stage: build
  image: espressif/docker-builder:1
  tags:
    - build_docker_amd64_brno
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
  variables:
    DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
-  before_script: []
  script:
    - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL
    - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi
@@ -364,67 +336,50 @@ build_docker:
    # Therefore, build a copy of the example located inside the container.
    - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build

-build_idf_exe:
+.test-on-windows:
+  extends:
+    - .before_script_slim
+    - .rules:protected-schedule
  stage: build
  image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
  tags:
    - build
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
-  before_script: []
+  script:
+    - cd $TEST_DIR
+    - mkdir build
+    - cd build
+    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
+    - cmake --build .
+
+build_idf_exe:
+  extends: .test-on-windows
  artifacts:
    paths:
      - tools/windows/idf_exe/build/idf-exe-v*.zip
    expire_in: 4 days
-  script:
-    - cd tools/windows/idf_exe/
-    - mkdir build
-    - cd build
-    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
-    - cmake --build .
+  variables:
+    TEST_DIR: tools/windows/idf_exe

 build_cmdlinerunner:
-  stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
-  tags:
-    - build
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
-  before_script: []
+  extends: .test-on-windows
  artifacts:
    paths:
      - tools/windows/tool_setup/cmdlinerunner/build/cmdlinerunner.dll
    expire_in: 4 days
-  script:
-    - cd tools/windows/tool_setup/cmdlinerunner
-    - mkdir build
-    - cd build
-    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
-    - cmake --build .
+  variables:
+    TEST_DIR: tools/windows/tool_setup/cmdlinerunner

 build_installer:
+  extends:
+    - .before_script_slim
+    - .rules:protected-schedule
  # using a different stage here to be able to use artifacts from build_cmdlinerunner job
  stage: host_test
  image: $CI_DOCKER_REGISTRY/wine-innosetup:1
  tags:
    - build
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
-  dependencies:
+  needs:
    - build_cmdlinerunner
-  before_script: []
  script:
    - cd tools/windows/tool_setup/
    - ./build_installer.sh
@@ -450,14 +405,15 @@ build_template_app:
  image:
    name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2
  before_script:
+    - source tools/ci/utils.sh
    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
-    - python $SUBMODULE_FETCH_TOOL
+    - fetch_submodules
    # Exclude the submodules, all paths ends with /**
-    # get all submodules configs | get all paths | add /** as suffix | xargs | replace all <space> to <comma>
-    - export SUBMODULES=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g')
+    - export SUBMODULES=$(get_all_submodules)
    # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
    - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
-    - export EXCLUSIONS="$SUBMODULES,$CUSTOM_EXCLUDES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
+    # Exclude the report dir
+    - export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
    - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR
  variables:
    GIT_DEPTH: 0
@@ -465,13 +421,13 @@ build_template_app:
    REPORT_DIR: examples/get-started/hello_world/tidybuild/report
  tags:
    - host_test
+  needs:
+    - clang_tidy_check_regular

 code_quality_check:
-  extends: .sonar_scan_template
-  dependencies:
-    - clang_tidy_check_regular
-  only:
-    - triggers
+  extends:
+    - .sonar_scan_template
+    - .rules:trigger
  allow_failure: true
  script:
    - export CI_MERGE_REQUEST_IID=`python ${CI_PROJECT_DIR}/tools/ci/ci_get_latest_mr_iid.py ${CI_COMMIT_BRANCH} | xargs`
@@ -497,14 +453,9 @@ code_quality_check:
      -Dsonar.branch.name=$CI_COMMIT_REF_NAME

 code_quality_report:
-  extends: .sonar_scan_template
-  only:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-    - schedules
-  dependencies:
-    - clang_tidy_check_regular
+  extends:
+    - .sonar_scan_template
+    - .rules:protected-schedule
  script:
    - sonar-scanner
      -Dsonar.host.url=$SONAR_HOST_URL
--- a/tools/ci/config/deploy.yml
+++ b/tools/ci/config/deploy.yml
@@ -1,16 +1,16 @@
-.clang_tidy_deploy_template:
+.deploy_job_template:
  stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - deploy
+
+.clang_tidy_deploy_template:
+  extends: .deploy_job_template
  tags:
    - deploy
    - shiny
  script:
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+    - add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER
    - export GIT_VER=$(git describe --always)
    - cd $IDF_PATH/examples/get-started/hello_world/tidybuild
    - mv report $GIT_VER
@@ -27,89 +27,56 @@ clang_tidy_deploy:
  extends: .clang_tidy_deploy_template
  # Override default stage to happen before the post_check
  stage: test_deploy
-  dependencies:
+  needs:
    - clang_tidy_check
    - clang_tidy_check_all
  variables:
    BOT_NEEDS_TRIGGER_BY_NAME: 1

 clang_tidy_deploy_regular:
-  extends: .clang_tidy_deploy_template
-  dependencies:
+  extends:
+    - .clang_tidy_deploy_template
+    - .rules:labels:static_analysis-only
+  needs:
    - clang_tidy_check_regular
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_LABEL_STATIC_ANALYSIS
-      - $BOT_LABEL_STATIC_ANALYSIS_ALL

 push_to_github:
-  stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - deploy
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-  when: on_success
-  dependencies: []
-  extends: .before_script_lesser
+  extends:
+    - .deploy_job_template
+    - .before_script_lesser
+    - .rules:protected-no_label
  script:
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $GH_PUSH_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+    - add_github_ssh_keys
    - git remote remove github &>/dev/null || true
    - git remote add github git@github.com:espressif/esp-idf.git
    - tools/ci/push_to_github.sh

 .deploy_docs_template:
-  extends: .before_script_lesser
-  stage: deploy
+  extends:
+    - .deploy_job_template
+    - .before_script_lesser
  image: $ESP_IDF_DOC_ENV_IMAGE
  tags:
    - deploy
    - shiny
-  dependencies:
+  needs:
    - build_docs_html
    - build_docs_pdf
  variables:
    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
    PYTHONUNBUFFERED: 1
  script:
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config
+    - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
    - export GIT_VER=$(git describe --always)
-
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py

-
 # deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
 deploy_docs_preview:
-  extends: .deploy_docs_template
+  extends:
+    - .deploy_docs_template
+    - .rules:labels:build_docs-slim
  # Override default stage to happen before the post_check
  stage: test_deploy
-  only:
-    refs:
-      - triggers
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD_DOCS
  variables:
    TYPE: "preview"
    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
@@ -121,15 +88,10 @@ deploy_docs_preview:

 # deploy docs to production webserver
 deploy_docs_production:
-  extends: .deploy_docs_template
-  only:
-    refs:
-      # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
+  # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
+  extends:
+    - .deploy_docs_template
+    - .rules:protected-no_label
  variables:
    TYPE: "preview"
    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
@@ -139,15 +101,13 @@ deploy_docs_production:
    DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"

 deploy_test_result:
-  stage: deploy
+  extends:
+    - .deploy_job_template
+    - .before_script_slim
+    - .rules:ref:master-schedule-always
  image: $CI_DOCKER_REGISTRY/bot-env
  tags:
    - deploy_test
-  when: always
-  only:
-    refs:
-      - master
-      - schedules
  artifacts:
    when: always
    paths:
@@ -161,21 +121,15 @@ deploy_test_result:
    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
-  before_script:
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
  script:
+    - add_gitlab_ssh_keys
    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
    - export REV_COUNT=$(git rev-list --count HEAD)
    - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
    # artifacts of job update_test_cases creates test-management folder
    # we need to remove it so we can clone test-management folder again
    - rm -rf test-management
-    - ./tools/ci/retry_failed.sh git clone $TEST_MANAGEMENT_REPO
+    - retry_failed git clone $TEST_MANAGEMENT_REPO
    - python3 $CHECKOUT_REF_SCRIPT test-management test-management
    - cd test-management
    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
--- a/tools/ci/config/host-test.yml
+++ b/tools/ci/config/host-test.yml
@@ -1,31 +1,22 @@
 .host_test_template:
+  extends: .rules:labels:host_test
  stage: host_test
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - host_test
  dependencies: []
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_HOST_TEST
-      - $BOT_LABEL_REGULAR_TEST

 .host_fuzzer_test_template:
-  stage: host_test
+  extends:
+    - .host_test_template
+    - .rules:labels:fuzzer_test-weekend_test-only
  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
-  tags:
-    - host_test
-  dependencies: []
  artifacts:
    when: always
    paths:
      - ${FUZZER_TEST_DIR}/out/crashes
      - ${FUZZER_TEST_DIR}/fuzz_output.log
    expire_in: 1 week
-  only:
-    variables:
-      - $BOT_LABEL_FUZZER_TEST
-      - $BOT_LABEL_WEEKEND_TEST
  script:
    - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
    - cd ${FUZZER_TEST_DIR}
@@ -42,16 +33,13 @@ test_nvs_on_host:
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh make test

 test_nvs_coverage:
-  extends: .host_test_template
+  extends:
+    - .host_test_template
+    - .rules:labels:nvs_coverage-only
  artifacts:
    paths:
      - components/nvs_flash/test_nvs_host/coverage_report
    expire_in: 1 week
-  only:
-    refs:
-      - triggers
-    variables:
-      - $BOT_LABEL_NVS_COVERAGE
  script:
    - cd components/nvs_flash/test_nvs_host
    - make coverage_report
@@ -279,24 +267,16 @@ test_mkdfu:
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py

 test_docs:
-  stage: host_test
+  extends: .host_test_template
  image: $ESP_IDF_DOC_ENV_IMAGE
-  tags:
-    - host_test
-  dependencies: []
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_HOST_TEST
-      - $BOT_LABEL_REGULAR_TEST
  artifacts:
    when: on_failure
    paths:
-        - docs/test/_build/*/*/*/html/*
+      - docs/test/_build/*/*/*/html/*
    expire_in: 1 week
  script:
-  - cd ${IDF_PATH}/docs/test
-  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
-  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
-  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
+    - cd ${IDF_PATH}/docs/test
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py

--- a/tools/ci/config/post_check.yml
+++ b/tools/ci/config/post_check.yml
@@ -2,9 +2,27 @@
 .show_submodule_urls: &show_submodule_urls |
  git config --get-regexp '^submodule\..*\.url$' || true

-check_submodule_sync:
-  extends: .check_job_template
+.post_check_base_template:
  stage: post_check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - host_test
+  dependencies: []
+
+.post_check_job_template:
+  extends:
+    - .post_check_base_template
+    - .before_script_lesser_nofilter
+
+.post_check_job_template_with_filter:
+  extends:
+    - .post_check_base_template
+    - .before_script_lesser
+
+check_submodule_sync:
+  extends:
+    - .before_script_slim
+    - .post_check_job_template
  tags:
    - github_sync
  retry: 2
@@ -12,8 +30,6 @@ check_submodule_sync:
    GIT_STRATEGY: clone
    SUBMODULES_TO_FETCH: "none"
    PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
-  before_script: []
-  after_script: []
  script:
    - git submodule deinit --force .
    # setting the default remote URL to the public one, to resolve relative location URLs
@@ -26,40 +42,31 @@ check_submodule_sync:
  allow_failure: true # remove this line when esp32s3 support in esptool got merged into master

 check_ut_cmake_make:
-  extends: .check_job_template_with_filter
-  stage: post_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  extends:
+    - .post_check_job_template_with_filter
+    - .rules:dev
  tags:
    - build
-  except:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  dependencies: []
  script:
    - tools/ci/check_ut_cmake_make.sh

 check_artifacts_expire_time:
-  extends: .check_job_template
-  stage: post_check
+  extends: .post_check_job_template
  script:
    # check if we have set expire time for all artifacts
    - python tools/ci/check_artifacts_expire_time.py

 check_pipeline_triggered_by_label:
-  extends: .check_job_template
-  stage: post_check
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL
+  extends:
+    - .post_check_job_template
+    - .rules:dev
  script:
    # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
    # We want to make sure some jobs are always executed to detect regression.
    - test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }

 check_commit_msg:
-  extends: .check_job_template
-  stage: post_check
+  extends: .post_check_job_template
  script:
    - git status
    - git log -n10 --oneline
--- a/tools/ci/config/post_deploy.yml
+++ b/tools/ci/config/post_deploy.yml
@@ -1,11 +1,8 @@
-.check_doc_links_template: &check_doc_links_template
+.check_doc_links_template:
+  extends: .rules:protected
  stage: post_deploy
  image: $ESP_IDF_DOC_ENV_IMAGE
  tags: [ "build", "amd64", "internet" ]
-  only:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
  artifacts:
    when: always
    paths:
--- a/tools/ci/config/pre_check.yml
+++ b/tools/ci/config/pre_check.yml
@@ -1,61 +1,63 @@
+.pre_check_base_template:
+  stage: pre_check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - host_test
+  dependencies: []
+
+.pre_check_job_template:
+  extends:
+    - .pre_check_base_template
+    - .before_script_lesser_nofilter
+
+.pre_check_job_template_with_filter:
+  extends:
+    - .pre_check_base_template
+    - .before_script_lesser
+
 check_line_endings:
-  extends: .check_job_template
+  extends: .pre_check_job_template
  script:
    - tools/ci/check-line-endings.sh ${IDF_PATH}

 check_permissions:
-  extends: .check_job_template
+  extends: .pre_check_job_template
  script:
    - tools/ci/check-executable.sh

 check_docs_lang_sync:
-  extends: .check_job_template
-  stage: pre_check
+  extends: .pre_check_job_template
  variables:
    SUBMODULES_TO_FETCH: "none"
  script:
    - cd docs
    - ./check_lang_folder_sync.sh

-check_docs_gh_links:
-  extends: .build_docs_template
-  stage: pre_check
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
-
 check_version:
-  extends: .check_job_template
  # Don't run this for feature/bugfix branches, so that it is possible to modify
  # esp_idf_version.h in a branch before tagging the next version.
-  only:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
+  extends:
+    - .pre_check_job_template
+    - .rules:protected
  script:
    - export IDF_PATH=$PWD
    - tools/ci/check_idf_version.sh

 check_examples_cmake_make:
-  extends: .check_job_template_with_filter
-  except:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
+  extends:
+    - .pre_check_job_template_with_filter
+    - .rules:dev
  script:
    - python ${IDF_PATH}/tools/ci/check_examples_cmake_make.py

 check_rom_api_header:
-  extends: .check_job_template_with_filter
+  extends: .pre_check_job_template_with_filter
  script:
    - tools/ci/check_examples_rom_header.sh
    - tools/ci/check_rom_apis.sh

 check_python_style:
-  extends: .python_lint_template
+  extends: .pre_check_base_template
  artifacts:
    when: on_failure
    paths:
@@ -65,7 +67,7 @@ check_python_style:
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH

 check_kconfigs:
-  extends: .check_job_template_with_filter
+  extends: .pre_check_job_template_with_filter
  artifacts:
    when: on_failure
    paths:
@@ -81,51 +83,31 @@ check_kconfigs:
    - ${IDF_PATH}/tools/check_kconfigs.py

 check_deprecated_kconfig_options:
-  extends: .check_job_template_with_filter
+  extends: .pre_check_job_template_with_filter
  script:
    - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py

 check_cmake_style:
-  extends: .check_job_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
+  extends: .pre_check_job_template
  script:
    tools/cmake/run_cmake_lint.sh

 check_wifi_lib_md5:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  extends: .pre_check_base_template
  tags:
    - build
  variables:
    SUBMODULES_TO_FETCH: "components/esp_wifi/lib"
-  dependencies: []
  script:
    - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32s2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh

-# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
-fast_template_app:
-  extends: .build_template_app_template
-  stage: pre_check
-  variables:
-    BUILD_COMMAND_ARGS: "-p"
-
 check_public_headers:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  extends:
+    - .pre_check_base_template
+    - .rules:labels:build
  tags:
    - build
-  variables:
-    BATCH_BUILD: "1"
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
  script:
    - python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-

@@ -140,16 +122,8 @@ check_public_headers:

 scan_tests:
  extends:
-    - .before_script_for_component_ut
    - .scan_build_tests
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_EXAMPLE_TEST
-      - $BOT_LABEL_CUSTOM_TEST
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2
+    - .rules:build_tests:target_test-weekend_test
  artifacts:
    paths:
      - $EXAMPLE_TEST_OUTPUT_DIR
@@ -163,13 +137,14 @@ scan_tests:
    COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
    PYTHON_VER: 3
  script:
+    - set_component_ut_vars
    - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b make --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
    - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
    - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
    - python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR

 check_readme_links:
-  extends: .check_job_template
+  extends: .pre_check_job_template
  tags: [ "amd64", "deploy", "internet" ]
  allow_failure: true
  variables:
@@ -178,11 +153,8 @@ check_readme_links:
    - python ${IDF_PATH}/tools/ci/check_readme_links.py

 .clang_tidy_check_template:
-  stage: pre_check
+  extends: .pre_check_base_template
  image: ${CI_DOCKER_REGISTRY}/clang-static-analysis
-  tags:
-    - host_test
-  dependencies: []
  artifacts:
    reports:
      junit: $IDF_PATH/output.xml
@@ -191,7 +163,7 @@ check_readme_links:
      - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
    expire_in: 1 day
  script:
-    - ./tools/ci/retry_failed.sh git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
+    - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
    # Setup parameters of triggered/regular job
    - export TRIGGERED_RELATIVE=${BOT_LABEL_STATIC_ANALYSIS-} && export TRIGGERED_ABSOLUTE=${BOT_LABEL_STATIC_ANALYSIS_ALL-} && export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
    - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
@@ -212,18 +184,15 @@ clang_tidy_check_all:
    BOT_LABEL_STATIC_ANALYSIS_ALL: 1

 check_codeowners:
-  extends: .check_job_template
+  extends: .pre_check_job_template
  script:
    - tools/codeowners.py ci-check

 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'
 check_version_tag:
-  extends: .check_job_template
-  only:
-    refs:
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
+  extends:
+    - .pre_check_job_template
+    - .rules:tag:release-no_label
  script:
    - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || echo "ESP-IDF versions must be annotated tags." && exit 1
--- a/tools/ci/config/rules.yml
+++ b/tools/ci/config/rules.yml
@@ -0,0 +1,249 @@
+# if anchors
+.if-ref-master: &if-ref-master
+  if: '$CI_COMMIT_REF_NAME == "master"'
+
+.if-tag-release-no_label: &if-tag-release-no_label
+  if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/ && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.if-schedule: &if-schedule
+  if: '$CI_PIPELINE_SOURCE == "schedule"'
+
+.if-trigger: &if-trigger
+  if: '$CI_PIPELINE_SOURCE == "trigger"'
+
+.if-label-regular_test: &if-label-regular_test
+  if: '$BOT_LABEL_REGULAR_TEST'
+
+.if-label-build: &if-label-build
+  if: '$BOT_LABEL_BUILD'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS'
+
+.if-label-integration_test: &if-label-integration_test
+  if: '$BOT_LABEL_INTEGRATION_TEST'
+
+.if-label-unit_test: &if-label-unit_test
+  if: '$BOT_LABEL_UNIT_TEST'
+
+.if-label-unit_test_s2: &if-label-unit_test_s2
+  if: '$BOT_LABEL_UNIT_TEST_S2'
+
+.if-label-unit_test-all_targets: &if-label-unit_test-all_targets
+  if: '$BOT_LABEL_UNIT_TEST || $BOT_LABEL_UNIT_TEST_S2'
+
+.if-label-weekend_test: &if-label-weekend_test
+  if: '$BOT_LABEL_WEEKEND_TEST'
+
+.if-label-example_test: &if-label-example_test
+  if: '$BOT_LABEL_EXAMPLE_TEST'
+
+.if-label-custom_test: &if-label-custom_test
+  if: '$BOT_LABEL_CUSTOM_TEST'
+
+.if-label-host_test: &if-label-host_test
+  if: '$BOT_LABEL_HOST_TEST'
+
+.if-label-fuzzer_test: &if-label-fuzzer_test
+  if: '$BOT_LABEL_FUZZER_TEST'
+
+.if-label-nvs_coverage: &if-label-nvs_coverage
+  if: '$BOT_LABEL_NVS_COVERAGE'
+
+.if-label-static_analysis: &if-label-static_analysis
+  if: '$BOT_LABEL_STATIC_ANALYSIS || $BOT_LABEL_STATIC_ANALYSIS_ALL'
+
+.if-label-iperf_stress_test: &if-label-iperf_stress_test
+  if: '$BOT_LABEL_IPERF_STRESS_TEST'
+
+.if-os-mac: &if-os-mac
+  if: '$BOT_LABEL_MACOS_TEST'
+
+# Rules templates
+.rules:protected:
+  rules:
+    - <<: *if-protected
+
+.rules:protected-no_label:
+  rules:
+    - <<: *if-protected-no_label
+
+.rules:protected-schedule:
+  rules:
+    - <<: *if-protected
+    - <<: *if-schedule
+
+.rules:trigger:
+  rules:
+    - <<: *if-trigger
+
+.rules:dev:
+  rules:
+    - <<: *if-trigger
+    - <<: *if-dev-push
+
+.rules:os:mac_os:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-os-mac
+
+.rules:tag:release-no_label:
+  rules:
+    - <<: *if-tag-release-no_label
+
+.rules:ref:master-schedule:
+  rules:
+    - <<: *if-ref-master
+    - <<: *if-schedule
+
+.rules:ref:master-schedule-always:
+  rules:
+    - <<: *if-ref-master
+      when: always
+    - <<: *if-schedule
+      when: always
+
+.rules:labels:static_analysis-only:
+  rules:
+    - <<: *if-label-static_analysis
+
+.rules:labels:build:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-regular_test
+    - <<: *if-label-build
+
+.rules:labels:build_docs:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-build_docs
+
+.rules:labels:build_docs-slim:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build_docs
+
+.rules:labels:weekend_test-only:
+  rules:
+    - <<: *if-label-weekend_test
+
+.rules:labels:iperf_stress_test-only:
+  rules:
+    - <<: *if-label-iperf_stress_test
+
+.rules:labels:fuzzer_test-weekend_test-only:
+  rules:
+    - <<: *if-label-fuzzer_test
+    - <<: *if-label-weekend_test
+
+.rules:labels:nvs_coverage-only:
+  rules:
+    - <<: *if-label-nvs_coverage
+
+.rules:labels:host_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-regular_test
+    - <<: *if-label-host_test
+
+.rules:tests:example_test-schedule:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-example_test
+    - <<: *if-schedule
+
+.rules:tests:custom_test-schedule:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-custom_test
+    - <<: *if-schedule
+
+.rules:tests:unit_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-unit_test
+
+.rules:tests:unit_test_s2-only:
+  rules:
+    - <<: *if-label-unit_test_s2
+
+.rules:tests:integration_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-integration_test
+
+.rules:tests:target_test-integration_test-weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-example_test
+    - <<: *if-label-custom_test
+    - <<: *if-label-unit_test-all_targets
+    - <<: *if-label-integration_test
+    - <<: *if-label-weekend_test
+
+.rules:build_tests:integration_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-integration_test
+
+.rules:build_tests:weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-weekend_test
+
+.rules:build_tests:unit_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-unit_test-all_targets
+
+.rules:build_tests:example_test-weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-example_test
+    - <<: *if-label-weekend_test
+
+.rules:build_tests:custom_test-weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-custom_test
+    - <<: *if-label-weekend_test
+
+.rules:build_tests:target_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-example_test
+    - <<: *if-label-custom_test
+    - <<: *if-label-unit_test-all_targets
+
+.rules:build_tests:target_test-weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-example_test
+    - <<: *if-label-custom_test
+    - <<: *if-label-unit_test-all_targets
+    - <<: *if-label-weekend_test
--- a/tools/ci/config/target-test.yml
+++ b/tools/ci/config/target-test.yml
@@ -1,24 +1,13 @@
 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"
 .define_config_file_name: &define_config_file_name |
-    JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
-    JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
-    CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"
+  JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
+  JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
+  CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"

-.example_test_template:
+.target_test_job_template:
  stage: target_test
-  when: on_success
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_EXAMPLE_TEST
-  dependencies:
+  needs:
    - assign_test
  artifacts:
    when: always
@@ -26,10 +15,9 @@
      - $LOG_PATH
    expire_in: 1 week
    reports:
-        junit: $LOG_PATH/*/XUNIT_RESULT.xml
+      junit: $LOG_PATH/*/XUNIT_RESULT.xml
  variables:
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
@@ -38,133 +26,70 @@
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    - cd tools/ci/python_packages/tiny_test_fw/bin
    # run test
    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE

-.example_debug_template:
-  stage: target_test
-  when: on_success
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_EXAMPLE_TEST
-  dependencies:
-    - assign_test
-  artifacts:
-    when: always
-    paths:
-      - $LOG_PATH
-    expire_in: 1 week
-    reports:
-        junit: $LOG_PATH/*/XUNIT_RESULT.xml
+.example_test_template:
+  extends:
+    - .target_test_job_template
+    - .rules:tests:example_test-schedule
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
-    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
-  script:
-    - *define_config_file_name
-    # first test if config file exists, if not exist, exit 0
-    - test -e $CONFIG_FILE || exit 0
-    # clone test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - cd tools/ci/python_packages/tiny_test_fw/bin
-    # run test
-    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
+
+.example_debug_template:
+  extends: .example_test_template
+  variables:
+    SUBMODULES_TO_FETCH: "all"

 .test_app_template:
-  extends: .example_test_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_CUSTOM_TEST
+  extends:
+    - .target_test_job_template
+    - .rules:tests:custom_test-schedule
  variables:
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
-    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"

 .component_ut_template:
  extends:
-    - .before_script_for_component_ut
-    - .example_test_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_UNIT_TEST
+    - .target_test_job_template
+    - .rules:tests:unit_test
  variables:
+    TEST_CASE_PATH: "$COMPONENT_UT_DIRS"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
    PYTHON_VER: 3
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
+    - set_component_ut_vars
    # clone test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    - cd tools/ci/python_packages/tiny_test_fw/bin
    # run test
-    - python Runner.py $COMPONENT_UT_DIRS -c $CONFIG_FILE -e $ENV_FILE
+    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE

 .unit_test_template:
-  extends: .example_test_template
-  stage: target_test
-  dependencies:
-    - assign_test
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_UNIT_TEST
+  extends:
+    - .target_test_job_template
+    - .rules:tests:unit_test
  variables:
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
-    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
    PYTHON_VER: 3

-.test_template:
-  stage: target_test
-  when: on_success
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_INTEGRATION_TEST
-  dependencies:
+.integration_test_template:
+  extends:
+    - .target_test_job_template
+    - .rules:tests:integration_test
+  needs:
    - assign_test
    - build_ssc_esp32
-  artifacts:
-    when: always
-    reports:
-        junit: $LOG_PATH/*/XUNIT_RESULT.xml
-    paths:
-      - $LOG_PATH
-    expire_in: 1 week
  variables:
-    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
    LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
@@ -179,56 +104,48 @@
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone local test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
    # can not retry if downing git lfs files failed, so using empty_branch first.
-    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
    # run test
    - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH

 test_weekend_mqtt:
-  extends: .example_test_template
-  stage: target_test
+  extends:
+    - .example_test_template
+    - .rules:labels:weekend_test-only
  tags:
    - ESP32
    - Example_WIFI
-  only:
-    variables:
-      - $BOT_LABEL_WEEKEND_TEST
  variables:
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
    CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"

 test_weekend_network:
-  extends: .example_test_template
-  stage: target_test
+  extends:
+    - .example_test_template
+    - .rules:labels:weekend_test-only
  image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG
  tags:
    - ESP32
    - Example_WIFI
-  only:
-    variables:
-      - $BOT_LABEL_WEEKEND_TEST
  variables:
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
    CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"

 example_test_001A:
  extends: .example_test_template
  parallel: 4
  artifacts:
-      when: always
-      paths:
-        - $CI_PROJECT_DIR/examples/*/*/*.log
-      expire_in: 1 week
+    paths:
+      - $CI_PROJECT_DIR/examples/*/*/*.log
  tags:
    - ESP32
    - Example_WIFI
@@ -252,17 +169,6 @@ example_test_002:
  tags:
    - ESP32
    - Example_ShieldBox_Basic
-  script:
-    - *define_config_file_name
-    # first test if config file exists, if not exist, exit 0
-    - test -e $CONFIG_FILE || exit 0
-    # clone test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - cd tools/ci/python_packages/tiny_test_fw/bin
-    # run test
-    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
-

 .example_test_003:
  extends: .example_test_template
@@ -289,11 +195,10 @@ example_test_005:
    - Example_WIFI_BT

 example_test_006:
-  extends: .example_test_template
+  extends:
+    - .example_test_template
+    - .rules:labels:iperf_stress_test-only
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
-  only:
-    variables:
-      - $BOT_LABEL_IPERF_STRESS_TEST
  tags:
    - ESP32
    - Example_ShieldBox
@@ -322,10 +227,8 @@ example_test_009:
    - ESP32
    - test_jtag_arm
  artifacts:
-      when: always
-      paths:
-        - $CI_PROJECT_DIR/examples/*/*/*.log
-      expire_in: 1 week
+    paths:
+      - $CI_PROJECT_DIR/examples/*/*/*.log
  variables:
    SETUP_TOOLS: "1"
    PYTHON_VER: 3
@@ -342,11 +245,9 @@ example_test_011:
    - ESP32
    - Example_T2_RS485
  artifacts:
-      when: always
-      expire_in: 1 week
-      paths:
-        - $CI_PROJECT_DIR/examples/protocols/modbus/serial/*.log
-        - $LOG_PATH
+    paths:
+      - $CI_PROJECT_DIR/examples/protocols/modbus/serial/*.log
+      - $LOG_PATH
  variables:
    SETUP_TOOLS: "1"

@@ -374,11 +275,9 @@ example_test_015:
    - ESP32
    - Example_PPP
  artifacts:
-      when: always
-      expire_in: 1 week
-      paths:
-        - $CI_PROJECT_DIR/examples/*/*/*.log
-        - $LOG_PATH
+    paths:
+      - $CI_PROJECT_DIR/examples/*/*/*.log
+      - $LOG_PATH

 example_test_016:
  extends: .example_test_template
@@ -386,11 +285,9 @@ example_test_016:
    - ESP32
    - Example_Modbus_TCP
  artifacts:
-      when: always
-      expire_in: 1 week
-      paths:
-        - $CI_PROJECT_DIR/examples/*/*/*.log
-        - $LOG_PATH
+    paths:
+      - $CI_PROJECT_DIR/examples/*/*/*.log
+      - $LOG_PATH

 test_app_test_001:
  extends: .test_app_template
@@ -399,10 +296,8 @@ test_app_test_001:
    - ESP32
    - test_jtag_arm
  artifacts:
-      when: always
-      paths:
-        - $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
-      expire_in: 1 week
+    paths:
+      - $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
  variables:
    SETUP_TOOLS: "1"
    PYTHON_VER: 3
@@ -578,21 +473,9 @@ UT_034:
    - UT_T1_ESP_FLASH

 .unit_test_s2_template:
-  extends: .unit_test_template
-  dependencies:
-    - assign_test
-  only:
-    refs:
-      # Due to lack of runners, the tests are only done by manual trigger
-      # - master
-      # - /^release\/v/
-      # - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      # Due to lack of runners, S2 tests will only be triggered with label (@bot test will not trigger)
-      # - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_UNIT_TEST_S2
+  extends:
+    - .unit_test_template
+    - .rules:tests:unit_test_s2-only # due to the lack of runners, s2 tests will only be triggered by label

 UT_035:
  extends: .unit_test_s2_template
@@ -610,7 +493,7 @@ UT_036:

 # ToDo: re-enable this job when ESP32-S2 LEDC runner installed
 # UT_037:
-#   extends: .unit_test_template
+#   extends: .unit_test_s2_template
 #   tags:
 #     - ESP32S2_IDF
 #     - UT_T1_LEDC
@@ -648,7 +531,7 @@ UT_045:
    - ESP32_IDF
    - UT_SDIO
    - psram
-    
+
 UT_046:
  extends: .unit_test_template
  tags:
@@ -656,7 +539,7 @@ UT_046:
    - UT_T1_GPIO

 nvs_compatible_test:
-  extends: .test_template
+  extends: .integration_test_template
  artifacts:
    when: always
    paths:
@@ -671,12 +554,12 @@ nvs_compatible_test:
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone local test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
    # can not retry if downing git lfs files failed, so using empty_branch first.
-    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    # prepare nvs bins
    - cd auto_test_script
@@ -685,129 +568,129 @@ nvs_compatible_test:
    - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH

 IT_001:
-  extends: .test_template
+  extends: .integration_test_template
  parallel: 3
  tags:
    - ESP32_IDF
    - SSC_T1_4

 IT_002:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T1_2

 IT_003:
-  extends: .test_template
+  extends: .integration_test_template
  parallel: 14
  tags:
    - ESP32_IDF
    - SSC_T2_5

 IT_004:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T1_APC

 IT_005:
-  extends: .test_template
+  extends: .integration_test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T1_5

 IT_006:
-  extends: .test_template
+  extends: .integration_test_template
  parallel: 12
  tags:
    - ESP32_IDF
    - SSC_T1_6

 IT_007:
-  extends: .test_template
+  extends: .integration_test_template
  parallel: 3
  tags:
    - ESP32_IDF
    - SSC_T1_7

 IT_008:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T1_8

 IT_009:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T1_3

 IT_011:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T1_MESH1

 IT_012:
-  extends: .test_template
+  extends: .integration_test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T2_MESH1

 IT_013:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T3_MESH1

 IT_014:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T6_MESH1

 IT_015:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T12_MESH1

 IT_016:
-  extends: .test_template
+  extends: .integration_test_template
  allow_failure: true
  tags:
    - ESP32_IDF
    - SSC_T50_MESH1

 IT_017:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T1_MESH2

 IT_018:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T1_9

 IT_019:
-  extends: .test_template
+  extends: .integration_test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T2_2

 IT_020:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T2_3

 IT_021:
-  extends: .test_template
+  extends: .integration_test_template
  tags:
    - ESP32_IDF
    - SSC_T2_4
--- a/tools/ci/executable-list.txt
+++ b/tools/ci/executable-list.txt
@@ -58,10 +58,10 @@ tools/ci/mirror-submodule-update.sh
 tools/ci/multirun_with_pyenv.sh
 tools/ci/normalize_clangtidy_path.py
 tools/ci/push_to_github.sh
-tools/ci/retry_failed.sh
 tools/ci/test_build_system.sh
 tools/ci/test_build_system_cmake.sh
 tools/ci/test_configure_ci_environment.sh
+tools/ci/utils.sh
 tools/cmake/convert_to_cmake.py
 tools/cmake/run_cmake_lint.sh
 tools/codeowners.py
--- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py
+++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py
@@ -20,6 +20,7 @@ BUILD_ALL_LABELS = [
    'BOT_LABEL_BUILD',
    'BOT_LABEL_BUILD_ALL_APPS',
    'BOT_LABEL_REGULAR_TEST',
+    'BOT_LABEL_WEEKEND_TEST',
 ]


--- a/tools/ci/retry_failed.sh
+++ b/tools/ci/retry_failed.sh
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-#
-# Retries a command RETRY_ATTEMPTS times in case of failure
-#
-# Inspired by https://stackoverflow.com/a/8351489
-#
-
-max_attempts=${RETRY_ATTEMPTS-3}
-RETRY_TIMEWAIT=${RETRY_TIMEWAIT-1}
-attempt=1
-exitCode=0
-whole_start=$(date +%s)
-attempt_start=whole_start
-
-while true; do
-  if "$@" ; then
-    exitCode=0
-    break
-  else
-    exitCode=$?
-  fi
-
-  if (( $attempt >= $max_attempts )) ; then
-    break
-  fi
-
-  echo "Failed! ("$@") Spent time $(( $(date '+%s') - ${attempt_start} )) sec. Retrying in ${RETRY_TIMEWAIT}..." 1>&2
-  sleep $RETRY_TIMEWAIT
-  attempt=$(( attempt + 1 ))
-  RETRY_TIMEWAIT=$(( RETRY_TIMEWAIT * 2 ))
-  attempt_start=$(date +%s)
-done
-
-if [[ $exitCode != 0 ]] ; then
-  echo -n "Totally failed! ("$@")" 1>&2
-else
-  echo -n "Done ("$@")" 1>&2
-fi
-
-echo " Spent time $(( $(date '+%s') - ${whole_start} )) sec in total" 1>&2
-
-exit $exitCode
--- a/tools/ci/utils.sh
+++ b/tools/ci/utils.sh
@@ -0,0 +1,119 @@
+# Modified from https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/utils.sh
+
+# before each job, we need to check if this job is filtered by bot stage/job filter
+function apply_bot_filter() {
+  python "${IDF_PATH}"/tools/ci/apply_bot_filter.py || exit 0
+}
+
+function add_ssh_keys() {
+  local key_string="${1}"
+  mkdir -p ~/.ssh
+  chmod 700 ~/.ssh
+  echo -n "${key_string}" >~/.ssh/id_rsa_base64
+  base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
+  chmod 600 ~/.ssh/id_rsa
+}
+
+function add_gitlab_ssh_keys() {
+  add_ssh_keys "${GITLAB_KEY}"
+  echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >>~/.ssh/config
+
+  # For gitlab geo nodes
+  if [ "${LOCAL_GITLAB_SSH_SERVER:-}" ]; then
+    SRV=${LOCAL_GITLAB_SSH_SERVER##*@} # remove the chars before @, which is the account
+    SRV=${SRV%%:*}                     # remove the chars after :, which is the port
+    printf "Host %s\n\tStrictHostKeyChecking no\n" "${SRV}" >>~/.ssh/config
+  fi
+}
+
+function add_github_ssh_keys() {
+  add_ssh_keys "${GH_PUSH_KEY}"
+  echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >>~/.ssh/config
+}
+
+function add_doc_server_ssh_keys() {
+  local key_string="${1}"
+  local server_url="${2}"
+  local server_user="${3}"
+  add_ssh_keys "${key_string}"
+  echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config
+}
+
+function fetch_submodules() {
+  python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
+}
+
+function get_all_submodules() {
+  git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g'
+}
+
+function set_component_ut_vars() {
+  local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt"
+  export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d)
+  export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs)
+  echo "COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES written into export"
+}
+
+function error() {
+  printf "\033[0;31m%s\n\033[0m" "${1}" >&2
+}
+
+function info() {
+  printf "\033[0;32m%s\n\033[0m" "${1}" >&2
+}
+
+function warning() {
+  printf "\033[0;33m%s\n\033[0m" "${1}" >&2
+}
+
+function run_cmd() {
+  local start=$(date +%s)
+  eval "$@"
+  local ret=$?
+  local end=$(date +%s)
+  local duration=$((end - start))
+
+  if [[ $ret -eq 0 ]]; then
+    info "(\$ $*) succeeded in ${duration} seconds."
+    return 0
+  else
+    error "(\$ $*) failed in ${duration} seconds."
+    return $ret
+  fi
+}
+
+# Retries a command RETRY_ATTEMPTS times in case of failure
+# Inspired by https://stackoverflow.com/a/8351489
+function retry_failed() {
+  local max_attempts=${RETRY_ATTEMPTS-3}
+  local timeout=${RETRY_TIMEWAIT-1}
+  local attempt=1
+  local exitCode=0
+
+  whole_start=$(date +%s)
+  while true; do
+    if run_cmd "$@"; then
+      exitCode=0
+      break
+    else
+      exitCode=$?
+    fi
+
+    if ((attempt >= max_attempts)); then
+      break
+    fi
+
+    error "Retrying in ${timeout} seconds..."
+    sleep $timeout
+    attempt=$((attempt + 1))
+    timeout=$((timeout * 2))
+  done
+
+  local duration=$(($(date '+%s') - whole_start))
+  if [[ $exitCode != 0 ]]; then
+    error "Totally failed! Spent $duration sec in total"
+  else
+    info "Done! Spent $duration sec in total"
+  fi
+  return $exitCode
+}