Merge branch 'feature/optimize_ci_config_file' into 'master'

CI: use features from latest Gitlab in CI config file

See merge request idf/esp-idf!4896

components/idf_test/unit_test/TestCaseScript/IDFUnitTest/__init__.py

@@ -1 +0,0 @@
-__all__ = ["UnitTest"]

components/idf_test/unit_test/TestEnvAll.yml

@@ -1,292 +0,0 @@
-test environment:
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: AT_T1_1,
-  test environment detail: 'PC has 2 wired NIC connected to AP.
-
-    PC has 1 WiFi NIC.
-
-    1 AT target connect with PC by UART (AT and LOG port).', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: AT_T1_2,
-  test environment detail: 'PC has 1 WiFi NIC.
-
-    1 AT target connect with PC by UART (AT and LOG port).', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: AT_T1_3,
-  test environment detail: 'Able to access WAN after connect to AP.
-
-    1 AT target connect with PC by UART (AT and LOG port).', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: AT_T1_ADC,
-  test environment detail: 'PC has 1 wired NIC connected to AP.
-
-    Analog input connect to AT1 TOUT.
-
-    Multimeter connect to input, able to measure input voltage.
-
-    1 AT target connect with PC by UART (AT and LOG port).', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: AT_T1_APC1,
-  test environment detail: "PC has 1 wired NIC connected to AP.\nPC has 1 wired NIC\
-    \ connected to APC (static IP within the same subnet with APC). \nAPC control\
-    \ AP power supply. \nPC has 1 WiFi NIC. \n1 AT target connect with PC by UART\
-    \ (AT and LOG port).", test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: AT_T1_APC2,
-  test environment detail: "Able to access WAN after connect to AP.\nPC has 1 wired\
-    \ NIC connected to APC (static IP within the same subnet with APC). \nAPC control\
-    \ AP power supply.\nPC has 1 WiFi NIC.\n1 AT target connect with PC by UART (AT\
-    \ and LOG port).", test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: AT_T1_HighSpeedUART,
-  test environment detail: 'PC has 2 wired NIC connected to AP.
-
-    PC has 1 WiFi NIC.
-
-    1 AT target connect with PC by high speed UART (AT and LOG port).', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: AT_T1_SmartConfigIOT,
-  test environment detail: '1 AT target connect with PC by UART (AT and LOG port).
-
-    PC has 1 wired NIC connect to Common AP.
-
-    Several AP are placed near AT target.
-
-    Several smart phone installed test APK are placed near AT target.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 2.0, script path: EnvBase.py, tag: AT_T2_1,
-  test environment detail: 'PC has 1 wired NIC connected to AP.
-
-    PC has 1 WiFi NIC.
-
-    2 AT target connect with PC by UART (AT and LOG port).', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 2.0, script path: EnvBase.py, tag: AT_T2_JAP,
-  test environment detail: "Several AP are placed near AT target.\nPC has 1 wired\
-    \ NIC connected to APC (static IP within the same subnet with APC).\nAPC control\
-    \ power supply for all APs. \n2 AT target connect with PC by UART (AT and LOG\
-    \ port).", test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 2.0, script path: EnvBase.py, tag: AT_T2_Sleep,
-  test environment detail: 'AP support DTIM placed with AT target.
-
-    2 AT target connect with PC by UART (AT and LOG port).
-
-    Multimeter connect with PC via GPIB.
-
-    Series multimeter between GND and VCC of AT1.
-
-    AT1''s light sleep wakeup pin and wakeup indication connect with AT2''s GPIO.',
-  test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 2.0, script path: EnvBase.py, tag: AT_T2_SmartConfig,
-  test environment detail: '2 AT target connect with PC by UART (AT and LOG port).
-
-    PC has 1 WiFi NIC.
-
-    One HT20 AP and One HT40 AP are placed near target.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, UART ports: 'SSC1
-
-    SSC2', additional param list: '', basic param list: '', script path: EnvBase.py,
-  tag: IR_T2_1, test environment detail: '[TBD] 本测试为非自动测试, 红外能够做到数据收发吻合即可通过', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 2.0, script path: EnvBase.py, tag: NVS_T1_1,
-  test environment detail: '1 NVS target connect with PC by UART.
-
-    1 SSC target connect with PC by UART.
-
-    SSC2 GPIO connect to NVS1 power control pin.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, UART ports: SSC_1, additional param list: '',
-  basic param list: '', script path: EnvBase.py, tag: PWM_T1_1, test environment detail: "[TBD]\
-    \ 1. PWM OS SDK 以及 Non-OS SDK的测试建议分开进行, 放在不同的文件夹, 防止文件命名混淆\n2. 分析CSV文件的Python脚本只能分析单个channel\
-    \ \n3. 如果Init脚本打印\"Network Error\" 检查TCP Server是不是正常发送data", test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_1,
-  test environment detail: 'PC has 2 wired NIC connected to AP.
-
-    PC has 1 WiFi NIC.
-
-    1 SSC target connect with PC by UART.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_2,
-  test environment detail: 'Able to access WAN after connect to AP.
-
-    1 SSC target connect with PC by UART.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_8089,
-  test environment detail: 'PC has 1 wired NIC connected to AP.
-
-    1 8089 tablet able to run iperf test placed near SSC1.
-
-    1 SSC target connect with PC by UART.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_ADC,
-  test environment detail: 'PC has 1 wired NIC connected to AP.
-
-    Analog input connect to SSC1 TOUT.
-
-    Multimeter connect to input, able to measure input voltage.
-
-    1 SSC target connect with PC by UART.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_APC,
-  test environment detail: "PC has 1 wired NIC connected to AP.\nPC has 1 wired NIC\
-    \ connected to APC (static IP within the same subnet with APC). \nAPC control\
-    \ AP power supply. \nPC has 1 WiFi NIC. \n1 SSC target connect with PC by UART.",
-  test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_Enterprise,
-  test environment detail: "AP use WPA2-Etherprise is placed near SSC1. \n1 SSC target\
-    \ connect with PC by UART.", test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_IOT1,
-  test environment detail: 'PC has 1 WiFi NIC.
-
-    1 SSC target connect with PC by UART.
-
-    AP todo IOT test are placed near SSC1.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 2.0, script path: EnvBase.py, tag: SSC_T1_InitData,
-  test environment detail: '2 SSC target connect with PC by UART.
-
-    SSC1 use 40M crystal oscillator.
-
-    SSC2 use normal 26M crystal oscillator.
-
-    SSC2 GPIO connect to SSC1 power control pin.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_ShieldBox,
-  test environment detail: 'refer to figure.
-
-    All APs and APC should be set to the same IP subnet.
-
-    PC wired NIC should set static IP address within the same subnet with AP.
-
-    Must use onboard wired NIC.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_Sleep1,
-  test environment detail: 'AP support DTIM placed with AT target.
-
-    SSC target connect with Raspberry Pi by UART.
-
-    Multimeter connect with Raspberry Pi via GPIB.
-
-    Series multimeter between GND and VCC of SSC1.
-
-    SSC1''s light sleep wakeup pin and wakeup indication connect with Raspberry Pi''s
-    GPIO.
-
-    SSC1''s XPD connect with RSTB.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_Sleep2,
-  test environment detail: 'AP support DTIM placed with AT target.
-
-    SSC target connect with Raspberry Pi by UART.
-
-    Multimeter connect with Raspberry Pi via GPIB.
-
-    Series multimeter between GND and VCC of SSC1.
-
-    SSC1''s RSTB pin connect with Raspberry Pi''s GPIO.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_TempBox,
-  test environment detail: '1 SSC target connect with PC by UART.
-
-    Put SSC target to temperature box.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, UART ports: SSC_1, additional param list: '',
-  basic param list: '', script path: EnvBase.py, tag: SSC_T1_Timer, test environment detail: '[TBD]
-    通过串口工具调节Timer, 将GPIO_13端口连接到逻辑分析仪', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_VDD33,
-  test environment detail: '1 SSC target connect with PC by UART.
-
-    Multimeter connect to VDD33, able to measure voltage.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T1_WEP,
-  test environment detail: '1 SSC target connect with PC by UART.
-
-    One WEP share key AP placed near SSC1.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 2.0, script path: EnvBase.py, tag: SSC_T2_1,
-  test environment detail: 'PC has 1 wired NIC connected to AP.
-
-    PC has 1 WiFi NIC.
-
-    2 SSC target connect with PC by UART.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 2.0, UART ports: 'SSC1
-
-    SSC2', additional param list: '', basic param list: '', script path: EnvBase.py,
-  tag: SSC_T2_GPIO1, test environment detail: '[TBD] 2个ESP_8266通过UART连到PC, ESP_8266的
-    GPIO_6相连', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 2.0, UART ports: 'SSC1
-
-    SSC2', additional param list: '', basic param list: '', script path: EnvBase.py,
-  tag: SSC_T2_GPIO2, test environment detail: '[TBD] 1. 2个ESP_8266通过UART连到PC, ESP_8266的
-    GPIO_15通过面包板相连
-
-    2. 可借助面包板, 将GPIO_15, 以及中断函数被打开的8266板的GPIO_2 相连', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 2.0, UART ports: 'SSC1
-
-    SSC2', additional param list: '', basic param list: '', script path: EnvBase.py,
-  tag: SSC_T2_GPIO3, test environment detail: '[TBD] 2个ESP_8266通过UART连到PC, ESP_8266之间需要测试的Target_GPIO相连',
-  test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 2.0, script path: EnvBase.py, tag: SSC_T2_JAP,
-  test environment detail: 'PC has 1 wired NIC connected to APC.
-
-    APC control the power supply of multiple APs.
-
-    2 SSC target connect with PC by UART.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 2.0, script path: EnvBase.py, tag: SSC_T2_PhyMode,
-  test environment detail: '2 SSC target connect with PC by UART.
-
-    PC has one WiFi NIC support capture wlan packet using libpcap.
-
-    Set 4 AP with phy mode 11b, 11g, 11n HT20, 11n HT40.
-
-    Put 4 APs near SSC targets.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 2.0, script path: EnvBase.py, tag: SSC_T2_ShieldBox,
-  test environment detail: '2 SSC target connect with PC by UART.
-
-    Put them to Shield box.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 2.0, script path: EnvBase.py, tag: SSC_T2_SmartConfig,
-  test environment detail: '2 SSC target connect with PC by UART.
-
-    PC has 1 WiFi NIC.
-
-    One HT20 AP and One HT40 AP are placed near target.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 3.0, script path: EnvBase.py, tag: SSC_T3_PhyMode,
-  test environment detail: '3 SSC target connect with PC by UART.
-
-    PC has one WiFi NIC support capture wlan packet using libpcap.
-
-    Set 4 AP with (HT20, channel1), (HT20, channel2), (HT40, channel1), (HT40, channel2).
-
-    Put 4 APs near SSC targets.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 5.0, script path: EnvBase.py, tag: SSC_T5_1,
-  test environment detail: 5 SSC target connect with PC by UART., test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 5.0, script path: EnvBase.py, tag: SSC_T5_IOT1,
-  test environment detail: '5 SSC targets connect with PC by UART.
-
-    some Android smart phone are placed near SSC targets.', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: SSC_T6_1,
-  test environment detail: 'PC has 1 wired NIC connected to AP.
-
-    PC has 1 WiFi NIC.
-
-    6 SSC target connect with PC by UART.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: TempSensor_T1_1,
-  test environment detail: 'Tempeture sensor target connect with PC by UART.
-
-    AP support DTIM placed with AT target.
-
-    Multimeter connect with PC via GPIB.
-
-    Series multimeter between GND and VCC of TempSensor1.
-
-    PC has 1 wired NIC connected to switch.
-
-    APC, AP also connect with swtich.
-
-    All devices connected with switch use the same IP subnet.
-
-    APC control AP power supply.', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, UART ports: SSC_1, additional param list: '',
-  basic param list: '', script path: EnvBase.py, tag: UART_T1_1, test environment detail: '[TBD]
-    将ESP_8266通过UART连到PC', test script: EnvBase}
-- {PC OS: '', Special: Y, Target Count: 1.0, UART ports: 'SSC1
-
-    SSC2', additional param list: '', basic param list: '', script path: EnvBase.py,
-  tag: UART_T1_2, test environment detail: '[TBD] ESP_8266通过UART_0通过USB, UART_1 TXD
-    通过 TTLcable 连到PC', test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: UT_T1_1,
-  test environment detail: Environment for running ESP32 unit tests, test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: UT_T1_SDMODE,
-  test environment detail: Environment for running sd card sd mode unit tests, test script: EnvBase}
-- {PC OS: '', Special: N, Target Count: 1.0, script path: EnvBase.py, tag: UT_T1_SPIMODE,
-  test environment detail: Environment for running sd card spi mode unit tests, test script: EnvBase}
-- {PC OS: linux, Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: WebServer_T1_1,
-  test environment detail: 'Web Server target connect with PC by UART.
-
-    PC has 1 wired NIC connected to switch.
-
-    APC, AP also connect with swtich.
-
-    All devices connected with switch use same IP subnet.
-
-    APC control AP power supply.', test script: EnvBase}
-- {PC OS: linux, Special: Y, Target Count: 1.0, script path: EnvBase.py, tag: WebServer_T1_2,
-  test environment detail: 'Web Server target connect with PC by UART.
-
-    4 PC with WiFi NIC placed near WebServer1.', test script: EnvBase}

tools/ci/build_examples.sh

@@ -48,9 +48,6 @@ die() {
 
 echo "build_examples running in ${PWD}"
 
-# only 0 or 1 arguments
-[ $# -le 1 ] || die "Have to run as $(basename $0) [<JOB_NAME>]"
-
 export BATCH_BUILD=1
 export V=0 # only build verbose if there's an error
 
@@ -65,26 +62,14 @@ SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
 
 EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name Makefile | grep -v "/build_system/cmake/" | sort )
 
-if [ $# -eq 0 ]
+if [ -z {CI_NODE_TOTAL} ]
 then
     START_NUM=0
     END_NUM=999
 else
-    JOB_NAME=$1
+    JOB_NUM=${CI_NODE_INDEX}
-
-    # parse text prefix at the beginning of string 'some_your_text_NUM'
-    # (will be 'some_your_text' without last '_')
-    JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' )
-    [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
-
-    # parse number 'NUM' at the end of string 'some_your_text_NUM'
-    # NOTE: Getting rid of the leading zero to get the decimal
-    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' )
-    [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
-
     # count number of the jobs
-    NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${IDF_PATH}/.gitlab-ci.yml" )
+    NUM_OF_JOBS=${CI_NODE_TOTAL}
-    [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad"
 
     # count number of examples
     NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l )
@@ -96,10 +81,10 @@ else
     [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
 
     # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
-    START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
+    START_NUM=$(( (${JOB_NUM} - 1) * ${NUM_OF_EX_PER_JOB} ))
     [ -z ${START_NUM} ] && die "START_NUM is bad"
 
-    END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} ))
+    END_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
     [ -z ${END_NUM} ] && die "END_NUM is bad"
 fi
 
@@ -168,6 +153,8 @@ build_example () {
 
 EXAMPLE_NUM=0
 
+echo "Current job will build example ${START_NUM} - ${END_NUM}"
+
 for EXAMPLE_PATH in ${EXAMPLE_PATHS}
 do
     if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]

tools/ci/build_examples_cmake.sh

@@ -68,26 +68,14 @@ SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
 
 EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name CMakeLists.txt | grep -v "/components/" | grep -v "/common_components/" | grep -v "/main/" | grep -v "/idf_as_lib/stubs/" | sort )
 
-if [ $# -eq 0 ]
+if [ -z {CI_NODE_TOTAL} ]
 then
     START_NUM=0
     END_NUM=999
 else
-    JOB_NAME=$1
+    JOB_NUM=${CI_NODE_INDEX}
-
-    # parse text prefix at the beginning of string 'some_your_text_NUM'
-    # (will be 'some_your_text' without last '_')
-    JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' )
-    [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
-
-    # parse number 'NUM' at the end of string 'some_your_text_NUM'
-    # NOTE: Getting rid of the leading zero to get the decimal
-    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' )
-    [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
-
     # count number of the jobs
-    NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${IDF_PATH}/.gitlab-ci.yml" )
+    NUM_OF_JOBS=${CI_NODE_TOTAL}
-    [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad"
 
     # count number of examples
     NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l )
@@ -99,10 +87,10 @@ else
     [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
 
     # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
-    START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
+    START_NUM=$(( (${JOB_NUM} - 1) * ${NUM_OF_EX_PER_JOB} ))
     [ -z ${START_NUM} ] && die "START_NUM is bad"
 
-    END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} ))
+    END_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
     [ -z ${END_NUM} ] && die "END_NUM is bad"
 fi
 
@@ -156,6 +144,8 @@ build_example () {
 
 EXAMPLE_NUM=0
 
+echo "Current job will build example ${START_NUM} - ${END_NUM}"
+
 for EXAMPLE_PATH in ${EXAMPLE_PATHS}
 do
     if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]

tools/ci/checkout_project_ref.py

@@ -7,32 +7,67 @@ import os
 import json
 import argparse
 import subprocess
+import re
 
 
-def checkout_branch(proj_name, customized_revision, default_ref_name):
+IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)")
+
+
+def target_branch_candidates(proj_name):
+    """
+    :return: a list of target branch candidates, from highest priority to lowest priority.
+    """
+    candidates = [
+        # branch name (or tag name) of current IDF
+        os.getenv("CI_COMMIT_REF_NAME"),
+        # CI_MERGE_REQUEST_TARGET_BRANCH_NAME
+        os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"),
+    ]
+    # revision defined in bot message
+    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
+    if customized_project_revisions:
+        customized_project_revisions = json.loads(customized_project_revisions)
     try:
-        ref_to_use = customized_revision[proj_name.lower()]
+        ref_to_use = customized_project_revisions[proj_name.lower()]
+        # highest priority, insert to head of list
+        candidates.insert(0, ref_to_use)
     except (KeyError, TypeError):
-        ref_to_use = default_ref_name
+        pass
+    # branch name read from IDF
+    git_describe = subprocess.check_output(["git", "describe", "--tags", "HEAD"])
+    match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe)
+    if match:
+        major_revision = match.group(1)
+        minor_revision = match.group(2)
+        # release branch
+        candidates.append("release/v{}.{}".format(major_revision, minor_revision))
+        # branch to match all major branches, like v3.x or v3
+        candidates.append("release/v{}.x".format(major_revision))
+        candidates.append("release/v{}".format(major_revision))
 
-    try:
+    return [c for c in candidates if c]  # filter out null value
-        subprocess.check_call(["git", "checkout", ref_to_use])
-        print("CI using ref {} for project {}".format(ref_to_use, proj_name))
-    except subprocess.CalledProcessError:
-        print("using default branch")
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("project",
                         help="the name of project")
+    parser.add_argument("project_relative_path",
+                        help="relative path of project to IDF repository directory")
 
     args = parser.parse_args()
-    project_name = args.project
 
-    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
+    candidate_branches = target_branch_candidates(args.project)
-    if customized_project_revisions:
-        customized_project_revisions = json.loads(customized_project_revisions)
-    ci_ref_name = os.getenv("CI_COMMIT_REF_NAME")
 
-    checkout_branch(project_name, customized_project_revisions, ci_ref_name)
+    # change to project dir for checkout
+    os.chdir(args.project_relative_path)
+
+    for candidate in candidate_branches:
+        try:
+            subprocess.check_call(["git", "checkout", candidate])
+            print("CI using ref {} for project {}".format(candidate, args.project))
+            break
+        except subprocess.CalledProcessError:
+            pass
+    else:
+        print("using default branch")

tools/ci/config/assign-test.yml

@@ -0,0 +1,74 @@
+
+assign_test:
+  tags:
+    - assign_test
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  stage: assign_test
+  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
+  # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
+  dependencies:
+    - build_ssc
+    - build_esp_idf_tests_make
+    - build_esp_idf_tests_cmake
+  variables:
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
+  artifacts:
+    paths:
+      - components/idf_test/*/CIConfigs
+      - components/idf_test/*/TC.sqlite
+      - $EXAMPLE_CONFIG_OUTPUT_PATH
+    expire_in: 1 week
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_UNIT_TEST
+      - $BOT_LABEL_INTEGRATION_TEST
+      - $BOT_LABEL_EXAMPLE_TEST
+  script:
+    # assign example tests
+    - python $TEST_FW_PATH/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
+    # assign unit test cases
+    - python $TEST_FW_PATH/CIAssignUnitTest.py $IDF_PATH/components/idf_test/unit_test/TestCaseAll.yml $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
+    # clone test script to assign tests
+    - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - cd auto_test_script
+    # assgin integration test cases
+    - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin
+
+update_test_cases:
+  stage: assign_test
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env
+  tags:
+    - deploy_test
+  only:
+    refs:
+      - master
+      - schedules
+  dependencies:
+    - build_esp_idf_tests_make
+    - build_esp_idf_tests_cmake
+  artifacts:
+    when: always
+    paths:
+      - ${CI_PROJECT_DIR}/test-management/*.log
+    expire_in: 1 week
+  variables:
+    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
+    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
+    PYTHON_VER: 3
+  script:
+    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
+    - git clone $TEST_MANAGEMENT_REPO
+    - python $CHECKOUT_REF_SCRIPT test-management test-management
+    - cd test-management
+    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
+    # update unit test cases
+    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILE -r $GIT_SHA
+    # update example test cases
+    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA
+    # organize test cases
+    - python OrganizeTestCases.py $JIRA_TEST_MANAGEMENT_PROJECT

tools/ci/config/build.yml

@@ -0,0 +1,247 @@
+
+.build_template:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  variables:
+    BATCH_BUILD: "1"
+    V: "0"
+
+.build_esp_idf_unit_test_template:
+  extends: .build_template
+  artifacts:
+    paths:
+      - tools/unit-test-app/output
+      - components/idf_test/unit_test/TestCaseAll.yml
+    expire_in: 3 days
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_UNIT_TEST
+      - $BOT_LABEL_REGULAR_TEST
+
+build_template_app:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  variables:
+    BATCH_BUILD: "1"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    # Set the variable for 'esp-idf-template' testing
+    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
+    - git clone ${ESP_IDF_TEMPLATE_GIT}
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
+    - cd esp-idf-template
+    # Try to use the same branch name for esp-idf-template that we're
+    # using on esp-idf. If it doesn't exist then just stick to the default
+    # branch
+    - make defconfig
+    # Test debug build (default)
+    - make all V=1
+    # Now test release build
+    - make clean
+    - sed -i.bak -e's/CONFIG_OPTIMIZATION_LEVEL_DEBUG\=y/CONFIG_OPTIMIZATION_LEVEL_RELEASE=y/' sdkconfig
+    - make all V=1
+    # Check if there are any stray printf/ets_printf references in WiFi libs
+    - cd ../components/esp_wifi/lib_esp32
+    - test $(xtensa-esp32-elf-nm *.a | grep -w printf | wc -l) -eq 0
+    - test $(xtensa-esp32-elf-nm *.a | grep -w ets_printf | wc -l) -eq 0
+
+
+build_ssc:
+  extends: .build_template
+  parallel: 3
+  artifacts:
+    paths:
+      - SSC/ssc_bin
+    expire_in: 1 week
+  variables:
+    SSC_CONFIG_FOLDER: "$CI_PROJECT_DIR/SSC/configs/ESP32_IDF"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_INTEGRATION_TEST
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    - git clone $SSC_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT SSC SSC
+    - cd SSC
+    - MAKEFLAGS= ./ci_build_ssc.sh
+
+build_esp_idf_tests_make:
+  extends: .build_esp_idf_unit_test_template
+  script:
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+    - cd $CI_PROJECT_DIR/tools/unit-test-app
+    - MAKEFLAGS= make help # make sure kconfig tools are built in single process
+    - make ut-clean-all-configs
+    - make ut-build-all-configs
+    - python tools/UnitTestParser.py
+    # Check if the tests demand Make built binaries. If not, delete them
+    - if [ "$UNIT_TEST_BUILD_SYSTEM" == "make" ]; then exit 0; fi
+    - rm -rf builds output sdkconfig
+    - rm $CI_PROJECT_DIR/components/idf_test/unit_test/TestCaseAll.yml
+
+build_esp_idf_tests_cmake:
+  extends: .build_esp_idf_unit_test_template
+  script:
+    - export PATH="$IDF_PATH/tools:$PATH"
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+    - cd $CI_PROJECT_DIR/tools/unit-test-app
+    - idf.py ut-clean-all-configs
+    - idf.py ut-build-all-configs
+    - python tools/UnitTestParser.py
+    # Check if the tests demand CMake built binaries. If not, delete them
+    - if [ "$UNIT_TEST_BUILD_SYSTEM" == "cmake" ]; then exit 0; fi
+    - rm -rf builds output sdkconfig
+    - rm $CI_PROJECT_DIR/components/idf_test/unit_test/TestCaseAll.yml
+
+build_examples_make:
+  extends: .build_template
+  parallel: 8
+  # This is a workaround for a rarely encountered issue with building examples in CI.
+  # Probably related to building of Kconfig in 'make clean' stage
+  retry: 1
+  artifacts:
+    when: always
+    paths:
+      - build_examples/*/*/*/build/*.bin
+      - build_examples/*/*/*/sdkconfig
+      - build_examples/*/*/*/build/*.elf
+      - build_examples/*/*/*/build/*.map
+      - build_examples/*/*/*/build/download.config
+      - build_examples/*/*/*/build/bootloader/*.bin
+      - $LOG_PATH
+    expire_in: 3 days
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples_make"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
+  script:
+    # it's not possible to build 100% out-of-tree and have the "artifacts"
+    # mechanism work, but this is the next best thing
+    - rm -rf build_examples
+    - mkdir build_examples
+    - cd build_examples
+    # build some of examples
+    - mkdir -p ${LOG_PATH}
+    - ${IDF_PATH}/tools/ci/build_examples.sh
+
+# same as above, but for CMake
+build_examples_cmake:
+  extends: .build_template
+  parallel: 5
+  artifacts:
+    when: always
+    paths:
+      - build_examples_cmake/*/*/*/build/*.bin
+      - build_examples_cmake/*/*/*/sdkconfig
+      - build_examples_cmake/*/*/*/build/*.elf
+      - build_examples_cmake/*/*/*/build/*.map
+      - build_examples_cmake/*/*/*/build/flasher_args.json
+      - build_examples_cmake/*/*/*/build/bootloader/*.bin
+      - $LOG_PATH
+    expire_in: 3 days
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
+  script:
+    # it's not possible to build 100% out-of-tree and have the "artifacts"
+    # mechanism work, but this is the next best thing
+    - rm -rf build_examples_cmake
+    - mkdir build_examples_cmake
+    - cd build_examples_cmake
+    # build some of examples
+    - mkdir -p ${LOG_PATH}
+    - ${IDF_PATH}/tools/ci/build_examples_cmake.sh
+
+# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
+
+build_docs:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build_docs
+  artifacts:
+    when: always
+    paths:
+      # English version of documentation
+      - docs/en/doxygen-warning-log.txt
+      - docs/en/sphinx-warning-log.txt
+      - docs/en/sphinx-warning-log-sanitized.txt
+      - docs/en/_build/html
+      - docs/sphinx-err-*
+      # Chinese version of documentation
+      - docs/zh_CN/doxygen-warning-log.txt
+      - docs/zh_CN/sphinx-warning-log.txt
+      - docs/zh_CN/sphinx-warning-log-sanitized.txt
+      - docs/zh_CN/_build/html
+    expire_in: 3 days
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_BUILD_DOCS
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    - cd docs
+    - ./check_lang_folder_sync.sh
+    - cd en
+    - make gh-linkcheck
+    - make html
+    - ../check_doc_warnings.sh
+    - cd ../zh_CN
+    - make gh-linkcheck
+    - make html
+    - ../check_doc_warnings.sh
+
+verify_cmake_style:
+  extends: .check_job_template
+  stage: build
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    tools/cmake/run_cmake_lint.sh
+
+test_build_system:
+  extends: .build_template
+  script:
+    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
+    - rm -rf test_build_system
+    - mkdir test_build_system
+    - cd test_build_system
+    - ${IDF_PATH}/tools/ci/test_build_system.sh
+
+test_build_system_cmake:
+  extends: .build_template
+  script:
+    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
+    - rm -rf test_build_system
+    - mkdir test_build_system
+    - cd test_build_system
+    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh

tools/ci/config/check.yml

@@ -0,0 +1,122 @@
+
+# copy from .gitlab-ci.yml as anchor is not global
+.show_submodule_urls: &show_submodule_urls |
+  git config --get-regexp '^submodule\..*\.url$' || true
+
+check_line_endings:
+  extends: .check_job_template
+  script:
+    - tools/ci/check-line-endings.sh ${IDF_PATH}
+
+check_commit_msg:
+  extends: .check_job_template
+  script:
+    - git status
+    - git log -n10 --oneline
+    # commit start with "WIP: " need to be squashed before merge
+    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
+
+check_permissions:
+  extends: .check_job_template
+  script:
+    - tools/ci/check-executable.sh
+
+check_version:
+  extends: .check_job_template
+  # Don't run this for feature/bugfix branches, so that it is possible to modify
+  # esp_idf_version.h in a branch before tagging the next version.
+  only:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  script:
+    - export IDF_PATH=$PWD
+    - tools/ci/check_idf_version.sh
+
+check_examples_cmake_make:
+  extends: .check_job_template_with_filter
+  except:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  script:
+    - tools/ci/check_examples_cmake_make.sh
+
+check_python_style:
+  extends: .check_job_template_with_filter
+  artifacts:
+    when: on_failure
+    paths:
+      - flake8_output.txt
+    expire_in: 1 week
+  script:
+    # run it only under Python 3 (it is very slow under Python 2)
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
+
+check_kconfigs:
+  extends: .check_job_template_with_filter
+  artifacts:
+    when: on_failure
+    paths:
+      - components/*/Kconfig*.new
+      - examples/*/*/*/Kconfig*.new
+      - examples/*/*/*/*/Kconfig*.new
+      - tools/*/Kconfig*.new
+      - tools/*/*/Kconfig*.new
+      - tools/*/*/*/Kconfig*.new
+    expire_in: 1 week
+  script:
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
+    - ${IDF_PATH}/tools/check_kconfigs.py
+
+check_ut_cmake_make:
+  extends: .check_job_template_with_filter
+  stage: check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  except:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  dependencies: []
+  script:
+    - tools/ci/check_ut_cmake_make.sh
+
+check_submodule_sync:
+  extends: .check_job_template
+  tags:
+    - github_sync
+  retry: 2
+  variables:
+    GIT_STRATEGY: clone
+    GIT_SUBMODULE_STRATEGY: none
+    PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
+  before_script: []
+  after_script: []
+  script:
+    - git submodule deinit --force .
+    # setting the default remote URL to the public one, to resolve relative location URLs
+    - git config remote.origin.url ${PUBLIC_IDF_URL}
+    # check if all submodules are correctly synced to public repostory
+    - git submodule init
+    - *show_submodule_urls
+    - git submodule update --recursive
+    - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
+
+check_artifacts_expire_time:
+  extends: .check_job_template
+  script:
+    # check if we have set expire time for all artifacts
+    - python tools/ci/check_artifacts_expire_time.py
+
+check_pipeline_triggered_by_label:
+  extends: .check_job_template
+  stage: post_check
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL
+  script:
+    # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
+    # We want to make sure some jobs are always executed to detect regression.
+    - test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }

tools/ci/config/deploy.yml

@@ -0,0 +1,153 @@
+
+.clang_tidy_deploy_template:
+  stage: deploy
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  tags:
+    - deploy
+  script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+    - export GIT_VER=$(git describe --always)
+    - cd $IDF_PATH/examples/get-started/hello_world/tidybuild
+    - mv report $GIT_VER
+    - tar czvf $GIT_VER.tar.gz $GIT_VER
+    - export STATIC_REPORT_PATH="web/static_analysis/esp-idf/"
+    - ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy"
+    - scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy
+    - ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+    # add link to view the report
+    - echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html"
+    - test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; }
+
+clang_tidy_deploy:
+  extends: .clang_tidy_deploy_template
+  dependencies:
+    - clang_tidy_check
+    - clang_tidy_check_all
+  variables:
+    BOT_NEEDS_TRIGGER_BY_NAME: 1
+
+clang_tidy_deploy_regular:
+  extends: .clang_tidy_deploy_template
+  dependencies:
+    - clang_tidy_check_regular
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_LABEL_STATIC_ANALYSIS
+      - $BOT_LABEL_STATIC_ANALYSIS_ALL
+
+push_to_github:
+  stage: deploy
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - deploy
+  only:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  when: on_success
+  dependencies: []
+  extends: .before_script_lesser
+  script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $GH_PUSH_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+    - git remote remove github &>/dev/null || true
+    - git remote add github git@github.com:espressif/esp-idf.git
+    - tools/ci/push_to_github.sh
+
+deploy_docs:
+  stage: deploy
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - deploy
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD_DOCS
+  dependencies:
+    - build_docs
+  extends: .before_script_lesser
+  script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+    - export GIT_VER=$(git describe --always)
+    - cd docs/en/_build/
+    - mv html $GIT_VER
+    - tar czvf $GIT_VER.tar.gz $GIT_VER
+    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/en
+    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/en && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+    - cd ../../zh_CN/_build/
+    - mv html $GIT_VER
+    - tar czvf $GIT_VER.tar.gz $GIT_VER
+    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/zh_CN
+    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/zh_CN && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+    # add link to preview doc
+    - echo "[document preview][en] $CI_DOCKER_REGISTRY/docs/esp-idf/en/${GIT_VER}/index.html"
+    - echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html"
+
+deploy_test_result:
+  stage: deploy
+  image: $CI_DOCKER_REGISTRY/bot-env
+  tags:
+    - deploy_test
+  when: always
+  only:
+    refs:
+      - master
+      - schedules
+  artifacts:
+    when: always
+    paths:
+      - ${CI_PROJECT_DIR}/test-management/*.log
+      # save all test logs as artifacts, make it easier to track errors
+      - ${CI_PROJECT_DIR}/TEST_LOGS
+      - $CI_PROJECT_DIR/$CI_COMMIT_SHA
+    expire_in: 1 mos
+  variables:
+    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
+    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
+  before_script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+  script:
+    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
+    - export REV_COUNT=$(git rev-list --count HEAD)
+    - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
+    # artifacts of job update_test_cases creates test-management folder
+    # we need to remove it so we can clone test-management folder again
+    - rm -r test-management
+    - git clone $TEST_MANAGEMENT_REPO
+    - python3 $CHECKOUT_REF_SCRIPT test-management test-management
+    - cd test-management
+    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
+    # update test results
+    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS ${CI_PROJECT_DIR}/${CI_COMMIT_SHA} --pipeline_url ${CI_PIPELINE_URL}

tools/ci/config/host-test.yml

@@ -0,0 +1,286 @@
+
+.host_test_template:
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - host_test
+  dependencies: []
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_HOST_TEST
+      - $BOT_LABEL_REGULAR_TEST
+
+.host_fuzzer_test_template:
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+  tags:
+    - host_test
+  dependencies: []
+  artifacts:
+    when: always
+    paths:
+      - ${FUZZER_TEST_DIR}/out/crashes
+      - ${FUZZER_TEST_DIR}/fuzz_output.log
+    expire_in: 1 week
+  only:
+    variables:
+      - $BOT_LABEL_FUZZER_TEST
+      - $BOT_LABEL_WEEKEND_TEST
+  script:
+    - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
+    - cd ${FUZZER_TEST_DIR}
+    # run AFL fuzzer for one hour
+    - ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
+    - ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
+    # check no crashes found
+    - test -z "$(ls out/crashes/)" || exit 1
+
+.clang_tidy_check_template:
+  stage: host_test
+  image: ${CI_DOCKER_REGISTRY}/clang-static-analysis
+  tags:
+    - host_test
+  dependencies: []
+  artifacts:
+    reports:
+      junit: $IDF_PATH/output.xml
+    when: always
+    paths:
+      - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
+    expire_in: 1 day
+  script:
+    - git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
+    # Setup parameters of triggered/regular job
+    - export TRIGGERED_RELATIVE=${BOT_LABEL_STATIC_ANALYSIS-} && export TRIGGERED_ABSOLUTE=${BOT_LABEL_STATIC_ANALYSIS_ALL-} && export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
+    - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
+
+test_nvs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/nvs_flash/test_nvs_host
+    - make test
+
+test_nvs_coverage:
+  extends: .host_test_template
+  artifacts:
+    paths:
+      - components/nvs_flash/test_nvs_host/coverage_report
+    expire_in: 1 week
+  only:
+    refs:
+      - triggers
+    variables:
+      - $BOT_LABEL_NVS_COVERAGE
+  script:
+    - cd components/nvs_flash/test_nvs_host
+    - make coverage_report
+
+test_partition_table_on_host:
+  extends: .host_test_template
+  tags:
+    - build
+  script:
+    - cd components/partition_table/test_gen_esp32part_host
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./gen_esp32part_tests.py
+
+test_wl_on_host:
+  extends: .host_test_template
+  artifacts:
+    paths:
+      - components/wear_levelling/test_wl_host/coverage_report.zip
+    expire_in: 1 week
+  script:
+    - cd components/wear_levelling/test_wl_host
+    - make test
+
+test_fatfs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/fatfs/test_fatfs_host/
+    - make test
+
+test_ldgen_on_host:
+  extends: .host_test_template
+  script:
+    - cd tools/ldgen/test
+    - ./test_fragments.py
+    - ./test_generation.py
+
+clang_tidy_check:
+  extends: .clang_tidy_check_template
+  variables:
+    BOT_NEEDS_TRIGGER_BY_NAME: 1
+    BOT_LABEL_STATIC_ANALYSIS: 1
+
+clang_tidy_check_regular:
+  extends: .clang_tidy_check_template
+
+clang_tidy_check_all:
+  extends: .clang_tidy_check_template
+  variables:
+    BOT_NEEDS_TRIGGER_BY_NAME: 1
+    BOT_LABEL_STATIC_ANALYSIS_ALL: 1
+
+test_mdns_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/mdns/test_afl_fuzz_host
+
+test_lwip_dns_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dns
+
+test_lwip_dhcp_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dhcp_client
+
+test_lwip_dhcps_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dhcp_server
+
+test_spiffs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/spiffs/test_spiffs_host/
+    - make test
+
+test_multi_heap_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/heap/test_multi_heap_host
+    - ./test_all_configs.sh
+
+test_confserver:
+  extends: .host_test_template
+  script:
+    - cd tools/kconfig_new/test
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confserver.py
+
+test_idf_monitor:
+  extends: .host_test_template
+  artifacts:
+    # save artifacts always in order to access results which were retried without consequent failure
+    when: always
+    paths:
+      - tools/test_idf_monitor/outputs/*
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/test_idf_monitor
+    - ./run_test_idf_monitor.py
+
+test_idf_size:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/test_idf_size/output
+      - tools/test_idf_size/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/test_idf_size
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+
+test_idf_tools:
+  extends: .host_test_template
+  script:
+    # Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment
+    - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/root/.espressif/tools\|/opt/espressif${CUSTOM_TOOLCHAIN_PATH:+\|${CUSTOM_TOOLCHAIN_PATH}}" | tr "\n" ":"); echo ${p%:})
+    - cd ${IDF_PATH}/tools/test_idf_tools
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
+
+test_esp_err_to_name_on_host:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - components/esp32/esp_err_to_name.c
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./gen_esp_err_to_name.py
+    - git diff --exit-code -- ../components/esp32/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./gen_esp_err_to_name.py
+    - git diff --exit-code -- ../components/esp32/esp_err_to_name.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
+
+test_esp_efuse_table_on_host:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - components/efuse/esp32/esp_efuse_table.c
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/components/efuse/
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
+    - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
+    - git diff --exit-code -- ../components/esp32/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
+    - cd ${IDF_PATH}/components/efuse/test_efuse_host
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+
+test_espcoredump:
+  extends: .host_test_template
+  artifacts:
+    when: always
+    paths:
+      - components/espcoredump/test/.coverage
+      - components/espcoredump/test/output
+    expire_in: 1 week
+  script:
+    - cd components/espcoredump/test/
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_espcoredump.sh
+
+test_logtrace_proc:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/esp_app_trace/test/logtrace/output
+      - tools/esp_app_trace/test/logtrace/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+
+test_sysviewtrace_proc:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/esp_app_trace/test/sysview/output
+      - tools/esp_app_trace/test/sysview/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+
+check_doc_links:
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - check_doc_links
+  only:
+    refs:
+      # can only be triggered
+      - triggers
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD_DOCS
+  artifacts:
+    paths:
+      - docs/_build/linkcheck
+    expire_in: 1 week
+  script:
+    # must be triggered with CHECK_LINKS=Yes, otherwise exit without test
+    - test "$CHECK_LINKS" = "Yes" || exit 0
+    # can only run on master branch (otherwise the commit is not on Github yet)
+    - test "${CI_COMMIT_REF_NAME}" = "master" || exit 0
+    - cd docs
+    - make linkcheck

tools/ci/config/target-test.yml

@@ -0,0 +1,572 @@
+
+# for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
+# we need to convert to pattern "job_name_index.yml"
+.define_config_file_name: &define_config_file_name |
+    JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
+    JOG_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
+    CONFIG_FILE="${CONFIG_FILE_PATH}/${JOG_FULL_NAME}.yml"
+
+.example_test_template:
+  stage: target_test
+  when: on_success
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_EXAMPLE_TEST
+  dependencies:
+    - assign_test
+    - build_examples_make
+    - build_examples_cmake
+  artifacts:
+    when: always
+    paths:
+      - $LOG_PATH
+    expire_in: 1 week
+    reports:
+        junit: $LOG_PATH/*/XUNIT_RESULT.xml
+  variables:
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
+  script:
+    - *define_config_file_name
+    # first test if config file exists, if not exist, exit 0
+    - test -e $CONFIG_FILE || exit 0
+    # clone test env configs
+    - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
+    - cd $TEST_FW_PATH
+    # run test
+    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
+
+.unit_test_template:
+  extends: .example_test_template
+  stage: target_test
+  dependencies:
+    - assign_test
+    - build_esp_idf_tests_make
+    - build_esp_idf_tests_cmake
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_UNIT_TEST
+  variables:
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
+
+.test_template:
+  stage: target_test
+  when: on_success
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_INTEGRATION_TEST
+  dependencies:
+    - assign_test
+    - build_ssc
+  artifacts:
+    when: always
+    reports:
+        junit: $LOG_PATH/*/XUNIT_RESULT.xml
+    paths:
+      - $LOG_PATH
+    expire_in: 1 week
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
+    LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
+    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
+    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+  script:
+    - *define_config_file_name
+    # first test if config file exists, if not exist, exit 0
+    - test -e $CONFIG_FILE || exit 0
+    # clone local test env configs
+    - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
+    # clone test bench
+    - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - cd auto_test_script
+    # run test
+    - python CIRunner.py -l "$LOG_PATH/$JOG_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
+
+test_weekend_mqtt:
+  extends: .example_test_template
+  stage: target_test
+  tags:
+    - ESP32
+    - Example_WIFI
+  only:
+    variables:
+      - $BOT_LABEL_WEEKEND_TEST
+  variables:
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
+    CONFIG_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/config.yml"
+
+test_weekend_network:
+  extends: .example_test_template
+  stage: target_test
+  image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - ESP32
+    - Example_WIFI
+  only:
+    variables:
+      - $BOT_LABEL_WEEKEND_TEST
+  variables:
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
+    CONFIG_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/config.yml"
+
+example_test_001:
+  extends: .example_test_template
+  parallel: 2
+  tags:
+    - ESP32
+    - Example_WIFI
+
+example_test_002:
+  extends: .example_test_template
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - ESP32
+    - Example_ShieldBox_Basic
+
+.example_test_003:
+  extends: .example_test_template
+  tags:
+    - ESP32
+    - Example_SDIO
+
+example_test_004:
+  extends: .example_test_template
+  tags:
+    - ESP32
+    - Example_CAN
+
+example_test_005:
+  extends: .example_test_template
+  tags:
+    - ESP32
+    - Example_WIFI_BT
+
+example_test_006:
+  extends: .example_test_template
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  only:
+    variables:
+      - $BOT_LABEL_IPERF_STRESS_TEST
+  tags:
+    - ESP32
+    - Example_ShieldBox
+
+example_test_007:
+  extends: .example_test_template
+  tags:
+    - ESP32
+    - Example_I2C_CCS811_SENSOR
+
+UT_001:
+  extends: .unit_test_template
+  parallel: 50
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
+UT_002:
+  extends: .unit_test_template
+  parallel: 18
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - psram
+
+UT_003:
+  extends: .unit_test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - UT_T1_SDMODE
+
+UT_004:
+  extends: .unit_test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - UT_T1_SPIMODE
+
+UT_005:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_SDMODE
+    - psram
+
+UT_006:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_SPIMODE
+    - psram
+
+UT_007:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_GPIO
+
+UT_008:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_GPIO
+    - psram
+
+UT_009:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_PCNT
+
+UT_010:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_PCNT
+    - psram
+
+UT_011:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_LEDC
+
+UT_012:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_LEDC
+    - psram
+
+UT_013:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T2_RS485
+
+UT_014:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_RS485
+    - psram
+
+UT_015:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_RMT
+
+UT_016:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_RMT
+    - psram
+
+UT_017:
+  extends: .unit_test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - EMMC
+
+UT_018:
+  extends: .unit_test_template
+  parallel: 5
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - 8Mpsram
+
+UT_019:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - Example_SPI_Multi_device
+
+UT_020:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - Example_SPI_Multi_device
+    - psram
+
+UT_021:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T2_I2C
+
+UT_022:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_I2C
+    - psram
+
+UT_023:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_MCPWM
+
+UT_024:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_MCPWM
+    - psram
+
+UT_025:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_I2S
+
+UT_026:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_I2S
+    - psram
+
+UT_027:
+  extends: .unit_test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+
+UT_028:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+    - psram
+
+UT_029:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+    - 8Mpsram
+
+# Gitlab parallel max value is 50. We need to create another UT job if parallel is larger than 50.
+UT_030:
+  extends: .unit_test_template
+  parallel: 5
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
+nvs_compatible_test:
+  extends: .test_template
+  artifacts:
+    when: always
+    paths:
+      - $LOG_PATH
+      - nvs_wifi.bin
+    expire_in: 1 mos
+  tags:
+    - ESP32_IDF
+    - NVS_Compatible
+  script:
+    - *define_config_file_name
+    # first test if config file exists, if not exist, exit 0
+    - test -e $CONFIG_FILE || exit 0
+    # clone local test env configs
+    - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
+    # clone test bench
+    - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - cd auto_test_script
+    # prepare nvs bins
+    - ./Tools/prepare_nvs_bin.sh
+    # run test
+    - python CIRunner.py -l "$LOG_PATH/$JOG_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
+
+IT_001:
+  extends: .test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - SSC_T1_4
+
+IT_002:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_2
+
+IT_003:
+  extends: .test_template
+  parallel: 13
+  tags:
+    - ESP32_IDF
+    - SSC_T2_5
+
+IT_004:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_APC
+
+IT_005:
+  extends: .test_template
+  parallel: 2
+  tags:
+    - ESP32_IDF
+    - SSC_T1_5
+
+IT_006:
+  extends: .test_template
+  parallel: 8
+  tags:
+    - ESP32_IDF
+    - SSC_T1_6
+
+IT_007:
+  extends: .test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - SSC_T1_7
+
+IT_008:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_8
+
+IT_009:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_3
+
+IT_010:
+  extends: .test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - SSC_T5_1
+
+IT_011:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_MESH1
+
+IT_012:
+  extends: .test_template
+  parallel: 2
+  tags:
+    - ESP32_IDF
+    - SSC_T2_MESH1
+
+IT_013:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T3_MESH1
+
+IT_014:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T6_MESH1
+
+IT_015:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T12_MESH1
+
+IT_016:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T50_MESH1
+
+IT_017:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_MESH2
+
+IT_018:
+  extends: .test_template
+  parallel: 2
+  tags:
+    - ESP32_IDF
+    - SSC_T1_9
+
+IT_019:
+  extends: .test_template
+  parallel: 2
+  tags:
+    - ESP32_IDF
+    - SSC_T2_2
+
+IT_020:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T2_3
+
+IT_021:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T2_4

tools/tiny-test-fw/Utility/CIAssignTest.py

@@ -134,6 +134,19 @@ class AssignTest(object):
         self.jobs = self._parse_gitlab_ci_config(ci_config_file)
         self.case_group = case_group
 
+    @staticmethod
+    def _handle_parallel_attribute(job_name, job):
+        jobs_out = []
+        try:
+            for i in range(job["parallel"]):
+                jobs_out.append(GitlabCIJob.Job(job, job_name + "_{}".format(i + 1)))
+        except KeyError:
+            # Gitlab don't allow to set parallel to 1.
+            # to make test job name same ($CI_JOB_NAME_$CI_NODE_INDEX),
+            # we append "_" to jobs don't have parallel attribute
+            jobs_out.append(GitlabCIJob.Job(job, job_name + "_"))
+        return jobs_out
+
     def _parse_gitlab_ci_config(self, ci_config_file):
 
         with open(ci_config_file, "r") as f:
@@ -142,7 +155,7 @@ class AssignTest(object):
         job_list = list()
         for job_name in ci_config:
             if self.CI_TEST_JOB_PATTERN.search(job_name) is not None:
-                job_list.append(GitlabCIJob.Job(ci_config[job_name], job_name))
+                job_list.extend(self._handle_parallel_attribute(job_name, ci_config[job_name]))
         job_list.sort(key=lambda x: x["name"])
         return job_list
 

tools/unit-test-app/tools/UnitTestParser.py

@@ -28,8 +28,8 @@ TEST_CASE_PATTERN = {
 class Parser(object):
     """ parse unit test cases from build files and create files for test bench """
 
-    TAG_PATTERN = re.compile("([^=]+)(=)?(.+)?")
+    TAG_PATTERN = re.compile(r"([^=]+)(=)?(.+)?")
-    DESCRIPTION_PATTERN = re.compile("\[([^]\[]+)\]")  # noqa: W605 - regular expression
+    DESCRIPTION_PATTERN = re.compile(r"\[([^]\[]+)\]")
     CONFIG_PATTERN = re.compile(r"{([^}]+)}")
     TEST_GROUPS_PATTERN = re.compile(r"TEST_GROUPS=(.*)$")
 
@@ -261,7 +261,12 @@ class Parser(object):
         dump parsed test cases to YAML file for test bench input
         :param test_cases: parsed test cases
         """
-        with open(os.path.join(self.idf_path, self.TEST_CASE_FILE), "w+") as f:
+        filename = os.path.join(self.idf_path, self.TEST_CASE_FILE)
+        try:
+            os.mkdir(os.path.dirname(filename))
+        except OSError:
+            pass
+        with open(os.path.join(filename), "w+") as f:
             yaml.dump({"test cases": test_cases}, f, allow_unicode=True, default_flow_style=False)
 
     def copy_module_def_file(self):