Merge branch 'ci/split_assign_test_and_add_c3_integration_test_4.4' into 'release/v4.4'

Ci: split assign test and add c3 integration test 4.4

See merge request espressif/esp-idf!16925

.gitlab/ci/assign-test.yml

@@ -1,54 +1,134 @@
-assign_test:
+.assign_test_template:
-  extends: .rules:test:any_test
-  tags:
-    - assign_test
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
   stage: assign_test
-  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
+  tags:
-  # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
+    - assign_test
-  dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here.
-    - build_ssc_esp32
-    - build_esp_idf_tests_cmake_esp32
-    - build_esp_idf_tests_cmake_esp32s2
-    - build_esp_idf_tests_cmake_esp32s3
-    - build_esp_idf_tests_cmake_esp32c3
   variables:
-    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
+    SUBMODULES_TO_FETCH: components/esptool_py/esptool
-    EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples"
+  artifacts:
-    CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps"
+    paths:
-    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
+      - ${TEST_DIR}/test_configs
+      - ${BUILD_DIR}/artifact_index.json
+    when: always
+    expire_in: 1 week
+  script:
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+
+assign_example_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:example_test
+  needs:
+    - job: build_examples_cmake_esp32
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: example_test
+    TEST_DIR: ${CI_PROJECT_DIR}/examples
+    BUILD_DIR: ${CI_PROJECT_DIR}/build_examples
+
+assign_custom_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:custom_test
+  needs:
+    - job: build_test_apps_esp32
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: custom_test
+    TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
+    BUILD_DIR: ${CI_PROJECT_DIR}/build_test_apps
+
+assign_component_ut:
+  extends:
+    - .assign_test_template
+    - .rules:build:component_ut
+  needs:
+    - job: build_component_ut_esp32
+      artifacts: false
+      optional: true
+    - job: build_component_ut_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_component_ut_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_component_ut_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: component_ut
+    TEST_DIR: ${CI_PROJECT_DIR}/component_ut
+    BUILD_DIR: ${CI_PROJECT_DIR}/build_component_ut
+  script:
     # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh`
-    COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut"
+    - set_component_ut_vars
-    INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+
+assign_unit_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:unit_test
+  needs:
+    - job: build_esp_idf_tests_cmake_esp32
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32s2
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32c3
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32s3
+      optional: true
+  variables:
+    TEST_TYPE: unit_test
+    TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/unit_test
+    BUILD_DIR: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
+  script:
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+
+assign_integration_test:
+  extends:
+    - .assign_test_template
+    - .rules:test:integration_test
+  needs:
+    - build_ssc_esp32
+    - build_ssc_esp32c3
+  artifacts:
+    paths:
+      - $TEST_DIR/test_configs
+  variables:
+    TEST_DIR: ${CI_PROJECT_DIR}/tools/ci/integration_test
+    BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin
     INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
     ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
-    # auto_test_script only supports python 3.7.x
     PYTHON_VER: 3.7.7
-  artifacts:
-    paths:
-      - components/idf_test/*/CIConfigs
-      - $EXAMPLE_TEST_DIR/test_configs
-      - $CUSTOM_TEST_DIR/test_configs
-      - $COMPONENT_UT_OUTPUT_DIR/test_configs
-      - build_examples/artifact_index.json
-      - build_test_apps/artifact_index.json
-      - build_component_ut/artifact_index.json
-      - tools/unit-test-app/builds/artifact_index.json
-    expire_in: 1 week
   script:
-    - set_component_ut_vars
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
     # clone test script to assign tests
     # can not retry if downing git lfs files failed, so using empty_branch first.
     - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
     - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     # assign integration test cases
-    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
+    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs
 
 update_test_cases:
   extends: .rules:ref:master-schedule
@@ -70,7 +150,7 @@ update_test_cases:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
     BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
-    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
+    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
     PYTHON_VER: 3.7.7
   script:
     - export GIT_SHA=$(echo ${PIPELINE_COMMIT_SHA} | cut -c 1-8)

.gitlab/ci/dependencies/dependencies.yml

@@ -61,6 +61,9 @@
   patterns:
     - build_components
     - build_system
+  included_in:
+    - "build:{0}"
+    - build:target_test
 
 build:integration_test:
   labels:
@@ -68,6 +71,8 @@ build:integration_test:
   patterns:
     - build_components
     - build_system
+  included_in:
+    - build:target_test
 
 ####################
 # Target Test Jobs #
@@ -79,14 +84,14 @@ build:integration_test:
   labels: # For each rule, use labels <test_type> and <test_type>-<target>
     - "{0}"
     - "{0}_{1}"
+    - target_test
   patterns: # For each rule, use patterns <test_type> and build-<test_type>
     - "{0}"
     - "build-{0}"
   included_in: # Parent rules
+    - "build:{0}"
     - "build:{0}-{1}"
     - build:target_test
-    - test:target_test
-    - test:any_test
 
 # -------------
 # Special Cases
@@ -99,33 +104,30 @@ build:integration_test:
     - "component_ut_{0}"
     - unit_test
     - "unit_test_{0}"
+    - target_test
   patterns:
     - component_ut
     - "build-component_ut-{0}"
   included_in:
+    - build:component_ut
     - "build:component_ut-{0}"
     - build:target_test
-    - test:target_test
-    - test:any_test
 
 "test:integration_test":
   labels:
-    - "integration_test"
+    - integration_test
+    - target_test
   patterns:
-    - "integration_test"
+    - integration_test
   included_in:
-    - "build:integration_test"
+    - build:integration_test
     - build:target_test
-    - test:target_test
-    - test:any_test
 
 "test:host_test":
   labels:
     - host_test
   patterns:
     - host_test
-  included_in:
-    - test:any_test
 
 "test:submodule":
   labels:
@@ -140,17 +142,17 @@ build:integration_test:
   labels:
     - iperf_stress_test
   included_in:
+    - build:example_test
     - build:example_test-esp32
     - build:target_test
-    - test:any_test
 
 "labels:weekend_test": # custom test
   labels:
     - weekend_test
   included_in:
+    - build:custom_test
     - build:custom_test-esp32
     - build:target_test
-    - test:any_test
 
 "labels:nvs_coverage": # host_test
   labels:

.gitlab/ci/rules.yml

@@ -89,6 +89,7 @@
 
 .patterns-integration_test: &patterns-integration_test
   - "tools/ci/python_packages/tiny_test_fw/**/*"
+  - "tools/ci/integration_test/**/*"
 
 .patterns-host_test: &patterns-host_test
   - ".gitlab/ci/host-test.yml"
@@ -336,6 +337,9 @@
 .if-label-submodule: &if-label-submodule
   if: '$BOT_LABEL_SUBMODULE || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*submodule(?:,[^,\n\r]+)*$/i'
 
+.if-label-target_test: &if-label-target_test
+  if: '$BOT_LABEL_TARGET_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i'
+
 .if-label-unit_test: &if-label-unit_test
   if: '$BOT_LABEL_UNIT_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test(?:,[^,\n\r]+)*$/i'
 
@@ -366,12 +370,35 @@
     - <<: *if-dev-push
       changes: *patterns-build_system
 
+.rules:build:component_ut:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-label-component_ut
+    - <<: *if-label-component_ut_esp32
+    - <<: *if-label-component_ut_esp32c3
+    - <<: *if-label-component_ut_esp32s2
+    - <<: *if-label-component_ut_esp32s3
+    - <<: *if-label-target_test
+    - <<: *if-label-unit_test
+    - <<: *if-label-unit_test_esp32
+    - <<: *if-label-unit_test_esp32c3
+    - <<: *if-label-unit_test_esp32s2
+    - <<: *if-label-unit_test_esp32s3
+    - <<: *if-dev-push
+      changes: *patterns-build_components
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-component_ut
+
 .rules:build:component_ut-esp32:
   rules:
     - <<: *if-protected
     - <<: *if-label-build
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32
     - <<: *if-dev-push
@@ -387,6 +414,7 @@
     - <<: *if-label-build
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32c3
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32c3
     - <<: *if-dev-push
@@ -402,6 +430,7 @@
     - <<: *if-label-build
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32s2
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32s2
     - <<: *if-dev-push
@@ -417,6 +446,7 @@
     - <<: *if-label-build
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32s3
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32s3
     - <<: *if-dev-push
@@ -426,12 +456,31 @@
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
+.rules:build:custom_test:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-label-custom_test
+    - <<: *if-label-custom_test_esp32
+    - <<: *if-label-custom_test_esp32c3
+    - <<: *if-label-custom_test_esp32s2
+    - <<: *if-label-custom_test_esp32s3
+    - <<: *if-label-target_test
+    - <<: *if-label-weekend_test
+    - <<: *if-dev-push
+      changes: *patterns-build_components
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-custom_test
+
 .rules:build:custom_test-esp32:
   rules:
     - <<: *if-protected
     - <<: *if-label-build
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32
+    - <<: *if-label-target_test
     - <<: *if-label-weekend_test
     - <<: *if-dev-push
       changes: *patterns-build_components
@@ -446,6 +495,7 @@
     - <<: *if-label-build
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32c3
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build_components
     - <<: *if-dev-push
@@ -459,6 +509,7 @@
     - <<: *if-label-build
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32s2
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build_components
     - <<: *if-dev-push
@@ -472,6 +523,7 @@
     - <<: *if-label-build
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32s3
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build_components
     - <<: *if-dev-push
@@ -487,6 +539,26 @@
     - <<: *if-dev-push
       changes: *patterns-docker
 
+.rules:build:example_test:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-label-example_test
+    - <<: *if-label-example_test_esp32
+    - <<: *if-label-example_test_esp32c3
+    - <<: *if-label-example_test_esp32s2
+    - <<: *if-label-example_test_esp32s3
+    - <<: *if-label-iperf_stress_test
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-build-example_test
+    - <<: *if-dev-push
+      changes: *patterns-build_components
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-example_test
+
 .rules:build:example_test-esp32:
   rules:
     - <<: *if-protected
@@ -494,6 +566,7 @@
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32
     - <<: *if-label-iperf_stress_test
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -509,6 +582,7 @@
     - <<: *if-label-build
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32c3
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -524,6 +598,7 @@
     - <<: *if-label-build
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32s2
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -539,6 +614,7 @@
     - <<: *if-label-build
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32s3
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -553,6 +629,7 @@
     - <<: *if-protected
     - <<: *if-label-build
     - <<: *if-label-integration_test
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build_components
     - <<: *if-dev-push
@@ -590,6 +667,7 @@
     - <<: *if-label-example_test_esp32s3
     - <<: *if-label-integration_test
     - <<: *if-label-iperf_stress_test
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32
     - <<: *if-label-unit_test_esp32c3
@@ -613,10 +691,28 @@
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
+.rules:build:unit_test:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-label-target_test
+    - <<: *if-label-unit_test
+    - <<: *if-label-unit_test_esp32
+    - <<: *if-label-unit_test_esp32c3
+    - <<: *if-label-unit_test_esp32s2
+    - <<: *if-label-unit_test_esp32s3
+    - <<: *if-dev-push
+      changes: *patterns-build_components
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-unit_test
+
 .rules:build:unit_test-esp32:
   rules:
     - <<: *if-protected
     - <<: *if-label-build
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32
     - <<: *if-dev-push
@@ -630,6 +726,7 @@
   rules:
     - <<: *if-protected
     - <<: *if-label-build
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32c3
     - <<: *if-dev-push
@@ -643,6 +740,7 @@
   rules:
     - <<: *if-protected
     - <<: *if-label-build
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32s2
     - <<: *if-dev-push
@@ -656,6 +754,7 @@
   rules:
     - <<: *if-protected
     - <<: *if-label-build
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32s3
     - <<: *if-dev-push
@@ -692,50 +791,6 @@
   rules:
     - <<: *if-label-weekend_test
 
-.rules:test:any_test:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build-only
-      when: never
-    - <<: *if-label-component_ut
-    - <<: *if-label-component_ut_esp32
-    - <<: *if-label-component_ut_esp32c3
-    - <<: *if-label-component_ut_esp32s2
-    - <<: *if-label-component_ut_esp32s3
-    - <<: *if-label-custom_test
-    - <<: *if-label-custom_test_esp32
-    - <<: *if-label-custom_test_esp32c3
-    - <<: *if-label-custom_test_esp32s2
-    - <<: *if-label-custom_test_esp32s3
-    - <<: *if-label-example_test
-    - <<: *if-label-example_test_esp32
-    - <<: *if-label-example_test_esp32c3
-    - <<: *if-label-example_test_esp32s2
-    - <<: *if-label-example_test_esp32s3
-    - <<: *if-label-host_test
-    - <<: *if-label-integration_test
-    - <<: *if-label-iperf_stress_test
-    - <<: *if-label-unit_test
-    - <<: *if-label-unit_test_esp32
-    - <<: *if-label-unit_test_esp32c3
-    - <<: *if-label-unit_test_esp32s2
-    - <<: *if-label-unit_test_esp32s3
-    - <<: *if-label-weekend_test
-    - <<: *if-dev-push
-      changes: *patterns-build-example_test
-    - <<: *if-dev-push
-      changes: *patterns-component_ut
-    - <<: *if-dev-push
-      changes: *patterns-custom_test
-    - <<: *if-dev-push
-      changes: *patterns-example_test
-    - <<: *if-dev-push
-      changes: *patterns-host_test
-    - <<: *if-dev-push
-      changes: *patterns-integration_test
-    - <<: *if-dev-push
-      changes: *patterns-unit_test
-
 .rules:test:component_ut-esp32:
   rules:
     - <<: *if-protected
@@ -743,6 +798,7 @@
       when: never
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32
     - <<: *if-dev-push
@@ -755,6 +811,7 @@
       when: never
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32c3
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32c3
     - <<: *if-dev-push
@@ -767,6 +824,7 @@
       when: never
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32s2
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32s2
     - <<: *if-dev-push
@@ -779,6 +837,7 @@
       when: never
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32s3
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32s3
     - <<: *if-dev-push
@@ -791,6 +850,7 @@
       when: never
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -801,6 +861,7 @@
       when: never
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32c3
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -811,6 +872,7 @@
       when: never
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32s2
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -821,6 +883,7 @@
       when: never
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32s3
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -831,6 +894,7 @@
       when: never
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -843,6 +907,7 @@
       when: never
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32c3
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -855,6 +920,7 @@
       when: never
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32s2
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -867,6 +933,7 @@
       when: never
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32s3
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -887,6 +954,7 @@
     - <<: *if-label-build-only
       when: never
     - <<: *if-label-integration_test
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-integration_test
 
@@ -899,50 +967,12 @@
     - <<: *if-dev-push
       changes: *patterns-submodule
 
-.rules:test:target_test:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build-only
-      when: never
-    - <<: *if-label-component_ut
-    - <<: *if-label-component_ut_esp32
-    - <<: *if-label-component_ut_esp32c3
-    - <<: *if-label-component_ut_esp32s2
-    - <<: *if-label-component_ut_esp32s3
-    - <<: *if-label-custom_test
-    - <<: *if-label-custom_test_esp32
-    - <<: *if-label-custom_test_esp32c3
-    - <<: *if-label-custom_test_esp32s2
-    - <<: *if-label-custom_test_esp32s3
-    - <<: *if-label-example_test
-    - <<: *if-label-example_test_esp32
-    - <<: *if-label-example_test_esp32c3
-    - <<: *if-label-example_test_esp32s2
-    - <<: *if-label-example_test_esp32s3
-    - <<: *if-label-integration_test
-    - <<: *if-label-unit_test
-    - <<: *if-label-unit_test_esp32
-    - <<: *if-label-unit_test_esp32c3
-    - <<: *if-label-unit_test_esp32s2
-    - <<: *if-label-unit_test_esp32s3
-    - <<: *if-dev-push
-      changes: *patterns-build-example_test
-    - <<: *if-dev-push
-      changes: *patterns-component_ut
-    - <<: *if-dev-push
-      changes: *patterns-custom_test
-    - <<: *if-dev-push
-      changes: *patterns-example_test
-    - <<: *if-dev-push
-      changes: *patterns-integration_test
-    - <<: *if-dev-push
-      changes: *patterns-unit_test
-
 .rules:test:unit_test-esp32:
   rules:
     - <<: *if-protected
     - <<: *if-label-build-only
       when: never
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32
     - <<: *if-dev-push
@@ -953,6 +983,7 @@
     - <<: *if-protected
     - <<: *if-label-build-only
       when: never
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32c3
     - <<: *if-dev-push
@@ -963,6 +994,7 @@
     - <<: *if-protected
     - <<: *if-label-build-only
       when: never
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32s2
     - <<: *if-dev-push
@@ -973,6 +1005,7 @@
     - <<: *if-protected
     - <<: *if-label-build-only
       when: never
+    - <<: *if-label-target_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32s3
     - <<: *if-dev-push

.gitlab/ci/target-test.yml

@@ -7,13 +7,13 @@
 
 .target_test_job_template:
   stage: target_test
-  needs:
-    - assign_test
   artifacts:
     when: always
     paths:
       - "**/*.log"
       - $LOG_PATH
+    exclude:
+      - .git/**/*
     expire_in: 1 week
     reports:
       junit: $LOG_PATH/*/XUNIT_RESULT.xml
@@ -38,6 +38,8 @@
 
 .example_test_template:
   extends: .target_test_job_template
+  needs:
+    - assign_example_test
   variables:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
@@ -295,6 +297,8 @@ example_test_ESP32C3_SDSPI:
 
 .test_app_template:
   extends: .target_test_job_template
+  needs:
+    - assign_custom_test
   variables:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
@@ -381,6 +385,8 @@ test_app_test_flash_psram_f8r8:
 
 .component_ut_template:
   extends: .target_test_job_template
+  needs: # the assign already needs all the build jobs
+    - assign_component_ut
   variables:
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
   script:
@@ -443,9 +449,11 @@ component_ut_test_esp32c3:
 
 .unit_test_template:
   extends: .target_test_job_template
+  needs: # the assign already needs all the build jobs
+    - assign_unit_test
   variables:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/test_configs"
 
 .unit_test_esp32_template:
   extends:
@@ -761,18 +769,18 @@ component_ut_test_lan8720:
   extends:
     - .target_test_job_template
     - .rules:test:integration_test
-  needs:
+  needs: # the assign already needs all the build jobs
-    - assign_test
+    - assign_integration_test
-    - build_ssc_esp32
   variables:
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
-    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/ci/integration_test/test_configs"
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+    KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/tools/ci/integration_test/KnownIssues"
-    KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues"
     CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
-    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
+    PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py"
+    PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${PYTHONPATH}"
+    INITIAL_CONDITION_RETRY_COUNT: "1"
     # auto_test_script only supports python 3.7.x
     PYTHON_VER: 3.7.7
   script:
@@ -789,8 +797,15 @@ component_ut_test_lan8720:
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
     # run test
+    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
     - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
 
+.integration_test_esp32c3_template:
+  extends:
+    - .integration_test_template
+  variables:
+    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C3_IDF"
+
 nvs_compatible_test:
   extends: .integration_test_template
   artifacts:
@@ -818,6 +833,7 @@ nvs_compatible_test:
     - cd auto_test_script
     - ./tools/prepare_nvs_bin.sh
     # run test
+    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
     - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
 
 IT_001:
@@ -932,3 +948,10 @@ IT_022:
   tags:
     - ESP32_IDF
     - SSC_T3_2
+
+IT_C3_001:
+  extends: .integration_test_esp32c3_template
+  parallel: 6
+  tags:
+    - ESP32C3_IDF
+    - SSC_T2_5

tools/ci/checkout_project_ref.py

@@ -84,11 +84,20 @@ if __name__ == '__main__':
 
     ref_to_use = ''
     for candidate in candidate_branches:
-        # check if candidate branch exists
+        # check if the branch, tag or commit exists
-        branch_match = subprocess.check_output(['git', 'branch', '-a', '--list', 'origin/' + candidate])
+        try:
-        if branch_match:
+            subprocess.check_call(['git', 'cat-file', '-t', 'origin/{}'.format(candidate)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             ref_to_use = candidate
             break
+        except subprocess.CalledProcessError:
+            try:
+                # For customized commits
+                subprocess.check_call(['git', 'cat-file', '-t', candidate], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                ref_to_use = candidate
+                break
+            except subprocess.CalledProcessError:
+                pass
+            continue
 
     if ref_to_use:
         for _ in range(RETRY_COUNT):

tools/ci/integration_test/KnownIssues

@@ -1,2 +1,4 @@
 # MESH
 ESP32.MESH_EST_2109
+
+ESP32C3.NIMBLE_GAP_14009

tools/ci/integration_test/README.md

@@ -0,0 +1,21 @@
+# Integration Test Description
+
+## Case Lists
+- WiFi Standard cases for only esp32.
+- BLE Standard cases for esp32 and esp32c3.
+
+## Trigger
+- By labels:
+  - `integration_test`
+- By file changes:
+  - integration test related files
+- By bot:
+  - `@bot test with label: integration_test`
+
+## Advanced
+- There are labels can be used to run less integration test cases.
+  - These labels only take effect when the integration test has been triggered.
+  - label: `integration::wifi_only`
+    - Only run WiFi cases.
+  - label: `integration::ble_only`
+    - Only run BLE cases.

tools/ci/integration_test/prepare_test_bins.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+#
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import os
+
+import gitlab
+import gitlab_api
+from AutoTestScript.RunnerConfigs.Config import Config
+
+SSC_BUILD_JOB_MAP = {
+    'ESP32': 'build_ssc_esp32',
+    'ESP32C3': 'build_ssc_esp32c3',
+}
+NEEDED_FILES = [
+    'flasher_args.json',
+    'bootloader/bootloader.bin',
+    'partition_table/partition-table.bin',
+    'ssc.bin',
+    'ssc.elf',
+]
+IDF_PATH = os.environ.get('IDF_PATH')
+
+
+def try_to_download_artifacts(bin_path: str) -> None:
+    '''
+    bin_path: "SSC/ssc_bin/ESP32[C3]/SSC[_APP]"
+    '''
+    project_id = os.getenv('CI_PROJECT_ID')
+    pipeline_id = os.getenv('CI_PIPELINE_ID')
+    gitlab_inst = gitlab_api.Gitlab(project_id)
+    build_job_name = SSC_BUILD_JOB_MAP[bin_path.split('/')[-2]]
+    job_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
+    files_to_download = [os.path.join(bin_path, f) for f in NEEDED_FILES]
+    for job_info in job_list:
+        try:
+            gitlab_inst.download_artifact(job_info['id'], files_to_download, IDF_PATH)
+            print('Downloaded {} from {}'.format(bin_path, job_info['id']))
+            break
+        except gitlab.exceptions.GitlabError as e:
+            if e.response_code == 404:
+                continue
+            raise
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'test_config_file',
+        help='The test config file to be used.'
+    )
+    args = parser.parse_args()
+
+    configs = Config.parse(args.test_config_file)
+    test_bin_paths = configs.get_bin_paths()
+
+    for _path in test_bin_paths:
+        if os.path.exists(_path):
+            continue
+        relative_path = os.path.relpath(_path, IDF_PATH)
+        try_to_download_artifacts(relative_path)
+
+
+if __name__ == '__main__':
+    main()