diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index c03e2cdadd..6466d15bd8 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -1,54 +1,134 @@ -assign_test: - extends: .rules:test:any_test - tags: - - assign_test +.assign_test_template: image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG stage: assign_test - # gitlab ci do not support match job with RegEx or wildcard now in dependencies. - # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage. - dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here. - - build_ssc_esp32 - - build_esp_idf_tests_cmake_esp32 - - build_esp_idf_tests_cmake_esp32s2 - - build_esp_idf_tests_cmake_esp32s3 - - build_esp_idf_tests_cmake_esp32c3 + tags: + - assign_test variables: - SUBMODULES_TO_FETCH: "components/esptool_py/esptool" - EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples" - CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps" - UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test" + SUBMODULES_TO_FETCH: components/esptool_py/esptool + artifacts: + paths: + - ${TEST_DIR}/test_configs + - ${BUILD_DIR}/artifact_index.json + when: always + expire_in: 1 week + script: + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs + +assign_example_test: + extends: + - .assign_test_template + - .rules:build:example_test + needs: + - job: build_examples_cmake_esp32 + artifacts: false + optional: true + - job: build_examples_cmake_esp32s2 + artifacts: false + optional: true + - job: build_examples_cmake_esp32c3 + artifacts: false + optional: true + - job: build_examples_cmake_esp32s3 + artifacts: false + optional: true + variables: + TEST_TYPE: example_test + TEST_DIR: ${CI_PROJECT_DIR}/examples + BUILD_DIR: ${CI_PROJECT_DIR}/build_examples + +assign_custom_test: + extends: + - .assign_test_template + - .rules:build:custom_test + needs: + - job: build_test_apps_esp32 + artifacts: false + optional: true + - job: build_test_apps_esp32s2 + artifacts: false + optional: true + - job: build_test_apps_esp32c3 + artifacts: false + optional: true + - job: build_test_apps_esp32s3 + artifacts: false + optional: true + variables: + TEST_TYPE: custom_test + TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps + BUILD_DIR: ${CI_PROJECT_DIR}/build_test_apps + +assign_component_ut: + extends: + - .assign_test_template + - .rules:build:component_ut + needs: + - job: build_component_ut_esp32 + artifacts: false + optional: true + - job: build_component_ut_esp32s2 + artifacts: false + optional: true + - job: build_component_ut_esp32c3 + artifacts: false + optional: true + - job: build_component_ut_esp32s3 + artifacts: false + optional: true + variables: + TEST_TYPE: component_ut + TEST_DIR: ${CI_PROJECT_DIR}/component_ut + BUILD_DIR: ${CI_PROJECT_DIR}/build_component_ut + script: # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh` - COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut" - INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" + - set_component_ut_vars + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs + +assign_unit_test: + extends: + - .assign_test_template + - .rules:build:unit_test + needs: + - job: build_esp_idf_tests_cmake_esp32 + optional: true + - job: build_esp_idf_tests_cmake_esp32s2 + optional: true + - job: build_esp_idf_tests_cmake_esp32c3 + optional: true + - job: build_esp_idf_tests_cmake_esp32s3 + optional: true + variables: + TEST_TYPE: unit_test + TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/unit_test + BUILD_DIR: ${CI_PROJECT_DIR}/tools/unit-test-app/builds + script: + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs + +assign_integration_test: + extends: + - .assign_test_template + - .rules:test:integration_test + needs: + - build_ssc_esp32 + - build_ssc_esp32c3 + artifacts: + paths: + - $TEST_DIR/test_configs + variables: + TEST_DIR: ${CI_PROJECT_DIR}/tools/ci/integration_test + BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles" ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages - # auto_test_script only supports python 3.7.x PYTHON_VER: 3.7.7 - artifacts: - paths: - - components/idf_test/*/CIConfigs - - $EXAMPLE_TEST_DIR/test_configs - - $CUSTOM_TEST_DIR/test_configs - - $COMPONENT_UT_OUTPUT_DIR/test_configs - - build_examples/artifact_index.json - - build_test_apps/artifact_index.json - - build_component_ut/artifact_index.json - - tools/unit-test-app/builds/artifact_index.json - expire_in: 1 week script: - - set_component_ut_vars - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs # clone test script to assign tests # can not retry if downing git lfs files failed, so using empty_branch first. - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only # assign integration test cases - - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH + - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs update_test_cases: extends: .rules:ref:master-schedule @@ -70,7 +150,7 @@ update_test_cases: SUBMODULES_TO_FETCH: "components/esptool_py/esptool" UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test" BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml" - AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script" + PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages PYTHON_VER: 3.7.7 script: - export GIT_SHA=$(echo ${PIPELINE_COMMIT_SHA} | cut -c 1-8) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index db3ca17794..8c2bb7675d 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -61,6 +61,9 @@ patterns: - build_components - build_system + included_in: + - "build:{0}" + - build:target_test build:integration_test: labels: @@ -68,6 +71,8 @@ build:integration_test: patterns: - build_components - build_system + included_in: + - build:target_test #################### # Target Test Jobs # @@ -79,14 +84,14 @@ build:integration_test: labels: # For each rule, use labels and - - "{0}" - "{0}_{1}" + - target_test patterns: # For each rule, use patterns and build- - "{0}" - "build-{0}" included_in: # Parent rules + - "build:{0}" - "build:{0}-{1}" - build:target_test - - test:target_test - - test:any_test # ------------- # Special Cases @@ -99,33 +104,30 @@ build:integration_test: - "component_ut_{0}" - unit_test - "unit_test_{0}" + - target_test patterns: - component_ut - "build-component_ut-{0}" included_in: + - build:component_ut - "build:component_ut-{0}" - build:target_test - - test:target_test - - test:any_test "test:integration_test": labels: - - "integration_test" + - integration_test + - target_test patterns: - - "integration_test" + - integration_test included_in: - - "build:integration_test" + - build:integration_test - build:target_test - - test:target_test - - test:any_test "test:host_test": labels: - host_test patterns: - host_test - included_in: - - test:any_test "test:submodule": labels: @@ -140,17 +142,17 @@ build:integration_test: labels: - iperf_stress_test included_in: + - build:example_test - build:example_test-esp32 - build:target_test - - test:any_test "labels:weekend_test": # custom test labels: - weekend_test included_in: + - build:custom_test - build:custom_test-esp32 - build:target_test - - test:any_test "labels:nvs_coverage": # host_test labels: diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 47f6b138cc..bc7a64e790 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -89,6 +89,7 @@ .patterns-integration_test: &patterns-integration_test - "tools/ci/python_packages/tiny_test_fw/**/*" + - "tools/ci/integration_test/**/*" .patterns-host_test: &patterns-host_test - ".gitlab/ci/host-test.yml" @@ -336,6 +337,9 @@ .if-label-submodule: &if-label-submodule if: '$BOT_LABEL_SUBMODULE || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*submodule(?:,[^,\n\r]+)*$/i' +.if-label-target_test: &if-label-target_test + if: '$BOT_LABEL_TARGET_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i' + .if-label-unit_test: &if-label-unit_test if: '$BOT_LABEL_UNIT_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test(?:,[^,\n\r]+)*$/i' @@ -366,12 +370,35 @@ - <<: *if-dev-push changes: *patterns-build_system +.rules:build:component_ut: + rules: + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-component_ut + - <<: *if-label-component_ut_esp32 + - <<: *if-label-component_ut_esp32c3 + - <<: *if-label-component_ut_esp32s2 + - <<: *if-label-component_ut_esp32s3 + - <<: *if-label-target_test + - <<: *if-label-unit_test + - <<: *if-label-unit_test_esp32 + - <<: *if-label-unit_test_esp32c3 + - <<: *if-label-unit_test_esp32s2 + - <<: *if-label-unit_test_esp32s3 + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-component_ut + .rules:build:component_ut-esp32: rules: - <<: *if-protected - <<: *if-label-build - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32 + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32 - <<: *if-dev-push @@ -387,6 +414,7 @@ - <<: *if-label-build - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32c3 + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32c3 - <<: *if-dev-push @@ -402,6 +430,7 @@ - <<: *if-label-build - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32s2 + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32s2 - <<: *if-dev-push @@ -417,6 +446,7 @@ - <<: *if-label-build - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32s3 + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32s3 - <<: *if-dev-push @@ -426,12 +456,31 @@ - <<: *if-dev-push changes: *patterns-component_ut +.rules:build:custom_test: + rules: + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-custom_test + - <<: *if-label-custom_test_esp32 + - <<: *if-label-custom_test_esp32c3 + - <<: *if-label-custom_test_esp32s2 + - <<: *if-label-custom_test_esp32s3 + - <<: *if-label-target_test + - <<: *if-label-weekend_test + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-custom_test + .rules:build:custom_test-esp32: rules: - <<: *if-protected - <<: *if-label-build - <<: *if-label-custom_test - <<: *if-label-custom_test_esp32 + - <<: *if-label-target_test - <<: *if-label-weekend_test - <<: *if-dev-push changes: *patterns-build_components @@ -446,6 +495,7 @@ - <<: *if-label-build - <<: *if-label-custom_test - <<: *if-label-custom_test_esp32c3 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build_components - <<: *if-dev-push @@ -459,6 +509,7 @@ - <<: *if-label-build - <<: *if-label-custom_test - <<: *if-label-custom_test_esp32s2 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build_components - <<: *if-dev-push @@ -472,6 +523,7 @@ - <<: *if-label-build - <<: *if-label-custom_test - <<: *if-label-custom_test_esp32s3 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build_components - <<: *if-dev-push @@ -487,6 +539,26 @@ - <<: *if-dev-push changes: *patterns-docker +.rules:build:example_test: + rules: + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-example_test + - <<: *if-label-example_test_esp32 + - <<: *if-label-example_test_esp32c3 + - <<: *if-label-example_test_esp32s2 + - <<: *if-label-example_test_esp32s3 + - <<: *if-label-iperf_stress_test + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-build-example_test + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-example_test + .rules:build:example_test-esp32: rules: - <<: *if-protected @@ -494,6 +566,7 @@ - <<: *if-label-example_test - <<: *if-label-example_test_esp32 - <<: *if-label-iperf_stress_test + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build-example_test - <<: *if-dev-push @@ -509,6 +582,7 @@ - <<: *if-label-build - <<: *if-label-example_test - <<: *if-label-example_test_esp32c3 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build-example_test - <<: *if-dev-push @@ -524,6 +598,7 @@ - <<: *if-label-build - <<: *if-label-example_test - <<: *if-label-example_test_esp32s2 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build-example_test - <<: *if-dev-push @@ -539,6 +614,7 @@ - <<: *if-label-build - <<: *if-label-example_test - <<: *if-label-example_test_esp32s3 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build-example_test - <<: *if-dev-push @@ -553,6 +629,7 @@ - <<: *if-protected - <<: *if-label-build - <<: *if-label-integration_test + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build_components - <<: *if-dev-push @@ -590,6 +667,7 @@ - <<: *if-label-example_test_esp32s3 - <<: *if-label-integration_test - <<: *if-label-iperf_stress_test + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32 - <<: *if-label-unit_test_esp32c3 @@ -613,10 +691,28 @@ - <<: *if-dev-push changes: *patterns-unit_test +.rules:build:unit_test: + rules: + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-target_test + - <<: *if-label-unit_test + - <<: *if-label-unit_test_esp32 + - <<: *if-label-unit_test_esp32c3 + - <<: *if-label-unit_test_esp32s2 + - <<: *if-label-unit_test_esp32s3 + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-unit_test + .rules:build:unit_test-esp32: rules: - <<: *if-protected - <<: *if-label-build + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32 - <<: *if-dev-push @@ -630,6 +726,7 @@ rules: - <<: *if-protected - <<: *if-label-build + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32c3 - <<: *if-dev-push @@ -643,6 +740,7 @@ rules: - <<: *if-protected - <<: *if-label-build + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32s2 - <<: *if-dev-push @@ -656,6 +754,7 @@ rules: - <<: *if-protected - <<: *if-label-build + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32s3 - <<: *if-dev-push @@ -692,50 +791,6 @@ rules: - <<: *if-label-weekend_test -.rules:test:any_test: - rules: - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-host_test - - <<: *if-label-integration_test - - <<: *if-label-iperf_stress_test - - <<: *if-label-unit_test - - <<: *if-label-unit_test_esp32 - - <<: *if-label-unit_test_esp32c3 - - <<: *if-label-unit_test_esp32s2 - - <<: *if-label-unit_test_esp32s3 - - <<: *if-label-weekend_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-host_test - - <<: *if-dev-push - changes: *patterns-integration_test - - <<: *if-dev-push - changes: *patterns-unit_test - .rules:test:component_ut-esp32: rules: - <<: *if-protected @@ -743,6 +798,7 @@ when: never - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32 + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32 - <<: *if-dev-push @@ -755,6 +811,7 @@ when: never - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32c3 + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32c3 - <<: *if-dev-push @@ -767,6 +824,7 @@ when: never - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32s2 + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32s2 - <<: *if-dev-push @@ -779,6 +837,7 @@ when: never - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32s3 + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32s3 - <<: *if-dev-push @@ -791,6 +850,7 @@ when: never - <<: *if-label-custom_test - <<: *if-label-custom_test_esp32 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -801,6 +861,7 @@ when: never - <<: *if-label-custom_test - <<: *if-label-custom_test_esp32c3 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -811,6 +872,7 @@ when: never - <<: *if-label-custom_test - <<: *if-label-custom_test_esp32s2 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -821,6 +883,7 @@ when: never - <<: *if-label-custom_test - <<: *if-label-custom_test_esp32s3 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -831,6 +894,7 @@ when: never - <<: *if-label-example_test - <<: *if-label-example_test_esp32 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build-example_test - <<: *if-dev-push @@ -843,6 +907,7 @@ when: never - <<: *if-label-example_test - <<: *if-label-example_test_esp32c3 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build-example_test - <<: *if-dev-push @@ -855,6 +920,7 @@ when: never - <<: *if-label-example_test - <<: *if-label-example_test_esp32s2 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build-example_test - <<: *if-dev-push @@ -867,6 +933,7 @@ when: never - <<: *if-label-example_test - <<: *if-label-example_test_esp32s3 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build-example_test - <<: *if-dev-push @@ -887,6 +954,7 @@ - <<: *if-label-build-only when: never - <<: *if-label-integration_test + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-integration_test @@ -899,50 +967,12 @@ - <<: *if-dev-push changes: *patterns-submodule -.rules:test:target_test: - rules: - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-integration_test - - <<: *if-label-unit_test - - <<: *if-label-unit_test_esp32 - - <<: *if-label-unit_test_esp32c3 - - <<: *if-label-unit_test_esp32s2 - - <<: *if-label-unit_test_esp32s3 - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-integration_test - - <<: *if-dev-push - changes: *patterns-unit_test - .rules:test:unit_test-esp32: rules: - <<: *if-protected - <<: *if-label-build-only when: never + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32 - <<: *if-dev-push @@ -953,6 +983,7 @@ - <<: *if-protected - <<: *if-label-build-only when: never + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32c3 - <<: *if-dev-push @@ -963,6 +994,7 @@ - <<: *if-protected - <<: *if-label-build-only when: never + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32s2 - <<: *if-dev-push @@ -973,6 +1005,7 @@ - <<: *if-protected - <<: *if-label-build-only when: never + - <<: *if-label-target_test - <<: *if-label-unit_test - <<: *if-label-unit_test_esp32s3 - <<: *if-dev-push diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 362569305d..8f5a412ce9 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -7,13 +7,13 @@ .target_test_job_template: stage: target_test - needs: - - assign_test artifacts: when: always paths: - "**/*.log" - $LOG_PATH + exclude: + - .git/**/* expire_in: 1 week reports: junit: $LOG_PATH/*/XUNIT_RESULT.xml @@ -38,6 +38,8 @@ .example_test_template: extends: .target_test_job_template + needs: + - assign_example_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/examples" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" @@ -295,6 +297,8 @@ example_test_ESP32C3_SDSPI: .test_app_template: extends: .target_test_job_template + needs: + - assign_custom_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs" @@ -381,6 +385,8 @@ test_app_test_flash_psram_f8r8: .component_ut_template: extends: .target_test_job_template + needs: # the assign already needs all the build jobs + - assign_component_ut variables: CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs" script: @@ -443,9 +449,11 @@ component_ut_test_esp32c3: .unit_test_template: extends: .target_test_job_template + needs: # the assign already needs all the build jobs + - assign_unit_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app" - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs" + CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/test_configs" .unit_test_esp32_template: extends: @@ -761,18 +769,18 @@ component_ut_test_lan8720: extends: - .target_test_job_template - .rules:test:integration_test - needs: - - assign_test - - build_ssc_esp32 + needs: # the assign already needs all the build jobs + - assign_integration_test variables: LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles" - MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" - KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues" + CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/ci/integration_test/test_configs" + KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/tools/ci/integration_test/KnownIssues" CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py" - PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages + PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py" + PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${PYTHONPATH}" + INITIAL_CONDITION_RETRY_COUNT: "1" # auto_test_script only supports python 3.7.x PYTHON_VER: 3.7.7 script: @@ -789,8 +797,15 @@ component_ut_test_lan8720: - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues # run test + - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH +.integration_test_esp32c3_template: + extends: + - .integration_test_template + variables: + LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C3_IDF" + nvs_compatible_test: extends: .integration_test_template artifacts: @@ -818,6 +833,7 @@ nvs_compatible_test: - cd auto_test_script - ./tools/prepare_nvs_bin.sh # run test + - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH IT_001: @@ -932,3 +948,10 @@ IT_022: tags: - ESP32_IDF - SSC_T3_2 + +IT_C3_001: + extends: .integration_test_esp32c3_template + parallel: 6 + tags: + - ESP32C3_IDF + - SSC_T2_5 diff --git a/tools/ci/checkout_project_ref.py b/tools/ci/checkout_project_ref.py index 975b7e9f29..a64e5f488a 100755 --- a/tools/ci/checkout_project_ref.py +++ b/tools/ci/checkout_project_ref.py @@ -84,11 +84,20 @@ if __name__ == '__main__': ref_to_use = '' for candidate in candidate_branches: - # check if candidate branch exists - branch_match = subprocess.check_output(['git', 'branch', '-a', '--list', 'origin/' + candidate]) - if branch_match: + # check if the branch, tag or commit exists + try: + subprocess.check_call(['git', 'cat-file', '-t', 'origin/{}'.format(candidate)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) ref_to_use = candidate break + except subprocess.CalledProcessError: + try: + # For customized commits + subprocess.check_call(['git', 'cat-file', '-t', candidate], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ref_to_use = candidate + break + except subprocess.CalledProcessError: + pass + continue if ref_to_use: for _ in range(RETRY_COUNT): diff --git a/components/idf_test/integration_test/KnownIssues b/tools/ci/integration_test/KnownIssues similarity index 50% rename from components/idf_test/integration_test/KnownIssues rename to tools/ci/integration_test/KnownIssues index 5b91d4389c..78a03c9df3 100644 --- a/components/idf_test/integration_test/KnownIssues +++ b/tools/ci/integration_test/KnownIssues @@ -1,2 +1,4 @@ # MESH ESP32.MESH_EST_2109 + +ESP32C3.NIMBLE_GAP_14009 diff --git a/tools/ci/integration_test/README.md b/tools/ci/integration_test/README.md new file mode 100644 index 0000000000..16afb7650b --- /dev/null +++ b/tools/ci/integration_test/README.md @@ -0,0 +1,21 @@ +# Integration Test Description + +## Case Lists +- WiFi Standard cases for only esp32. +- BLE Standard cases for esp32 and esp32c3. + +## Trigger +- By labels: + - `integration_test` +- By file changes: + - integration test related files +- By bot: + - `@bot test with label: integration_test` + +## Advanced +- There are labels can be used to run less integration test cases. + - These labels only take effect when the integration test has been triggered. + - label: `integration::wifi_only` + - Only run WiFi cases. + - label: `integration::ble_only` + - Only run BLE cases. diff --git a/tools/ci/integration_test/prepare_test_bins.py b/tools/ci/integration_test/prepare_test_bins.py new file mode 100644 index 0000000000..8e6598a37e --- /dev/null +++ b/tools/ci/integration_test/prepare_test_bins.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os + +import gitlab +import gitlab_api +from AutoTestScript.RunnerConfigs.Config import Config + +SSC_BUILD_JOB_MAP = { + 'ESP32': 'build_ssc_esp32', + 'ESP32C3': 'build_ssc_esp32c3', +} +NEEDED_FILES = [ + 'flasher_args.json', + 'bootloader/bootloader.bin', + 'partition_table/partition-table.bin', + 'ssc.bin', + 'ssc.elf', +] +IDF_PATH = os.environ.get('IDF_PATH') + + +def try_to_download_artifacts(bin_path: str) -> None: + ''' + bin_path: "SSC/ssc_bin/ESP32[C3]/SSC[_APP]" + ''' + project_id = os.getenv('CI_PROJECT_ID') + pipeline_id = os.getenv('CI_PIPELINE_ID') + gitlab_inst = gitlab_api.Gitlab(project_id) + build_job_name = SSC_BUILD_JOB_MAP[bin_path.split('/')[-2]] + job_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) + files_to_download = [os.path.join(bin_path, f) for f in NEEDED_FILES] + for job_info in job_list: + try: + gitlab_inst.download_artifact(job_info['id'], files_to_download, IDF_PATH) + print('Downloaded {} from {}'.format(bin_path, job_info['id'])) + break + except gitlab.exceptions.GitlabError as e: + if e.response_code == 404: + continue + raise + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + 'test_config_file', + help='The test config file to be used.' + ) + args = parser.parse_args() + + configs = Config.parse(args.test_config_file) + test_bin_paths = configs.get_bin_paths() + + for _path in test_bin_paths: + if os.path.exists(_path): + continue + relative_path = os.path.relpath(_path, IDF_PATH) + try_to_download_artifacts(relative_path) + + +if __name__ == '__main__': + main() diff --git a/components/idf_test/integration_test/CIConfigs/nvs_compatible_test_.yml b/tools/ci/integration_test/test_configs/nvs_compatible_test_.yml similarity index 100% rename from components/idf_test/integration_test/CIConfigs/nvs_compatible_test_.yml rename to tools/ci/integration_test/test_configs/nvs_compatible_test_.yml