diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index c03e2cdadd..dfbd66e407 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -1,54 +1,133 @@ -assign_test: - extends: .rules:test:any_test - tags: - - assign_test +.assign_test_template: image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG stage: assign_test - # gitlab ci do not support match job with RegEx or wildcard now in dependencies. - # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage. - dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here. - - build_ssc_esp32 - - build_esp_idf_tests_cmake_esp32 - - build_esp_idf_tests_cmake_esp32s2 - - build_esp_idf_tests_cmake_esp32s3 - - build_esp_idf_tests_cmake_esp32c3 + tags: + - assign_test variables: - SUBMODULES_TO_FETCH: "components/esptool_py/esptool" - EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples" - CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps" - UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test" + SUBMODULES_TO_FETCH: components/esptool_py/esptool + artifacts: + paths: + - ${TEST_DIR}/test_configs + - ${BUILD_DIR}/artifact_index.json + when: always + expire_in: 1 week + script: + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs + +assign_example_test: + extends: + - .assign_test_template + - .rules:build:example_test + needs: + - job: build_examples_cmake_esp32 + artifacts: false + optional: true + - job: build_examples_cmake_esp32s2 + artifacts: false + optional: true + - job: build_examples_cmake_esp32c3 + artifacts: false + optional: true + - job: build_examples_cmake_esp32s3 + artifacts: false + optional: true + variables: + TEST_TYPE: example_test + TEST_DIR: ${CI_PROJECT_DIR}/examples + BUILD_DIR: ${CI_PROJECT_DIR}/build_examples + +assign_custom_test: + extends: + - .assign_test_template + - .rules:build:custom_test + needs: + - job: build_test_apps_esp32 + artifacts: false + optional: true + - job: build_test_apps_esp32s2 + artifacts: false + optional: true + - job: build_test_apps_esp32c3 + artifacts: false + optional: true + - job: build_test_apps_esp32s3 + artifacts: false + optional: true + variables: + TEST_TYPE: custom_test + TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps + BUILD_DIR: ${CI_PROJECT_DIR}/build_test_apps + +assign_component_ut: + extends: + - .assign_test_template + - .rules:build:component_ut + needs: + - job: build_component_ut_esp32 + artifacts: false + optional: true + - job: build_component_ut_esp32s2 + artifacts: false + optional: true + - job: build_component_ut_esp32c3 + artifacts: false + optional: true + - job: build_component_ut_esp32s3 + artifacts: false + optional: true + variables: + TEST_TYPE: component_ut + TEST_DIR: ${CI_PROJECT_DIR}/component_ut + BUILD_DIR: ${CI_PROJECT_DIR}/build_component_ut + script: # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh` - COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut" - INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" + - set_component_ut_vars + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs + +assign_unit_test: + extends: + - .assign_test_template + - .rules:build:unit_test + needs: + - job: build_esp_idf_tests_cmake_esp32 + optional: true + - job: build_esp_idf_tests_cmake_esp32s2 + optional: true + - job: build_esp_idf_tests_cmake_esp32c3 + optional: true + - job: build_esp_idf_tests_cmake_esp32s3 + optional: true + variables: + TEST_TYPE: unit_test + TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/unit_test + BUILD_DIR: ${CI_PROJECT_DIR}/tools/unit-test-app/builds + script: + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs + +assign_integration_test: + extends: + - .assign_test_template + - .rules:test:integration_test + needs: + - build_ssc_esp32 + artifacts: + paths: + - $TEST_DIR/test_configs + variables: + TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/integration_test + BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles" ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages - # auto_test_script only supports python 3.7.x PYTHON_VER: 3.7.7 - artifacts: - paths: - - components/idf_test/*/CIConfigs - - $EXAMPLE_TEST_DIR/test_configs - - $CUSTOM_TEST_DIR/test_configs - - $COMPONENT_UT_OUTPUT_DIR/test_configs - - build_examples/artifact_index.json - - build_test_apps/artifact_index.json - - build_component_ut/artifact_index.json - - tools/unit-test-app/builds/artifact_index.json - expire_in: 1 week script: - - set_component_ut_vars - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs # clone test script to assign tests # can not retry if downing git lfs files failed, so using empty_branch first. - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only # assign integration test cases - - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH + - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs update_test_cases: extends: .rules:ref:master-schedule diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index db3ca17794..b6d8baaabf 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -61,6 +61,8 @@ patterns: - build_components - build_system + included_in: + - "build:{0}" build:integration_test: labels: @@ -83,10 +85,10 @@ build:integration_test: - "{0}" - "build-{0}" included_in: # Parent rules + - "build:{0}" - "build:{0}-{1}" - build:target_test - test:target_test - - test:any_test # ------------- # Special Cases @@ -103,29 +105,26 @@ build:integration_test: - component_ut - "build-component_ut-{0}" included_in: + - build:component_ut - "build:component_ut-{0}" - build:target_test - test:target_test - - test:any_test "test:integration_test": labels: - - "integration_test" + - integration_test patterns: - - "integration_test" + - integration_test included_in: - - "build:integration_test" + - build:integration_test - build:target_test - test:target_test - - test:any_test "test:host_test": labels: - host_test patterns: - host_test - included_in: - - test:any_test "test:submodule": labels: @@ -140,17 +139,17 @@ build:integration_test: labels: - iperf_stress_test included_in: + - build:example_test - build:example_test-esp32 - build:target_test - - test:any_test "labels:weekend_test": # custom test labels: - weekend_test included_in: + - build:custom_test - build:custom_test-esp32 - build:target_test - - test:any_test "labels:nvs_coverage": # host_test labels: diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 47f6b138cc..75a101a35a 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -366,6 +366,27 @@ - <<: *if-dev-push changes: *patterns-build_system +.rules:build:component_ut: + rules: + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-component_ut + - <<: *if-label-component_ut_esp32 + - <<: *if-label-component_ut_esp32c3 + - <<: *if-label-component_ut_esp32s2 + - <<: *if-label-component_ut_esp32s3 + - <<: *if-label-unit_test + - <<: *if-label-unit_test_esp32 + - <<: *if-label-unit_test_esp32c3 + - <<: *if-label-unit_test_esp32s2 + - <<: *if-label-unit_test_esp32s3 + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-component_ut + .rules:build:component_ut-esp32: rules: - <<: *if-protected @@ -426,6 +447,23 @@ - <<: *if-dev-push changes: *patterns-component_ut +.rules:build:custom_test: + rules: + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-custom_test + - <<: *if-label-custom_test_esp32 + - <<: *if-label-custom_test_esp32c3 + - <<: *if-label-custom_test_esp32s2 + - <<: *if-label-custom_test_esp32s3 + - <<: *if-label-weekend_test + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-custom_test + .rules:build:custom_test-esp32: rules: - <<: *if-protected @@ -487,6 +525,25 @@ - <<: *if-dev-push changes: *patterns-docker +.rules:build:example_test: + rules: + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-example_test + - <<: *if-label-example_test_esp32 + - <<: *if-label-example_test_esp32c3 + - <<: *if-label-example_test_esp32s2 + - <<: *if-label-example_test_esp32s3 + - <<: *if-label-iperf_stress_test + - <<: *if-dev-push + changes: *patterns-build-example_test + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-example_test + .rules:build:example_test-esp32: rules: - <<: *if-protected @@ -613,6 +670,22 @@ - <<: *if-dev-push changes: *patterns-unit_test +.rules:build:unit_test: + rules: + - <<: *if-protected + - <<: *if-label-build + - <<: *if-label-unit_test + - <<: *if-label-unit_test_esp32 + - <<: *if-label-unit_test_esp32c3 + - <<: *if-label-unit_test_esp32s2 + - <<: *if-label-unit_test_esp32s3 + - <<: *if-dev-push + changes: *patterns-build_components + - <<: *if-dev-push + changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-unit_test + .rules:build:unit_test-esp32: rules: - <<: *if-protected @@ -692,50 +765,6 @@ rules: - <<: *if-label-weekend_test -.rules:test:any_test: - rules: - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-host_test - - <<: *if-label-integration_test - - <<: *if-label-iperf_stress_test - - <<: *if-label-unit_test - - <<: *if-label-unit_test_esp32 - - <<: *if-label-unit_test_esp32c3 - - <<: *if-label-unit_test_esp32s2 - - <<: *if-label-unit_test_esp32s3 - - <<: *if-label-weekend_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-host_test - - <<: *if-dev-push - changes: *patterns-integration_test - - <<: *if-dev-push - changes: *patterns-unit_test - .rules:test:component_ut-esp32: rules: - <<: *if-protected diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 362569305d..94ab7b9b2c 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -7,8 +7,6 @@ .target_test_job_template: stage: target_test - needs: - - assign_test artifacts: when: always paths: @@ -38,6 +36,8 @@ .example_test_template: extends: .target_test_job_template + needs: + - assign_example_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/examples" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" @@ -295,6 +295,8 @@ example_test_ESP32C3_SDSPI: .test_app_template: extends: .target_test_job_template + needs: + - assign_custom_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs" @@ -381,6 +383,8 @@ test_app_test_flash_psram_f8r8: .component_ut_template: extends: .target_test_job_template + needs: # the assign already needs all the build jobs + - assign_component_ut variables: CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs" script: @@ -443,9 +447,11 @@ component_ut_test_esp32c3: .unit_test_template: extends: .target_test_job_template + needs: # the assign already needs all the build jobs + - assign_unit_test variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app" - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs" + CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/test_configs" .unit_test_esp32_template: extends: @@ -762,14 +768,14 @@ component_ut_test_lan8720: - .target_test_job_template - .rules:test:integration_test needs: - - assign_test + - assign_integration_test - build_ssc_esp32 variables: LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles" MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" + CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/test_configs" KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues" CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages diff --git a/components/idf_test/integration_test/CIConfigs/nvs_compatible_test_.yml b/components/idf_test/integration_test/test_configs/nvs_compatible_test_.yml similarity index 100% rename from components/idf_test/integration_test/CIConfigs/nvs_compatible_test_.yml rename to components/idf_test/integration_test/test_configs/nvs_compatible_test_.yml