diff --git a/.gitignore b/.gitignore index 7aa73b086b..b8a8909c37 100644 --- a/.gitignore +++ b/.gitignore @@ -24,18 +24,6 @@ GPATH # cache dir .cache/ -# Components Unit Test Apps files -components/**/build/ -components/**/build_*_*/ -components/**/sdkconfig -components/**/sdkconfig.old - -# Example project files -examples/**/build/ -examples/**/build_esp*_*/ -examples/**/sdkconfig -examples/**/sdkconfig.old - # Doc build artifacts docs/_build/ docs/doxygen_sqlite3.db @@ -44,16 +32,23 @@ docs/doxygen_sqlite3.db docs/_static/DejaVuSans.ttf docs/_static/NotoSansSC-Regular.otf +# Components Unit Test Apps files +components/**/build/ +components/**/build_*_*/ +components/**/sdkconfig +components/**/sdkconfig.old + +# Example project files +examples/**/build/ +examples/**/build_*_*/ +examples/**/sdkconfig +examples/**/sdkconfig.old + # Unit test app files -tools/unit-test-app/sdkconfig -tools/unit-test-app/sdkconfig.old tools/unit-test-app/build tools/unit-test-app/build_*_*/ -tools/unit-test-app/output -tools/unit-test-app/test_configs - -# Unit Test CMake compile log folder -log_ut_cmake +tools/unit-test-app/sdkconfig +tools/unit-test-app/sdkconfig.old # test application build files tools/test_apps/**/build/ @@ -61,7 +56,8 @@ tools/test_apps/**/build_*_*/ tools/test_apps/**/sdkconfig tools/test_apps/**/sdkconfig.old -TEST_LOGS +TEST_LOGS/ +build_summary_*.xml # gcov coverage reports *.gcda @@ -101,8 +97,9 @@ managed_components # pytest log pytest_embedded_log/ -list_job_*.txt -size_info.txt +list_job*.txt +size_info*.txt +XUNIT_RESULT*.xml # clang config (for LSP) .clangd diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8f4c3eadf..87487c7620 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,5 +28,4 @@ include: - '.gitlab/ci/build.yml' - '.gitlab/ci/integration_test.yml' - '.gitlab/ci/host-test.yml' - - '.gitlab/ci/target-test.yml' - '.gitlab/ci/deploy.yml' diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 64f4020e56..798271d94f 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -1,7 +1,7 @@ .build_template: stage: build extends: - - .after_script:build:ccache + - .after_script:build:ccache:upload-when-fail image: $ESP_ENV_IMAGE tags: - build @@ -32,8 +32,6 @@ # keep the size info to help track the binary size - size_info.txt - "**/build*/size.json" - when: always - expire_in: 4 days script: # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v @@ -45,8 +43,8 @@ examples/bluetooth/esp_ble_mesh/ble_mesh_console examples/bluetooth/hci/controller_hci_uart_esp32 examples/wifi/iperf - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} # for detailed documents, please refer to .gitlab/ci/README.md#uploaddownload-artifacts-to-internal-minio-server - python tools/ci/artifacts_handler.py upload @@ -64,307 +62,14 @@ --copy-sdkconfig --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} $TEST_BUILD_OPTS_EXTRA - python tools/ci/artifacts_handler.py upload -.build_pytest_template: - extends: - - .build_cmake_template - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - -m \"not host_test\" - --pytest-apps - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - -.build_pytest_no_jtag_template: - extends: - - .build_cmake_template - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - -m \"not host_test and not jtag\" - --pytest-apps - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - -.build_pytest_jtag_template: - extends: - - .build_cmake_template - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - -m \"not host_test and jtag\" - --pytest-apps - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - -build_pytest_examples_esp32: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32 - parallel: 6 - variables: - IDF_TARGET: esp32 - TEST_DIR: examples - -build_pytest_examples_esp32s2: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32s2 - parallel: 3 - variables: - IDF_TARGET: esp32s2 - TEST_DIR: examples - -build_pytest_examples_esp32s3: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32s3 - parallel: 4 - variables: - IDF_TARGET: esp32s3 - TEST_DIR: examples - -build_pytest_examples_esp32c3: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32c3 - parallel: 4 - variables: - IDF_TARGET: esp32c3 - TEST_DIR: examples - -build_pytest_examples_esp32c2: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32c2 - parallel: 2 - variables: - IDF_TARGET: esp32c2 - TEST_DIR: examples - -build_pytest_examples_esp32c6: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32c6 - parallel: 2 - variables: - IDF_TARGET: esp32c6 - TEST_DIR: examples - -build_pytest_examples_esp32h2: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32h2 - parallel: 2 - variables: - IDF_TARGET: esp32h2 - TEST_DIR: examples - -build_pytest_examples_esp32p4: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32p4 - parallel: 2 - variables: - IDF_TARGET: esp32p4 - TEST_DIR: examples - -build_pytest_examples_jtag: # for all targets - extends: - - .build_pytest_jtag_template - - .rules:build:example_test - variables: - IDF_TARGET: all - TEST_DIR: examples - -build_pytest_components_esp32: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32 - parallel: 5 - variables: - IDF_TARGET: esp32 - TEST_DIR: components - -build_pytest_components_esp32s2: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32s2 - parallel: 4 - variables: - IDF_TARGET: esp32s2 - TEST_DIR: components - -build_pytest_components_esp32s3: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32s3 - parallel: 4 - variables: - IDF_TARGET: esp32s3 - TEST_DIR: components - -build_pytest_components_esp32c3: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32c3 - parallel: 4 - variables: - IDF_TARGET: esp32c3 - TEST_DIR: components - -build_pytest_components_esp32c2: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32c2 - parallel: 3 - variables: - IDF_TARGET: esp32c2 - TEST_DIR: components - -build_pytest_components_esp32c6: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32c6 - parallel: 3 - variables: - IDF_TARGET: esp32c6 - TEST_DIR: components - -build_pytest_components_esp32h2: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32h2 - parallel: 4 - variables: - IDF_TARGET: esp32h2 - TEST_DIR: components - -build_pytest_components_esp32p4: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32p4 - parallel: 4 - variables: - IDF_TARGET: esp32p4 - TEST_DIR: components - -build_only_components_apps: - extends: - - .build_cmake_template - - .rules:build:component_ut - parallel: 5 - script: - - set_component_ut_vars - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v - -t all - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - -build_pytest_test_apps_esp32: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32 - variables: - IDF_TARGET: esp32 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32s2: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32s2 - variables: - IDF_TARGET: esp32s2 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32s3: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32s3 - parallel: 2 - variables: - IDF_TARGET: esp32s3 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32c3: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32c3 - variables: - IDF_TARGET: esp32c3 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32c2: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32c2 - variables: - IDF_TARGET: esp32c2 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32c6: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32c6 - variables: - IDF_TARGET: esp32c6 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32h2: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32h2 - variables: - IDF_TARGET: esp32h2 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32p4: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32p4 - variables: - IDF_TARGET: esp32p4 - TEST_DIR: tools/test_apps - -build_only_tools_test_apps: - extends: - - .build_cmake_template - - .rules:build:custom_test - parallel: 9 - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py tools/test_apps -v - -t all - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - +###################### +# build_template_app # +###################### .build_template_app_template: extends: - .build_template @@ -376,12 +81,10 @@ build_only_tools_test_apps: BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt" BUILD_COMMAND_ARGS: "" artifacts: - when: always paths: - log_template_app/* - size_info.txt - build_template_app/**/size.json - expire_in: 1 week script: # Set the variable for 'esp-idf-template' testing - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} @@ -404,96 +107,27 @@ fast_template_app: BUILD_COMMAND_ARGS: "-p" #------------------------------------------------------------------------------ -build_examples_cmake_esp32: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32 - parallel: 8 - variables: - IDF_TARGET: esp32 - TEST_DIR: examples - -build_examples_cmake_esp32s2: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32s2 - parallel: 7 - variables: - IDF_TARGET: esp32s2 - TEST_DIR: examples - -build_examples_cmake_esp32s3: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32s3 - parallel: 11 - variables: - IDF_TARGET: esp32s3 - TEST_DIR: examples - -build_examples_cmake_esp32c2: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32c2 - parallel: 7 - variables: - IDF_TARGET: esp32c2 - TEST_DIR: examples - -build_examples_cmake_esp32c3: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32c3 - parallel: 9 - variables: - IDF_TARGET: esp32c3 - TEST_DIR: examples - -build_examples_cmake_esp32c6: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32c6 - parallel: 11 - variables: - IDF_TARGET: esp32c6 - TEST_DIR: examples - -build_examples_cmake_esp32h2: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32h2 - parallel: 9 - variables: - IDF_TARGET: esp32h2 - TEST_DIR: examples - -build_examples_cmake_esp32p4: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32p4 - parallel: 4 - variables: - IDF_TARGET: esp32p4 - TEST_DIR: examples - +######################################## +# Clang Build Apps Without Tests Cases # +######################################## build_clang_test_apps_esp32: extends: - .build_cmake_clang_template - - .rules:build:custom_test-esp32 + - .rules:build variables: IDF_TARGET: esp32 build_clang_test_apps_esp32s2: extends: - .build_cmake_clang_template - - .rules:build:custom_test-esp32s2 + - .rules:build variables: IDF_TARGET: esp32s2 build_clang_test_apps_esp32s3: extends: - .build_cmake_clang_template - - .rules:build:custom_test-esp32s3 + - .rules:build variables: IDF_TARGET: esp32s3 @@ -510,26 +144,29 @@ build_clang_test_apps_esp32s3: build_clang_test_apps_esp32c3: extends: - .build_clang_test_apps_riscv - - .rules:build:custom_test-esp32c3 + - .rules:build variables: IDF_TARGET: esp32c3 build_clang_test_apps_esp32c2: extends: - .build_clang_test_apps_riscv - - .rules:build:custom_test-esp32c2 + - .rules:build variables: IDF_TARGET: esp32c2 build_clang_test_apps_esp32c6: extends: - .build_clang_test_apps_riscv - - .rules:build:custom_test-esp32c6 + - .rules:build # TODO: c6 builds fail in master due to missing headers allow_failure: true variables: IDF_TARGET: esp32c6 +###################### +# Build System Tests # +###################### .test_build_system_template: stage: host_test extends: @@ -554,7 +191,6 @@ pytest_build_system: paths: - XUNIT_RESULT.xml - test_build_system - when: always expire_in: 2 days reports: junit: XUNIT_RESULT.xml @@ -571,7 +207,6 @@ pytest_build_system_macos: paths: - XUNIT_RESULT.xml - test_build_system - when: always expire_in: 2 days reports: junit: XUNIT_RESULT.xml @@ -603,7 +238,6 @@ pytest_build_system_win: paths: - XUNIT_RESULT.xml - test_build_system - when: always expire_in: 2 days reports: junit: XUNIT_RESULT.xml @@ -641,3 +275,44 @@ build_template_app: needs: - job: fast_template_app artifacts: false + +#################### +# Dynamic Pipeline # +#################### +generate_build_child_pipeline: + extends: + - .build_template + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - pipeline_variables + artifacts: + paths: + - build_child_pipeline.yml + - test_related_apps.txt + - non_test_related_apps.txt + script: + - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} + +build_child_pipeline: + stage: build + needs: + - job: fast_template_app + artifacts: false + - pipeline_variables + - generate_build_child_pipeline + variables: + IS_MR_PIPELINE: $IS_MR_PIPELINE + MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS + MR_MODIFIED_FILES: $MR_MODIFIED_FILES + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS + # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 + inherit: + variables: false + trigger: + include: + - artifact: build_child_pipeline.yml + job: generate_build_child_pipeline + strategy: depend diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 7323505d8d..32beb8b84e 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -85,6 +85,7 @@ variables: ################################################ .common_before_scripts: &common-before_scripts | source tools/ci/utils.sh + is_based_on_commits $REQUIRED_ANCESTOR_COMMITS if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then @@ -208,6 +209,10 @@ variables: - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} .after_script:build:ccache: + after_script: + - *show_ccache_statistics + +.after_script:build:ccache:upload-when-fail: after_script: - *show_ccache_statistics - *upload_failed_job_log_artifacts @@ -345,6 +350,9 @@ default: - *setup_tools_and_idf_python_venv - add_gitlab_ssh_keys - fetch_submodules + artifacts: + expire_in: 1 week + when: always retry: max: 2 when: diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 7ab9e4a0ef..4f87a9ab3b 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -66,104 +66,6 @@ included_in: - build:check -# --------------- -# Build Test Jobs -# --------------- -"build:{0}-{1}": - matrix: - - *target_test - - *all_targets - labels: - - build - patterns: - - build_components - - build_system - - build_target_test - - downloadable-tools - included_in: - - "build:{0}" - - build:target_test - -#################### -# Target Test Jobs # -#################### -"test:{0}-{1}": - matrix: - - *target_test - - *all_targets - labels: # For each rule, use labels and - - - "{0}" - - "{0}_{1}" - - target_test - patterns: # For each rule, use patterns and build- - - "{0}" - - "build-{0}" - included_in: # Parent rules - - "build:{0}" - - "build:{0}-{1}" - - build:target_test - -# ------------- -# Special Cases -# ------------- - -# To reduce the specific runners' usage. -# Do not create these jobs by default patterns on development branches -# Can be triggered by labels or related changes -"test:{0}-{1}-{2}": - matrix: - - *target_test - - *all_targets - - - wifi # pytest*wifi* - - ethernet # pytest*ethernet* - - sdio # pytest*sdio* - - usb # USB Device & Host tests - - adc # pytest*adc* - - i154 - - flash_multi - - ecdsa - - nvs_encr_hmac - patterns: - - "{0}-{1}-{2}" - - "{0}-{2}" - - "target_test-{2}" - labels: - - "{0}_{1}" - - "{0}" - - target_test - included_in: - - "build:{0}-{1}" - - "build:{0}" - - build:target_test - -# For example_test*flash_encryption_wifi_high_traffic jobs -# set `INCLUDE_NIGHTLY_RUN` variable when triggered on development branches -"test:example_test-{0}-include_nightly_run-rule": - matrix: - - - esp32 - - esp32c3 - specific_rules: - - "if-example_test-ota-include_nightly_run-rule" - included_in: - - "build:example_test-{0}" - - "build:example_test" - - build:target_test - -# For i154 runners -"test:example_test-i154": - patterns: - - "example_test-i154" - - "target_test-i154" - labels: - - target_test - - example_test - included_in: - - "build:example_test-esp32s3" - - "build:example_test-esp32c6" - - "build:example_test-esp32h2" - - "build:example_test" - - build:target_test - "test:host_test": labels: - host_test diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 6e77737a36..809b1612d7 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -121,7 +121,6 @@ build_docs_html_full: artifacts: false optional: true artifacts: - when: always paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* @@ -135,7 +134,6 @@ build_docs_html_full_prod: - .doc-rules:build:docs-full-prod dependencies: [] # Stop build_docs jobs from downloading all previous job's artifacts artifacts: - when: always paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* @@ -152,7 +150,6 @@ build_docs_html_partial: artifacts: false optional: true artifacts: - when: always paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* @@ -175,7 +172,6 @@ build_docs_pdf: artifacts: false optional: true artifacts: - when: always paths: - docs/_build/*/*/latex/* expire_in: 4 days @@ -188,7 +184,6 @@ build_docs_pdf_prod: - .doc-rules:build:docs-full-prod dependencies: [] # Stop build_docs jobs from downloading all previous job's artifacts artifacts: - when: always paths: - docs/_build/*/*/latex/* expire_in: 4 days @@ -266,11 +261,9 @@ check_doc_links: artifacts: false tags: ["build", "amd64", "internet"] artifacts: - when: always paths: - docs/_build/*/*/*.txt - docs/_build/*/*/linkcheck/*.txt - expire_in: 1 week allow_failure: true script: - cd docs diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index ab1d62e588..ed71ead85c 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -28,7 +28,6 @@ test_nvs_coverage: artifacts: paths: - components/nvs_flash/test_nvs_host/coverage_report - expire_in: 1 week script: - cd components/nvs_flash/test_nvs_host - make coverage_report @@ -65,7 +64,6 @@ test_reproducible_build: - "**/build*/*.bin" - "**/build*/bootloader/*.bin" - "**/build*/partition_table/*.bin" - expire_in: 1 week test_spiffs_on_host: extends: .host_test_template @@ -110,7 +108,6 @@ test_cli_installer: paths: - tools/tools.new.json - tools/test_idf_tools/test_python_env_logs.txt - expire_in: 1 week image: name: $ESP_ENV_IMAGE entrypoint: [""] # use system python3. no extra pip package installed @@ -130,7 +127,6 @@ test_cli_installer: when: on_failure paths: - components/efuse/${IDF_TARGET}/esp_efuse_table.c - expire_in: 1 week script: - cd ${IDF_PATH}/components/efuse/ - ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv @@ -173,7 +169,6 @@ test_logtrace_proc: paths: - tools/esp_app_trace/test/logtrace/output - tools/esp_app_trace/test/logtrace/.coverage - expire_in: 1 week script: - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace - ./test.sh @@ -185,7 +180,6 @@ test_sysviewtrace_proc: paths: - tools/esp_app_trace/test/sysview/output - tools/esp_app_trace/test/sysview/.coverage - expire_in: 1 week script: - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview - ./test.sh @@ -194,13 +188,11 @@ test_tools: extends: - .host_test_template artifacts: - when: always paths: - ${IDF_PATH}/*.out - ${IDF_PATH}/XUNIT_*.xml reports: junit: ${IDF_PATH}/XUNIT_*.xml - expire_in: 1 week variables: LC_ALL: C.UTF-8 INSTALL_QEMU: 1 # for test_idf_qemu.py @@ -280,13 +272,11 @@ test_pytest_qemu: - .host_test_template - .before_script:build artifacts: - when: always paths: - XUNIT_RESULT.xml - pytest_embedded_log/ reports: junit: XUNIT_RESULT.xml - expire_in: 1 week allow_failure: true # IDFCI-1752 parallel: matrix: @@ -299,8 +289,8 @@ test_pytest_qemu: --pytest-apps -m qemu --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - run_cmd pytest --target $IDF_TARGET @@ -316,22 +306,20 @@ test_pytest_linux: - .host_test_template - .before_script:build artifacts: - when: always paths: - XUNIT_RESULT.xml - pytest_embedded_log/ - "**/build*/build_log.txt" reports: junit: XUNIT_RESULT.xml - expire_in: 1 week script: - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv --target linux --pytest-apps -m host_test --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - run_cmd pytest --target linux @@ -339,3 +327,16 @@ test_pytest_linux: --junitxml=XUNIT_RESULT.xml --ignore-result-files known_failure_cases/known_failure_cases.txt --app-info-filepattern \"list_job_*.txt\" + +test_idf_pytest_plugin: + extends: + - .host_test_template + - .rules:patterns:idf-pytest-plugin + variables: + SUBMODULES_TO_FETCH: "none" + artifacts: + reports: + junit: XUNIT_RESULT.xml + script: + - cd tools/ci/idf_pytest + - pytest --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 6ff57bfc86..4282bf7997 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -84,7 +84,6 @@ check_chip_support_components: paths: - esp_hw_support_part.h - bootloader_support_part.h - expire_in: 1 week script: - python tools/ci/check_soc_headers_leak.py - find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py @@ -98,7 +97,6 @@ check_esp_err_to_name: when: on_failure paths: - components/esp_common/esp_err_to_name.c - expire_in: 1 week script: - cd ${IDF_PATH}/tools/ - ./gen_esp_err_to_name.py @@ -122,12 +120,6 @@ check_version_tag: script: - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1) -check_artifacts_expire_time: - extends: .pre_check_template - script: - # check if we have set expire time for all artifacts - - python tools/ci/check_artifacts_expire_time.py - check_test_scripts_build_test_rules: extends: - .pre_check_template @@ -153,9 +145,22 @@ pipeline_variables: tags: - build script: - - MODIFIED_FILES=$(echo $GIT_DIFF_OUTPUT | xargs) + # MODIFIED_FILES is a list of files that changed, could be used everywhere + - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env - - echo "MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | xargs)" >> pipeline.env + # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only + # for non MR pipeline, these are empty lists + - | + if [ $IS_MR_PIPELINE == "0" ]; then + echo "MR_MODIFIED_FILES=" >> pipeline.env + echo "MR_MODIFIED_COMPONENTS=" >> pipeline.env + else + MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';') + echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env + + MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';') + echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env + fi - | if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env @@ -165,4 +170,3 @@ pipeline_variables: artifacts: reports: dotenv: pipeline.env - expire_in: 4 days diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index f6d5b68d69..6132556695 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -23,19 +23,6 @@ .patterns-sonarqube-files: &patterns-sonarqube-files - "tools/ci/sonar_exclude_list.txt" -.patterns-example_test: &patterns-example_test - - "tools/ci/idf_pytest/**/*" - - "tools/ci/python_packages/gitlab_api.py" - - "tools/ci/python_packages/idf_http_server_test/**/*" - - "tools/ci/python_packages/idf_iperf_test_util/**/*" - - "tools/ci/python_packages/common_test_methods.py" - - - "tools/esp_prov/**/*" - - "examples/**/*" - -.patterns-build-example_test: &patterns-build-example_test - - "tools/ci/get_supported_examples.sh" - .patterns-build_components: &patterns-build_components # components files except "test*/" "host*/" folders # ?? to include folders less than 4 characters @@ -67,48 +54,6 @@ - "tools/ci/ci_build_apps.py" - "tools/test_build_system/**/*" -.patterns-custom_test: &patterns-custom_test - - "tools/ci/idf_pytest/**/*" - - "tools/ci/python_packages/gitlab_api.py" - - "tools/ci/python_packages/common_test_methods.py" - - - "tools/test_apps/**/*" - - "tools/ldgen/**/*" - -.patterns-component_ut: &patterns-component_ut - - "tools/ci/idf_pytest/**/*" - - "tools/ci/python_packages/gitlab_api.py" - - "tools/ci/python_packages/common_test_methods.py" - - "tools/test_apps/configs/sdkconfig.debug_helpers" - - - "components/**/*" - -.patterns-component_ut-i154: &patterns-component_ut-i154 - - "components/{esp_phy,esp_coex}/???[!t]*/**/*" - - "components/{esp_phy,esp_coex}/??[!s]?*/**/*" - - "components/{esp_phy,esp_coex}/???/**/*" - - "components/{esp_phy,esp_coex}/*" - - "components/ieee802154/**/*" - -.patterns-example_test-i154: &patterns-example_test-i154 - - "components/{esp_netif,ieee802154,lwip,openthread}/???[!t]*/**/*" - - "components/{esp_netif,ieee802154,lwip,openthread}/??[!s]?*/**/*" - - "components/{esp_netif,ieee802154,lwip,openthread}/???/**/*" - - "components/{esp_netif,ieee802154,lwip,openthread}/??/**/*" - - "components/{esp_netif,ieee802154,lwip,openthread}/*" - - "examples/common_components/iperf/**/*" - - "examples/openthread/**/*" - -.patterns-target_test-wifi: &patterns-target_test-wifi - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/???[!t]*/**/*" - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/??[!s]?*/**/*" - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/???/**/*" - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/??/**/*" - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/*" - -.patterns-component_ut-wifi: &patterns-component_ut-wifi - - "components/esp_wifi/**/*" - .patterns-build_macos: &patterns-build_macos - "tools/ci/test_configure_ci_environment.sh" @@ -197,127 +142,8 @@ - "components/bt/esp_ble_mesh/lib/lib" - ".gitmodules" -# for jobs: flash_encryption_wifi_high_traffic -.patterns-example_test-ota-nightly_run: &patterns-example_test-ota-nightly_run - - "examples/system/ota/**/*" - - "examples/common_components/protocol_examples_common/**/*" - - "components/app_update/include/*" - - "components/app_update/*" - - "components/esp_https_ota/**/*" - -# for jobs: example_test*ethernet* -.patterns-example_test-ethernet: &patterns-example_test-ethernet - - "tools/ci/python_packages/common_test_methods.py" - - "examples/common_components/**/*" - - "examples/protocols/**/*" - - "examples/system/ota/**/*" - - "examples/ethernet/iperf/**/*" - - "examples/network/vlan_support/**/*" - - "components/esp_eth/??[!s][!t]*/**/*" - - "components/esp_eth/???/**/*" - - "components/esp_eth/*" - - "components/esp_netif/esp_netif_handlers.c" - -# for jobs: example_test*wifi* -.patterns-example_test-wifi: &patterns-example_test-wifi - - "tools/ci/python_packages/common_test_methods.py" - - "examples/common_components/protocol_examples_common/**/*" - - "examples/protocols/**/*" - - "examples/wifi/**/*" - - "examples/network/simple_sniffer/**/*" - - - "components/mbedtls/port/dynamic/*" - # ota - - "examples/system/ota/**/*" - - "components/app_update/include/*" - - "components/app_update/*" - - "components/esp_https_ota/**/*" - -# for jobs: custom_test*wifi* -.patterns-custom_test-wifi: &patterns-custom_test-wifi - - "tools/test_apps/phy/**/*" - -# for jobs: SDIO related example_test -.patterns-example_test-sdio: &patterns-example_test-sdio - - "components/hal/sdio*.c" - - "components/hal/include/hal/sdio*.h" - - "components/esp_driver_sdspi/src/sdspi*.c" - - "components/esp_driver_sdio/src/sdio*.c" - - "components/esp_driver_sdmmc/src/sdmmc*.c" - - "components/esp_driver_sdspi/include/driver/sdspi*.h" - - "components/esp_driver_sdio/include/driver/sdio*.h" - - "components/esp_driver_sdmmc/include/driver/sdmmc*.h" - - "components/sdmmc/??[!s][!t]*/**/*" - - "components/sdmmc/???/**/*" - - "components/sdmmc/*" - - "examples/peripherals/sdio/**/*" - -# for jobs: SDIO related component_test -.patterns-component_ut-sdio: &patterns-component_ut-sdio - - "components/hal/sdio*.c" - - "components/hal/include/hal/sdio*.h" - - "components/esp_driver_sdspi/src/sdspi*.c" - - "components/esp_driver_sdio/src/sdio*.c" - - "components/esp_driver_sdmmc/src/sdmmc*.c" - - "components/esp_driver_sdspi/include/driver/sdspi*.h" - - "components/esp_driver_sdio/include/driver/sdio*.h" - - "components/esp_driver_sdmmc/include/driver/sdmmc*.h" - - "components/sdmmc/**/*" - -# for jobs: component_ut_pytest_flash_multi -.patterns-component_ut-flash_multi: &patterns-component_ut-flash_multi - - "components/spi_flash/**/*" - - "components/hal/spi_flash*.c" - - "components/hal/include/hal/spi_flash*.h" - -# for jobs: USB host and device examples -.patterns-example_test-usb: &patterns-example_test-usb - - "components/hal/usb*.c" - - "components/hal/esp32s*/include/hal/usb*.h" - - "components/usb/*include/**/*" - - "components/usb/*" - - "examples/peripherals/usb/host/**/**/**/*" - - "examples/peripherals/usb/device/**/**/*" - -# for jobs: USB component (Host) pytest test_app -.patterns-component_ut-usb: &patterns-component_ut-usb - - "components/hal/usb*.c" - - "components/hal/esp32s*/include/hal/usb*.h" - - "components/usb/**/*" - -# for jobs: *_pytest_esp32x_adc: -.patterns-target_test-adc: &patterns-target_test-adc - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/???[!t]*/**/*" - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/??[!s]?*/**/*" - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/???/**/*" - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/??/**/*" - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/*" - -.patterns-component_ut-adc: &patterns-component_ut-adc - - "components/esp_adc/**/*" - -.patterns-example_test-adc: &patterns-example_test-adc - - "examples/peripherals/adc/**/*" - -.patterns-target_test-ecdsa: &patterns-target_test-ecdsa - - "components/{hal,efuse}/???[!t]*/**/*" - - "components/{hal,efuse}/??[!s]?*/**/*" - - "components/{hal,efuse}/???/**/*" - - "components/{hal,efuse}/??/**/*" - - "components/{hal,efuse}/*" - - "components/mbedtls/port/ecdsa/*" - -.patterns-component_ut-nvs_encr_hmac: &patterns-component_ut-nvs_encr_hmac - - "components/nvs_flash/**/*" - - "components/nvs_sec_provider/**/*" - -.patterns-example_test-nvs_encr_hmac: &patterns-example_test-nvs_encr_hmac - - "components/{nvs_flash,nvs_sec_provider}/???[!t]*/**/*" - - "components/{nvs_flash,nvs_sec_provider}/??[!s]?*/**/*" - - "components/{nvs_flash,nvs_sec_provider}/???/**/*" - - "components/{nvs_flash,nvs_sec_provider}/??/**/*" - - "components/{nvs_flash,nvs_sec_provider}/*" - - "examples/security/nvs_encryption_hmac/**/*" +.patterns-idf-pytest-plugin: &patterns-idf-pytest-plugin + - "tools/ci/idf_pytest/**/*" ############## # if anchors # @@ -344,9 +170,6 @@ .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' -.if-merge_request: &if-merge_request - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - .if-schedule: &if-schedule if: '$CI_PIPELINE_SOURCE == "schedule"' @@ -356,9 +179,6 @@ .if-schedule-test-build-system-windows: &if-schedule-test-build-system-windows if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_BUILD_SYSTEM_TEST_WIN == "true"' -.if-trigger: &if-trigger - if: '$CI_PIPELINE_SOURCE == "trigger"' - .if-label-build-only: &if-label-build-only if: '$CI_JOB_STAGE == "target_test" && $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*For Maintainers: Only Build Tests(?:,[^,\n\r]+)*$/i' @@ -368,15 +188,6 @@ .if-revert-branch: &if-revert-branch if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^revert-/' -##################### -# Specific if rules # -##################### -.if-example_test-ota-include_nightly_run-rule: &if-example_test-ota-include_nightly_run-rule - <<: *if-dev-push - changes: *patterns-example_test-ota-nightly_run - variables: - INCLUDE_NIGHTLY_RUN: "1" - ######### # Rules # ######### @@ -398,10 +209,6 @@ - <<: *if-protected-no_label when: always -.rules:mr: - rules: - - <<: *if-merge_request - .rules:tag:release: rules: - <<: *if-tag-release @@ -447,6 +254,12 @@ - <<: *if-dev-push changes: *patterns-sonarqube-files +.rules:patterns:idf-pytest-plugin: + rules: + - <<: *if-protected + - <<: *if-dev-push + changes: *patterns-idf-pytest-plugin + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # DO NOT place comments or maintain any code from this line # @@ -462,99 +275,9 @@ .if-label-build: &if-label-build if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i' -.if-label-component_ut: &if-label-component_ut - if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32: &if-label-component_ut_esp32 - if: '$BOT_LABEL_COMPONENT_UT_ESP32 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32c2: &if-label-component_ut_esp32c2 - if: '$BOT_LABEL_COMPONENT_UT_ESP32C2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32c2(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32c3: &if-label-component_ut_esp32c3 - if: '$BOT_LABEL_COMPONENT_UT_ESP32C3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32c3(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32c5: &if-label-component_ut_esp32c5 - if: '$BOT_LABEL_COMPONENT_UT_ESP32C5 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32c5(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32c6: &if-label-component_ut_esp32c6 - if: '$BOT_LABEL_COMPONENT_UT_ESP32C6 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32c6(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32h2: &if-label-component_ut_esp32h2 - if: '$BOT_LABEL_COMPONENT_UT_ESP32H2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32h2(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32p4: &if-label-component_ut_esp32p4 - if: '$BOT_LABEL_COMPONENT_UT_ESP32P4 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32p4(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32s2: &if-label-component_ut_esp32s2 - if: '$BOT_LABEL_COMPONENT_UT_ESP32S2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32s2(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32s3: &if-label-component_ut_esp32s3 - if: '$BOT_LABEL_COMPONENT_UT_ESP32S3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32s3(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test: &if-label-custom_test - if: '$BOT_LABEL_CUSTOM_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32: &if-label-custom_test_esp32 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32c2: &if-label-custom_test_esp32c2 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32C2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32c2(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32c3: &if-label-custom_test_esp32c3 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32C3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32c3(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32c5: &if-label-custom_test_esp32c5 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32C5 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32c5(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32c6: &if-label-custom_test_esp32c6 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32C6 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32c6(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32h2: &if-label-custom_test_esp32h2 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32H2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32h2(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32p4: &if-label-custom_test_esp32p4 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32P4 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32p4(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32s2: &if-label-custom_test_esp32s2 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32S2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32s2(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32s3: &if-label-custom_test_esp32s3 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32S3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32s3(?:,[^,\n\r]+)*$/i' - .if-label-docker: &if-label-docker if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i' -.if-label-example_test: &if-label-example_test - if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32: &if-label-example_test_esp32 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32c2: &if-label-example_test_esp32c2 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32C2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32c2(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32c3: &if-label-example_test_esp32c3 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32C3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32c3(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32c5: &if-label-example_test_esp32c5 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32C5 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32c5(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32c6: &if-label-example_test_esp32c6 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32C6 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32c6(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32h2: &if-label-example_test_esp32h2 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32H2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32h2(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32p4: &if-label-example_test_esp32p4 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32P4 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32p4(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32s2: &if-label-example_test_esp32s2 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32S2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32s2(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32s3: &if-label-example_test_esp32s3 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32S3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32s3(?:,[^,\n\r]+)*$/i' - .if-label-host_test: &if-label-host_test if: '$BOT_LABEL_HOST_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*host_test(?:,[^,\n\r]+)*$/i' @@ -570,9 +293,6 @@ .if-label-submodule: &if-label-submodule if: '$BOT_LABEL_SUBMODULE || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*submodule(?:,[^,\n\r]+)*$/i' -.if-label-target_test: &if-label-target_test - if: '$BOT_LABEL_TARGET_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i' - .if-label-windows: &if-label-windows if: '$BOT_LABEL_WINDOWS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*windows(?:,[^,\n\r]+)*$/i' @@ -604,598 +324,6 @@ - <<: *if-dev-push changes: *patterns-downloadable-tools -.rules:build:component_ut: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-component_ut_esp32c5 - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-component_ut_esp32p4 - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32c5 - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-custom_test_esp32p4 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:build:docker: rules: - <<: *if-revert-branch @@ -1212,377 +340,6 @@ - <<: *if-dev-push changes: *patterns-submodule -.rules:build:example_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-example_test-ota-include_nightly_run-rule - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-example_test_esp32c5 - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-example_test_esp32p4 - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-example_test-ota-include_nightly_run-rule - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-example_test-ota-include_nightly_run-rule - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:build:macos: rules: - <<: *if-revert-branch @@ -1603,91 +360,15 @@ - <<: *if-revert-branch when: never - <<: *if-protected - - <<: *if-example_test-ota-include_nightly_run-rule - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-component_ut_esp32c5 - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-component_ut_esp32p4 - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32c5 - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-custom_test_esp32p4 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-example_test_esp32c5 - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-example_test_esp32p4 - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - <<: *if-dev-push changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-build_template-app - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi .rules:labels:nvs_coverage: rules: @@ -1702,977 +383,6 @@ - <<: *if-schedule-test-build-system-windows - <<: *if-label-windows -.rules:test:component_ut-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:component_ut-esp32-flash_multi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - -.rules:test:component_ut-esp32-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - -.rules:test:component_ut-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32c2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:component_ut-esp32c2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:component_ut-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32c3-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:component_ut-esp32c3-flash_multi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - -.rules:test:component_ut-esp32c3-nvs_encr_hmac: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - -.rules:test:component_ut-esp32c3-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - -.rules:test:component_ut-esp32c3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:component_ut-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32c6-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:component_ut-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32h2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:component_ut-esp32h2-ecdsa: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - -.rules:test:component_ut-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32s2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:component_ut-esp32s2-flash_multi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - -.rules:test:component_ut-esp32s2-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - -.rules:test:component_ut-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32s3-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:component_ut-esp32s3-flash_multi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - -.rules:test:component_ut-esp32s3-usb: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-usb - -.rules:test:component_ut-esp32s3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:custom_test-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32c2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:custom_test-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32c3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:custom_test-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32s2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:custom_test-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32s3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:example_test-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:example_test-esp32-ethernet: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - -.rules:test:example_test-esp32-include_nightly_run-rule: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-example_test-ota-include_nightly_run-rule - -.rules:test:example_test-esp32-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-sdio - -.rules:test:example_test-esp32-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:example_test-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32c2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:example_test-esp32c2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:example_test-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32c3-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:example_test-esp32c3-include_nightly_run-rule: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-example_test-ota-include_nightly_run-rule - -.rules:test:example_test-esp32c3-nvs_encr_hmac: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - -.rules:test:example_test-esp32c3-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-sdio - -.rules:test:example_test-esp32c3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:example_test-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32c6-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:example_test-esp32c6-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:example_test-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32h2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:example_test-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32s2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:example_test-esp32s2-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-sdio - -.rules:test:example_test-esp32s2-usb: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-usb - -.rules:test:example_test-esp32s2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:example_test-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32s3-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - -.rules:test:example_test-esp32s3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:test:example_test-i154: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-i154 - .rules:test:host_test: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index ef99c8bf45..e1ef1d0d54 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -6,8 +6,6 @@ clang_tidy_check: artifacts: paths: - clang_tidy_reports/ - when: always - expire_in: 1 day variables: IDF_TOOLCHAIN: clang script: @@ -23,10 +21,8 @@ check_pylint: needs: - pipeline_variables artifacts: - when: always reports: codequality: pylint.json - expire_in: 1 week script: - | if [ -n "$CI_MERGE_REQUEST_IID" ]; then @@ -72,10 +68,8 @@ check_pylint: GIT_DEPTH: 0 REPORT_PATTERN: clang_tidy_reports/**/*.txt artifacts: - when: always paths: - $REPORT_PATTERN - expire_in: 1 week dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. - clang_tidy_check diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml deleted file mode 100644 index 350b5d017f..0000000000 --- a/.gitlab/ci/target-test.yml +++ /dev/null @@ -1,1596 +0,0 @@ -.target_test_template: - image: $TARGET_TEST_ENV_IMAGE - extends: - - .before_script:fetch:target_test - stage: target_test - timeout: 1 hour - dependencies: [] - cache: - # Usually do not need submodule-cache in target_test - - key: pip-cache-${LATEST_GIT_TAG} - paths: - - .cache/pip - policy: pull - after_script: - - python tools/ci/artifacts_handler.py upload --type logs junit_reports - -.pytest_template: - extends: - - .target_test_template - artifacts: - when: always - paths: - - XUNIT_RESULT.xml - - pytest_embedded_log/ - reports: - junit: XUNIT_RESULT.xml - expire_in: 1 week - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - - if [ -n "$BUILD_JOB_NAME" ]; then - job_name=$BUILD_JOB_NAME; - else - job_name=${BUILD_JOB_PREFIX}$(python tools/ci/ci_get_mr_info.py target_in_tags $job_tags); - fi - - run_cmd python tools/ci/artifacts_handler.py download --job-name "$job_name" --type build_dir_without_map_and_elf_files - - if [ -n "$REQUIRES_ELF_FILES" ]; then - run_cmd python tools/ci/artifacts_handler.py download --job-name "$job_name" --type map_and_elf_files; - fi - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - -.pytest_examples_dir_template: - extends: .pytest_template - variables: - TEST_DIR: examples - BUILD_JOB_PREFIX: build_pytest_examples_ - -.pytest_examples_dir_jtag_template: - extends: .pytest_examples_dir_template - needs: - - job: build_pytest_examples_jtag - artifacts: false - variables: - BUILD_JOB_NAME: build_pytest_examples_jtag - REQUIRES_ELF_FILES: "1" - PYTEST_EXTRA_FLAGS: "--log-cli-level DEBUG" - -pytest_examples_esp32_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, generic ] - parallel: 3 - -pytest_examples_esp32_esp32eco3: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, esp32eco3 ] - -pytest_examples_esp32_ir_transceiver: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ir_transceiver ] - -pytest_examples_esp32_twai_transceiver: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, twai_transceiver ] - -pytest_examples_esp32_twai_network: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, twai_network ] - -pytest_examples_esp32_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32 - tags: [ esp32, jtag ] - -pytest_examples_esp32_ccs811: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ccs811 ] - -pytest_examples_esp32_sdio: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-sdio - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, sdio_master_slave ] - -pytest_examples_esp32s2_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2 - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, generic ] - parallel: 3 - -pytest_examples_esp32s2_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32s2 - tags: [ esp32s2, jtag ] - -pytest_examples_esp32s3_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, generic ] - parallel: 3 - -pytest_examples_esp32s3_usb_serial_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32s3 - tags: [ esp32s3, usb_serial_jtag ] - -pytest_examples_esp32s3_f4r8: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R8 ] - -pytest_examples_esp32c2_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2 - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, generic, xtal_40mhz ] - parallel: 3 - -pytest_examples_esp32c2_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32c2 - tags: [ esp32c2, jtag ] - -pytest_examples_esp32c3_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3 - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, generic ] - parallel: 3 - -pytest_examples_esp32c3_usb_serial_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32c3 - tags: [ esp32c3, usb_serial_jtag ] - -pytest_examples_esp32c3_flash_suspend: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3 - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, flash_suspend ] - -pytest_examples_esp32c6_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6 - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, generic ] - -pytest_examples_esp32c6_usj_device: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6 - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, usj_device ] - -pytest_examples_esp32h2_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32h2 - needs: - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32h2, generic ] - -pytest_examples_esp32p4_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32p4 - needs: - - job: build_pytest_examples_esp32p4 - artifacts: false - tags: [ esp32p4, generic ] - -pytest_examples_esp32_ethernet_ota: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet_ota ] - -pytest_examples_esp32_wifi_high_traffic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_high_traffic ] - -pytest_examples_esp32_flash_encryption_wifi_high_traffic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-include_nightly_run-rule - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, flash_encryption_wifi_high_traffic ] - -pytest_examples_esp32c3_flash_encryption_wifi_high_traffic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-include_nightly_run-rule - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, flash_encryption_wifi_high_traffic ] - -pytest_examples_esp32_ethernet: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet] - -pytest_examples_esp32_ethernet_httpbin: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, httpbin] - -pytest_examples_esp32_8mb_flash: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet_flash_8m ] - -pytest_examples_esp32_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_ap ] - -pytest_examples_esp32c3_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-wifi - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_ap ] - -pytest_examples_esp32s3_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3-wifi - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_ap ] - -pytest_examples_esp32s2_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-wifi - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, wifi_ap ] - -pytest_examples_esp32c2_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_ap, xtal_40mhz ] - -pytest_examples_esp32c2_26m_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_ap, xtal_26mhz ] - -pytest_examples_esp32c6_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6-wifi - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, wifi_ap ] - -pytest_examples_esp32_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_router ] - -pytest_examples_esp32c3_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-wifi - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_router ] - -pytest_examples_esp32s3_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3-wifi - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_router ] - -pytest_examples_esp32s2_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-wifi - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, wifi_router ] - -pytest_examples_esp32c2_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_router, xtal_40mhz ] - -pytest_examples_esp32c2_26m_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_router, xtal_26mhz ] - -pytest_examples_esp32c6_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6-wifi - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, wifi_router ] - -pytest_examples_esp32_wifi_iperf: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, Example_ShieldBox_Basic ] - -pytest_examples_esp32_wifi_wlan: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_wlan ] - -pytest_examples_esp32_ethernet_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet_router ] - -pytest_examples_esp32_ethernet_vlan: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet_vlan ] - -pytest_examples_esp32_ethernet_bridge: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, eth_w5500 ] - variables: - PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER}" - -pytest_examples_esp32_flash_encryption: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, flash_encryption ] - -pytest_examples_esp32_wifi_two_dut: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_two_dut ] - -pytest_examples_esp32c3_wifi_two_dut: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-wifi - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_two_dut ] - -pytest_examples_esp32s3_wifi_two_dut: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3-wifi - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_two_dut ] - -pytest_examples_esp32c2_wifi_two_dut: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_two_dut, xtal_26mhz ] - -pytest_examples_esp32c3_flash_encryption: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3 - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, flash_encryption ] - -pytest_examples_esp32c3_nvs_encr_hmac: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-nvs_encr_hmac - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, nvs_encr_hmac ] - -pytest_examples_esp32s2_usb_device: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-usb - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, usb_device ] - -pytest_examples_esp32_sdmmc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, sdcard_sdmode ] - -pytest_examples_esp32_sdspi: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-sdio - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, sdcard_spimode ] - -pytest_examples_esp32s2_sdspi: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-sdio - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, sdcard_spimode ] - -pytest_examples_esp32c3_sdspi: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-sdio - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, sdcard_spimode ] - -pytest_examples_esp32_extflash: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, external_flash ] - -pytest_examples_esp32_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-adc - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, adc ] - -pytest_examples_esp32s2_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-adc - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, adc ] - -pytest_examples_esp32s3_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3-adc - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, adc ] - -pytest_examples_esp32c3_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-adc - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, adc ] - -pytest_examples_esp32c2_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-adc - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, adc, xtal_26mhz] - -pytest_examples_esp32c6_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6-adc - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, adc ] - -pytest_examples_esp32h2_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32h2-adc - needs: - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32h2, adc ] - -pytest_examples_esp32s3_emmc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, emmc ] - -.pytest_components_dir_template: - extends: .pytest_template - variables: - TEST_DIR: components - BUILD_JOB_PREFIX: build_pytest_components_ - -pytest_components_esp32_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, generic ] - parallel: 7 - -pytest_components_esp32_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, generic_multi_device ] - -pytest_components_esp32_wifi_two_dut: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, wifi_two_dut ] - -pytest_components_esp32_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-adc - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, adc ] - -pytest_components_esp32_sdmmc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, sdcard ] - -pytest_components_esp32s3_sdmmc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, sdcard ] - -pytest_components_esp32_sdio: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-sdio - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, sdio_master_slave ] - -pytest_components_esp32_esp32c6_sdio: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-sdio - needs: - - job: build_pytest_components_esp32 - artifacts: false - - job: build_pytest_components_esp32c6 - artifacts: false - tags: [ esp32c6, sdio_multidev_32_c6 ] - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - # download the artifacts, requires esp32 and esp32c6 chips - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_components_esp32" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_components_esp32c6" - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - -pytest_components_esp32_lan8720: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_lan8720 ] - -pytest_components_esp32_rtl8201: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_rtl8201 ] - -pytest_components_esp32_w5500: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_w5500 ] - -pytest_components_esp32_ksz8851snl: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_ksz8851snl ] - -pytest_components_esp32_dm9051: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_dm9051 ] - -pytest_components_esp32_ksz8041: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_ksz8041 ] - -pytest_components_esp32_dp83848: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_dp83848 ] - -pytest_components_esp32_ethernet: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, ethernet ] - -pytest_components_esp32_flash_encryption: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, flash_encryption ] - -pytest_components_esp32_flash_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-flash_multi - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, flash_multi ] - -pytest_components_esp32_xtal32k: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, xtal32k ] - -pytest_components_esp32_no32kXtal: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, no32kXtal ] - -pytest_components_esp32_rs485_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, multi_dut_modbus_rs485 ] - -pytest_components_esp32_psramv0: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, psramv0 ] - -pytest_components_esp32s2_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2 - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, generic ] - parallel: 5 - -pytest_components_esp32s2_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2 - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, generic_multi_device ] - -pytest_components_esp32s2_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2-adc - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, adc ] - -pytest_components_esp32s2_flash_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2-flash_multi - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, flash_multi ] - -pytest_components_esp32s3_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, generic ] - parallel: 5 - -pytest_components_esp32s3_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, generic_multi_device ] - -pytest_components_esp32s3_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3-adc - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, adc ] - -pytest_components_esp32s3_octal_psram: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, octal_psram ] - -pytest_components_esp32s3_quad_psram: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, quad_psram ] - -pytest_components_esp32s3_flash_encryption_f4r8: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, flash_encryption_f4r8 ] - -pytest_components_esp32s3_flash_encryption_f8r8: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, flash_encryption_f8r8 ] - -pytest_components_esp32s3_flash_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3-flash_multi - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, flash_multi ] - -pytest_components_esp32s3_mspi_f4r4: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R4 ] - -pytest_components_esp32s3_mspi_f4r8: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R8 ] - -pytest_components_esp32s3_mspi_f8r8: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F8R8 ] - -pytest_components_esp32s3_usb_serial_jtag: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - build_pytest_components_esp32s3 - tags: [ esp32s3, usb_serial_jtag ] - -pytest_components_esp32c2_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2 - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, generic, xtal_40mhz ] - parallel: 3 - -pytest_components_esp32c2_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2-adc - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, adc, xtal_26mhz ] - -pytest_components_esp32c2_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2 - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, generic_multi_device, xtal_40mhz ] - -pytest_components_esp32c2_xtal_26mhz: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2 - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, generic, xtal_26mhz ] - -pytest_components_esp32c3_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, generic ] - parallel: 3 - -pytest_components_esp32c3_i2c_oled: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, i2c_oled ] - -pytest_components_esp32c3_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, generic_multi_device ] - -pytest_components_esp32c3_wifi_two_dut: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-wifi - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_two_dut ] - -pytest_components_esp32c3_usb_serial_jtag: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - build_pytest_components_esp32c3 - tags: [ esp32c3, usb_serial_jtag ] - -pytest_components_esp32s3_wifi_two_dut: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3-wifi - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_two_dut ] - -pytest_components_esp32c2_wifi_two_dut: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2-wifi - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_two_dut, xtal_26mhz ] - -pytest_components_esp32c3_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-adc - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, adc ] - -pytest_components_esp32c3_flash_encryption: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, flash_encryption ] - -pytest_components_esp32c3_nvs_encr_hmac: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-nvs_encr_hmac - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, nvs_encr_hmac ] - -pytest_components_esp32c3_flash_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-flash_multi - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, flash_multi ] - -pytest_components_esp32_sdspi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-sdio - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, sdcard_spimode ] - -pytest_components_esp32s2_sdspi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2-sdio - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, sdcard_spimode ] - -pytest_components_esp32c3_sdspi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-sdio - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, sdcard_spimode ] - -pytest_components_esp32c6_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c6 - needs: - - job: build_pytest_components_esp32c6 - artifacts: false - tags: [ esp32c6, generic ] - parallel: 2 - -pytest_components_esp32c6_usb_serial_jtag: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c6 - needs: - - build_pytest_components_esp32c6 - tags: [ esp32c6, usb_serial_jtag ] - -pytest_components_esp32h2_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2 - needs: - - job: build_pytest_components_esp32h2 - artifacts: false - tags: [ esp32h2, generic ] - parallel: 2 - -pytest_components_esp32p4_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32p4 - needs: - - job: build_pytest_components_esp32p4 - artifacts: false - tags: [ esp32p4, generic ] - parallel: 2 - -pytest_components_esp32h2_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2 - needs: - - job: build_pytest_components_esp32h2 - artifacts: false - tags: [ esp32h2, generic_multi_device ] - -pytest_components_esp32h2_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2-adc - needs: - - job: build_pytest_components_esp32h2 - artifacts: false - tags: [ esp32h2, adc ] - -pytest_components_esp32h2_ecdsa: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2-ecdsa - needs: - - job: build_pytest_components_esp32h2 - artifacts: false - tags: [ esp32h2, ecdsa_efuse ] - -pytest_components_esp32h2_usb_serial_jtag: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2 - needs: - - build_pytest_components_esp32h2 - tags: [ esp32h2, usb_serial_jtag ] - -pytest_components_esp32c6_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c6 - needs: - - job: build_pytest_components_esp32c6 - artifacts: false - tags: [ esp32c6, generic_multi_device ] - -pytest_components_esp32c6_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c6-adc - needs: - - job: build_pytest_components_esp32c6 - artifacts: false - tags: [ esp32c6, adc ] - -pytest_examples_esp32c6_i154: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6 - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, ieee802154 ] - -pytest_examples_openthread_br: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-i154 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - - job: build_pytest_examples_esp32c6 - artifacts: false - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32c6, openthread_br ] - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - # download the artifacts, requires s3, c6, h2 chips - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32s3" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32c6" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32h2" - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - -pytest_examples_openthread_bbr: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-i154 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - - job: build_pytest_examples_esp32c6 - artifacts: false - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32c6, openthread_bbr ] - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - # download the artifacts, requires s3, c6, h2 chips - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32s3" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32c6" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32h2" - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - -pytest_examples_openthread_sleep: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32h2 - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32c6, openthread_sleep ] - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - # download the artifacts, requires c6, h2 chips - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32c6" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32h2" - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - -pytest_examples_esp32h2_zigbee: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32h2 - needs: - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32h2, zigbee_multi_dut ] - -pytest_components_esp32s3_usb_host: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3-usb - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, usb_host_flash_disk ] - -.pytest_test_apps_dir_template: - extends: .pytest_template - variables: - TEST_DIR: tools/test_apps - BUILD_JOB_PREFIX: build_pytest_test_apps_ - REQUIRES_ELF_FILES: "1" - -pytest_test_apps_esp32_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32 - needs: - - job: build_pytest_test_apps_esp32 - artifacts: false - tags: [ esp32, generic ] - -pytest_test_apps_esp32_jtag: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32 - needs: - - job: build_pytest_test_apps_esp32 - artifacts: false - tags: [ esp32, jtag ] - variables: - PYTEST_EXTRA_FLAGS: "--log-cli-level DEBUG" - -pytest_test_apps_esp32_ethernet: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32 - needs: - - job: build_pytest_test_apps_esp32 - artifacts: false - tags: [ esp32, ethernet ] - -pytest_test_apps_esp32s2_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s2 - needs: - - job: build_pytest_test_apps_esp32s2 - artifacts: false - tags: [ esp32s2, generic ] - -pytest_test_apps_esp32s3_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3 - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, generic ] - -pytest_test_apps_esp32c2_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c2 - needs: - - job: build_pytest_test_apps_esp32c2 - artifacts: false - tags: [ esp32c2, generic, xtal_40mhz ] - -pytest_test_apps_esp32c3_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c3 - needs: - - job: build_pytest_test_apps_esp32c3 - artifacts: false - tags: [ esp32c3, generic ] - -pytest_test_apps_esp32c6_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c6 - needs: - - job: build_pytest_test_apps_esp32c6 - artifacts: false - tags: [ esp32c6, generic ] - -pytest_test_apps_esp32h2_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32h2 - needs: - - job: build_pytest_test_apps_esp32h2 - artifacts: false - tags: [ esp32h2, generic ] - -pytest_test_apps_esp32p4_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32p4 - needs: - - job: build_pytest_test_apps_esp32p4 - artifacts: false - tags: [ esp32p4, generic ] - -pytest_test_apps_esp32s3_mspi_f8r8: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3 - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F8R8 ] - -pytest_test_apps_esp32s3_mspi_f4r8: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3 - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R8 ] - -pytest_test_apps_esp32s3_mspi_f4r4: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3 - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R4 ] - -pytest_test_apps_esp32s2_wifi_two_dut: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s2-wifi - needs: - - job: build_pytest_test_apps_esp32s2 - artifacts: false - tags: [ esp32s2, wifi_two_dut ] - -pytest_test_apps_esp32s3_wifi_two_dut: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3-wifi - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_two_dut ] - -pytest_test_apps_esp32c2_wifi_two_dut: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c2-wifi - needs: - - job: build_pytest_test_apps_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_two_dut, xtal_26mhz ] - -pytest_test_apps_esp32c3_wifi_two_dut: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c3-wifi - needs: - - job: build_pytest_test_apps_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_two_dut] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8735049c26..6b19ed496f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,20 +91,13 @@ repos: always_run: true files: '\.gitlab/CODEOWNERS' pass_filenames: false - - id: check-rules-yml - name: Check rules.yml all rules have at lease one job applied, all rules needed exist - entry: tools/ci/check_rules_yml.py - language: python - files: '\.gitlab/ci/.+\.yml|\.gitlab-ci.yml|\.gitmodules' - pass_filenames: false - additional_dependencies: - - PyYAML == 5.3.1 - id: check-generated-rules name: Check rules are generated (based on .gitlab/ci/dependencies/dependencies.yml) entry: tools/ci/generate_rules.py language: python - files: '\.gitlab/ci/dependencies/.+|\.gitlab/ci/.*\.yml' + files: '\.gitlab/ci/dependencies/.+|\.gitlab/ci/.*\.yml|.gitlab-ci.yml' pass_filenames: false + require_serial: true additional_dependencies: - PyYAML == 5.3.1 - id: mypy-check @@ -115,6 +108,7 @@ repos: - 'mypy-extensions==0.4.3' - 'types-setuptools==57.4.14' - 'types-PyYAML==0.1.9' + - 'types-requests' exclude: > (?x)^( .*_pb2.py @@ -161,7 +155,7 @@ repos: require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf_build_apps~=1.0 + - idf-build-apps~=2.0.0rc1 - id: sort-build-test-rules-ymls name: sort .build-test-rules.yml files entry: tools/ci/check_build_test_rules.py sort-yaml @@ -185,6 +179,14 @@ repos: language: python always_run: true require_serial: true + - id: gitlab-yaml-linter + name: Check gitlab yaml files + entry: tools/ci/gitlab_yaml_linter.py + language: python + files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml' + pass_filenames: false + additional_dependencies: + - PyYAML == 5.3.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: diff --git a/components/driver/test_apps/.build-test-rules.yml b/components/driver/test_apps/.build-test-rules.yml index 7696651dbc..fd259747f7 100644 --- a/components/driver/test_apps/.build-test-rules.yml +++ b/components/driver/test_apps/.build-test-rules.yml @@ -21,6 +21,12 @@ components/driver/test_apps/i2s_test_apps/legacy_i2s_driver: components/driver/test_apps/legacy_adc_driver: disable: - if: SOC_ADC_SUPPORTED != 1 + depends_components: + - efuse + - esp_driver_i2s + - esp_driver_spi + depends_filepatterns: + - components/driver/deprecated/**/*adc* components/driver/test_apps/legacy_i2c_driver: disable_test: diff --git a/components/esp_eth/test_apps/pytest_esp_eth.py b/components/esp_eth/test_apps/pytest_esp_eth.py index 921636f441..2b49b9c6ad 100644 --- a/components/esp_eth/test_apps/pytest_esp_eth.py +++ b/components/esp_eth/test_apps/pytest_esp_eth.py @@ -204,6 +204,7 @@ def test_esp_emac_hal(dut: IdfDut) -> None: @pytest.mark.esp32 @pytest.mark.ip101 +@pytest.mark.temp_skip_ci(targets=['esp32'], reason='runner under maintenance') @pytest.mark.parametrize('config', [ 'default_ip101', ], indirect=True) diff --git a/components/fatfs/test_apps/.build-test-rules.yml b/components/fatfs/test_apps/.build-test-rules.yml index 0a1970ea43..15288644e4 100644 --- a/components/fatfs/test_apps/.build-test-rules.yml +++ b/components/fatfs/test_apps/.build-test-rules.yml @@ -15,25 +15,22 @@ components/fatfs/test_apps/flash_wl: disable_test: - if: IDF_TARGET not in ["esp32", "esp32c3"] reason: only one target per arch needed - depends_components: - esp_partition - spi_flash - fatfs - vfs - wear_leveling - + components/fatfs/test_apps/sdcard: disable: - if: IDF_TARGET == "esp32p4" temporary: true reason: target esp32p4 is not supported yet # TODO: IDF-7501 - disable_test: - if: IDF_TARGET not in ["esp32", "esp32c3"] temporary: true reason: lack of runners - depends_components: - esp_driver_sdmmc - esp_driver_spi diff --git a/components/mbedtls/.build-test-rules.yml b/components/mbedtls/.build-test-rules.yml index 705dd4b66b..f33fabc474 100644 --- a/components/mbedtls/.build-test-rules.yml +++ b/components/mbedtls/.build-test-rules.yml @@ -5,3 +5,7 @@ components/mbedtls/test_apps: - if: CONFIG_NAME == "psram" and SOC_SPIRAM_SUPPORTED != 1 - if: CONFIG_NAME == "psram_all_ext" and SOC_SPIRAM_SUPPORTED != 1 - if: CONFIG_NAME == "ecdsa_sign" and SOC_ECDSA_SUPPORTED != 1 + depends_components: + - efuse + depends_filepatterns: + - components/mbedtls/port/ecdsa/* diff --git a/components/spi_flash/test_apps/esp_flash_stress/README.md b/components/spi_flash/test_apps/esp_flash_stress/README.md index bf47d80ec6..a8b7833fa3 100644 --- a/components/spi_flash/test_apps/esp_flash_stress/README.md +++ b/components/spi_flash/test_apps/esp_flash_stress/README.md @@ -1,2 +1,2 @@ -| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 | -| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | diff --git a/components/usb/test_apps/.build-test-rules.yml b/components/usb/test_apps/.build-test-rules.yml index d11df13540..77a723b155 100644 --- a/components/usb/test_apps/.build-test-rules.yml +++ b/components/usb/test_apps/.build-test-rules.yml @@ -3,3 +3,8 @@ components/usb/test_apps: enable: - if: SOC_USB_OTG_SUPPORTED == 1 + depends_components: + - usb + depends_filepatterns: + - components/hal/usb*.c + - components/hal/esp32*/include/hal/usb*.h diff --git a/conftest.py b/conftest.py index 0fdc3ad45a..e9048d461f 100644 --- a/conftest.py +++ b/conftest.py @@ -1,53 +1,54 @@ -# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - # pylint: disable=W0621 # redefined-outer-name -# This file is a pytest root configuration file and provide the following functionalities: -# 1. Defines a few fixtures that could be used under the whole project. -# 2. Defines a few hook functions. -# # IDF is using [pytest](https://github.com/pytest-dev/pytest) and -# [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its example test framework. -# -# This is an experimental feature, and if you found any bug or have any question, please report to -# https://github.com/espressif/pytest-embedded/issues +# [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its test framework. + +# if you found any bug or have any question, +# please report to https://github.com/espressif/pytest-embedded/issues +# or discuss at https://github.com/espressif/pytest-embedded/discussions + +import os +import sys + +import gitlab + +if os.path.join(os.path.dirname(__file__), 'tools', 'ci') not in sys.path: + sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci')) + +if os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages') not in sys.path: + sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages')) import glob -import json +import io import logging import os import re -import sys +import typing as t +import zipfile from copy import deepcopy from datetime import datetime -from typing import Callable, Optional +import common_test_methods # noqa: F401 +import gitlab_api import pytest +import requests +import yaml from _pytest.config import Config from _pytest.fixtures import FixtureRequest +from artifacts_handler import ArtifactType +from dynamic_pipelines.constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME +from idf_ci.app import import_apps_from_txt +from idf_ci.uploader import AppUploader +from idf_ci_utils import IDF_PATH +from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS, PytestCase +from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, ITEM_PYTEST_CASE_KEY, IdfPytestEmbedded +from idf_pytest.utils import format_case_id from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.unity_tester import CaseTester -try: - from idf_ci_utils import IDF_PATH - from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS - from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, IdfPytestEmbedded - from idf_pytest.utils import format_case_id, get_target_marker_from_expr -except ImportError: - sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci')) - from idf_ci_utils import IDF_PATH - from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS - from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, IdfPytestEmbedded - from idf_pytest.utils import format_case_id, get_target_marker_from_expr - -try: - import common_test_methods # noqa: F401 -except ImportError: - sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages')) - import common_test_methods # noqa: F401 - ############ # Fixtures # @@ -100,9 +101,91 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str: return format_case_id(target, config, request.node.originalname, is_qemu=is_qemu, params=filtered_params) # type: ignore +@pytest.fixture(scope='session') +def pipeline_id(request: FixtureRequest) -> t.Optional[str]: + return request.config.getoption('pipeline_id', None) or os.getenv('PARENT_PIPELINE_ID', None) # type: ignore + + +class BuildReportDownloader: + def __init__(self, presigned_url_yaml: str) -> None: + self.app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = yaml.safe_load(presigned_url_yaml) + + def download_app( + self, app_build_path: str, artifact_type: ArtifactType = ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES + ) -> None: + url = self.app_presigned_urls_dict[app_build_path][artifact_type.value] + + logging.debug('Downloading app from %s', url) + with io.BytesIO() as f: + for chunk in requests.get(url).iter_content(chunk_size=1024 * 1024): + if chunk: + f.write(chunk) + + f.seek(0) + + with zipfile.ZipFile(f) as zip_ref: + zip_ref.extractall() + + +@pytest.fixture(scope='session') +def app_downloader(pipeline_id: t.Optional[str]) -> t.Union[AppUploader, BuildReportDownloader, None]: + if not pipeline_id: + return None + + if ( + 'IDF_S3_BUCKET' in os.environ + and 'IDF_S3_ACCESS_KEY' in os.environ + and 'IDF_S3_SECRET_KEY' in os.environ + and 'IDF_S3_SERVER' in os.environ + and 'IDF_S3_BUCKET' in os.environ + ): + return AppUploader(pipeline_id) + + logging.info('Downloading build report from the build pipeline %s', pipeline_id) + test_app_presigned_urls_file = None + try: + gl = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) + except gitlab.exceptions.GitlabAuthenticationError: + msg = """To download artifacts from gitlab, please create ~/.python-gitlab.cfg with the following content: + +[global] +default = internal +ssl_verify = true +timeout = 5 + +[internal] +url = +private_token = +api_version = 4 +""" + raise SystemExit(msg) + + for child_pipeline in gl.project.pipelines.get(pipeline_id, lazy=True).bridges.list(iterator=True): + if child_pipeline.name == 'build_child_pipeline': + for job in gl.project.pipelines.get(child_pipeline.downstream_pipeline['id'], lazy=True).jobs.list( + iterator=True + ): + if job.name == 'generate_pytest_build_report': + test_app_presigned_urls_file = gl.download_artifact( + job.id, [TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME] + )[0] + break + + if test_app_presigned_urls_file: + return BuildReportDownloader(test_app_presigned_urls_file) + + return None + + @pytest.fixture @multi_dut_fixture -def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str: +def build_dir( + request: FixtureRequest, + app_path: str, + target: t.Optional[str], + config: t.Optional[str], + app_downloader: t.Optional[AppUploader], +) -> str: """ Check local build dir with the following priority: @@ -114,14 +197,25 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st Returns: valid build directory """ - check_dirs = [] - if target is not None and config is not None: - check_dirs.append(f'build_{target}_{config}') - if target is not None: - check_dirs.append(f'build_{target}') - if config is not None: - check_dirs.append(f'build_{config}') - check_dirs.append('build') + # download from minio on CI + case: PytestCase = request._pyfuncitem.stash[ITEM_PYTEST_CASE_KEY] + if app_downloader: + # somehow hardcoded... + app_build_path = os.path.join(os.path.relpath(app_path, IDF_PATH), f'build_{target}_{config}') + if case.requires_elf_or_map: + app_downloader.download_app(app_build_path) + else: + app_downloader.download_app(app_build_path, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES) + check_dirs = [f'build_{target}_{config}'] + else: + check_dirs = [] + if target is not None and config is not None: + check_dirs.append(f'build_{target}_{config}') + if target is not None: + check_dirs.append(f'build_{target}') + if config is not None: + check_dirs.append(f'build_{config}') + check_dirs.append('build') for check_dir in check_dirs: binary_path = os.path.join(app_path, check_dir) @@ -138,13 +232,20 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st @pytest.fixture(autouse=True) @multi_dut_fixture -def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, object], None]) -> None: +def junit_properties(test_case_name: str, record_xml_attribute: t.Callable[[str, object], None]) -> None: """ This fixture is autoused and will modify the junit report test case name to .. """ record_xml_attribute('name', test_case_name) +@pytest.fixture(autouse=True) +@multi_dut_fixture +def ci_job_url(record_xml_attribute: t.Callable[[str, object], None]) -> None: + if ci_job_url := os.getenv('CI_JOB_URL'): + record_xml_attribute('ci_job_url', ci_job_url) + + @pytest.fixture(autouse=True) def set_test_case_name(request: FixtureRequest, test_case_name: str) -> None: request.node.funcargs['test_case_name'] = test_case_name @@ -154,7 +255,7 @@ def set_test_case_name(request: FixtureRequest, test_case_name: str) -> None: # Log Util Functions # ###################### @pytest.fixture -def log_performance(record_property: Callable[[str, object], None]) -> Callable[[str, str], None]: +def log_performance(record_property: t.Callable[[str, object], None]) -> t.Callable[[str, str], None]: """ log performance item with pre-defined format to the console and record it under the ``properties`` tag in the junit report if available. @@ -172,7 +273,7 @@ def log_performance(record_property: Callable[[str, object], None]) -> Callable[ @pytest.fixture -def check_performance(idf_path: str) -> Callable[[str, float, str], None]: +def check_performance(idf_path: str) -> t.Callable[[str, float, str], None]: """ check if the given performance item meets the passing standard or not """ @@ -186,9 +287,9 @@ def check_performance(idf_path: str) -> Callable[[str, float, str], None]: """ def _find_perf_item(operator: str, path: str) -> float: - with open(path, 'r') as f: + with open(path) as f: data = f.read() - match = re.search(r'#define\s+IDF_PERFORMANCE_{}_{}\s+([\d.]+)'.format(operator, item.upper()), data) + match = re.search(fr'#define\s+IDF_PERFORMANCE_{operator}_{item.upper()}\s+([\d.]+)', data) return float(match.group(1)) # type: ignore def _check_perf(operator: str, standard_value: float) -> None: @@ -198,7 +299,7 @@ def check_performance(idf_path: str) -> Callable[[str, float, str], None]: ret = value >= standard_value if not ret: raise AssertionError( - "[Performance] {} value is {}, doesn't meet pass standard {}".format(item, value, standard_value) + f"[Performance] {item} value is {value}, doesn't meet pass standard {standard_value}" ) path_prefix = os.path.join(idf_path, 'components', 'idf_test', 'include') @@ -212,7 +313,7 @@ def check_performance(idf_path: str) -> Callable[[str, float, str], None]: for performance_file in performance_files: try: standard = _find_perf_item(op, performance_file) - except (IOError, AttributeError): + except (OSError, AttributeError): # performance file doesn't exist or match is not found in it continue @@ -221,13 +322,13 @@ def check_performance(idf_path: str) -> Callable[[str, float, str], None]: break if not found_item: - raise AssertionError('Failed to get performance standard for {}'.format(item)) + raise AssertionError(f'Failed to get performance standard for {item}') return real_func @pytest.fixture -def log_minimum_free_heap_size(dut: IdfDut, config: str) -> Callable[..., None]: +def log_minimum_free_heap_size(dut: IdfDut, config: str) -> t.Callable[..., None]: def real_func() -> None: res = dut.expect(r'Minimum free heap size: (\d+) bytes') logging.info( @@ -247,12 +348,12 @@ def log_minimum_free_heap_size(dut: IdfDut, config: str) -> Callable[..., None]: return real_func -@pytest.fixture +@pytest.fixture(scope='session') def dev_password(request: FixtureRequest) -> str: return request.config.getoption('dev_passwd') or '' -@pytest.fixture +@pytest.fixture(scope='session') def dev_user(request: FixtureRequest) -> str: return request.config.getoption('dev_user') or '' @@ -274,66 +375,69 @@ def pytest_addoption(parser: pytest.Parser) -> None: '--dev-passwd', help='password associated with some specific device/service used during the test execution', ) - idf_group.addoption( - '--app-info-basedir', - default=IDF_PATH, - help='app info base directory. specify this value when you\'re building under a ' - 'different IDF_PATH. (Default: $IDF_PATH)', - ) idf_group.addoption( '--app-info-filepattern', help='glob pattern to specify the files that include built app info generated by ' - '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' - 'paths not exist in local file system if not listed recorded in the app info.', + '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' + 'paths not exist in local file system if not listed recorded in the app info.', + ) + idf_group.addoption( + '--pipeline-id', + help='main pipeline id, not the child pipeline id. Specify this option to download the artifacts ' + 'from the minio server for debugging purpose.', ) def pytest_configure(config: Config) -> None: # cli option "--target" - target = config.getoption('target') or '' + target = [_t.strip().lower() for _t in (config.getoption('target', '') or '').split(',') if _t.strip()] + + # add markers based on idf_pytest/constants.py + for name, description in { + **TARGET_MARKERS, + **ENV_MARKERS, + **SPECIAL_MARKERS, + }.items(): + config.addinivalue_line('markers', f'{name}: {description}') help_commands = ['--help', '--fixtures', '--markers', '--version'] for cmd in help_commands: if cmd in config.invocation_params.args: - target = 'unneeded' + target = ['unneeded'] break - if not target: # also could specify through markexpr via "-m" - target = get_target_marker_from_expr(config.getoption('markexpr') or '') + markexpr = config.getoption('markexpr') or '' + # check marker expr set via "pytest -m" + if not target and markexpr: + # we use `-m "esp32 and generic"` in our CI to filter the test cases + # this doesn't cover all use cases, but fit what we do in CI. + for marker in markexpr.split('and'): + marker = marker.strip() + if marker in TARGET_MARKERS: + target.append(marker) - apps_list = None - app_info_basedir = config.getoption('app_info_basedir') + # "--target" must be set + if not target: + raise SystemExit( + """Pass `--target TARGET[,TARGET...]` to specify all targets the test cases are using. + - for single DUT, we run with `pytest --target esp32` + - for multi DUT, we run with `pytest --target esp32,esp32,esp32s2` to indicate all DUTs +""" + ) + + apps = None app_info_filepattern = config.getoption('app_info_filepattern') if app_info_filepattern: - apps_list = [] - for file in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): - with open(file) as fr: - for line in fr.readlines(): - if not line.strip(): - continue - - # each line is a valid json - app_info = json.loads(line.strip()) - if app_info_basedir and app_info['app_dir'].startswith(app_info_basedir): - relative_app_dir = os.path.relpath(app_info['app_dir'], app_info_basedir) - apps_list.append(os.path.join(IDF_PATH, os.path.join(relative_app_dir, app_info['build_dir']))) - print('Detected app: ', apps_list[-1]) - else: - print( - f'WARNING: app_info base dir {app_info_basedir} not recognizable in {app_info["app_dir"]}, skipping...' - ) - continue + apps = [] + for f in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): + apps.extend(import_apps_from_txt(f)) config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( target=target, - sdkconfig=config.getoption('sdkconfig'), - apps_list=apps_list, + apps=apps, ) config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) - for name, description in {**TARGET_MARKERS, **ENV_MARKERS, **SPECIAL_MARKERS}.items(): - config.addinivalue_line('markers', f'{name}: {description}') - def pytest_unconfigure(config: Config) -> None: _pytest_embedded = config.stash.get(IDF_PYTEST_EMBEDDED_KEY, None) diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index 90509dd5c8..b07ef455da 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -1,30 +1,35 @@ -================= -pytest in ESP-IDF -================= +=============================== +ESP-IDF Tests with Pytest Guide +=============================== :link_to_translation:`zh_CN:[中文]` -ESP-IDF has numerous types of tests that are meant to be executed on an ESP chip (known as **on target testing**). Target tests are usually compiled as part of an IDF project used for testing (known as a **test app**), where test apps follows the same build, flash, and monitor process of any other standard IDF project. +ESP-IDF provides a variety of testing mechanisms that runs directly on target ESP chips (referred to as **target test**). These target tests are typically integrated into an ESP-IDF project specifically designed for testing purposes (known as a **test app**). Similar to standard ESP-IDF projects, test apps follow the same build, flash, and monitoring procedures. -Typically, on target testing will require a connected host (e.g., a PC) that is responsible for triggering a particular test case, providing test data, and inspecting test results. +In target testing, a connected host (for instance, a PC) is typically required to trigger specific test cases, provide test data, and evaluate test results. -ESP-IDF uses the pytest framework (and some pytest plugins) on the host side to automate on target testing. This guide introduces pytest in ESP-IDF and covers the following concepts: +On the host side, ESP-IDF employs the pytest framework (alongside certain pytest plugins) to automate target testing. This guide delves into pytest in ESP-IDF, covering the following aspects: -1. The different types of test apps in ESP-IDF. -2. Using the pytest framework in Python scripts to automate target testing. -3. ESP-IDF Continuous Integration (CI) target testing process. -4. How to run target tests locally with pytest. +1. Common concepts in ESP-IDF target testing. +2. Using the pytest framework in Python scripts for target testing automation. +3. ESP-IDF Continuous Integration (CI) target testing workflow. +4. Running target tests locally using pytest. 5. pytest tips and tricks. .. note:: In ESP-IDF, we use the following pytest plugins by default: - - `pytest-embedded `__ with default services ``esp,idf`` - - `pytest-rerunfailures `__ + - `pytest-embedded `__ with default services ``esp,idf`` + - `pytest-rerunfailures `__ + - `pytest-ignore-test-results `__ All the concepts and usages introduced in this guide are based on the default behavior of these plugins, thus may not be available in vanilla pytest. +.. important:: + + This guide specifically targets ESP-IDF contributors. Some of the concepts, like the custom markers, may not be directly applicable to personal projects using the ESP-IDF SDK. For running pytest-embedded in personal projects, please refer to `pytest-embedded documentation `__, and explore the `provided examples `__. + Installation ============ @@ -34,253 +39,179 @@ All dependencies could be installed by running the install script with the ``--e $ install.sh --enable-pytest +We have implemented several mechanisms to ensure the successful execution of all installation processes. If you encounter any issues during the installation, please submit an issue report to our `GitHub issue tracker `__. -Common Issues During Installation ---------------------------------- +Common Concepts +=============== -No Package 'dbus-1' Found -^^^^^^^^^^^^^^^^^^^^^^^^^ +A **test app** is a set of binaries which is being built from an IDF project that is used to test a particular feature of your project. Test apps are usually located under ``${IDF_PATH}/examples``, ``${IDF_PATH}/tools/test_apps``, and ``${IDF_PATH}/components//test_apps``. -.. code:: text +A **Device under test (DUT)** is a set of ESP chip(s) which connect to a host (e.g., a PC). The host is responsible for flashing the apps to the DUT, triggering the test cases, and inspecting the test results. - configure: error: Package requirements (dbus-1 >= 1.8) were not met: +A typical ESP-IDF project that contains a pytest script will have the following structure: - No package 'dbus-1' found +.. code-block:: text - Consider adjusting the PKG_CONFIG_PATH environment variable if you - installed software in a non-standard prefix. + . + └── my_app/ + ├── main/ + │ └── ... + ├── CMakeLists.txt + └── pytest_foo.py -If you encounter the error message above, you may need to install some missing packages. +Sometimes, for some multi-dut tests, one test case requires multiple test apps. In this case, the test app folder structure would be like this: -If you are using Ubuntu, you may need to run: +.. code-block:: text -.. code:: shell - - sudo apt-get install libdbus-glib-1-dev - -or - -.. code:: shell - - sudo apt-get install libdbus-1-dev - -For other Linux distributions, please Google the error message above and find which missing packages need to be installed for your particular distribution. - -Invalid Command 'bdist_wheel' -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code:: text - - error: invalid command 'bdist_wheel' - -If you encounter the error message above, you may need to install some missing Python packages such as: - -.. code:: shell - - python -m pip install -U pip - -or - -.. code:: shell - - python -m pip install wheel - -.. note:: - - Before running the pip commands, please make sure you are using the IDF Python virtual environment. - - -Test Apps -========= - -ESP-IDF contains different types of test apps that can be automated using pytest. - -Component Tests ---------------- - -ESP-IDF components typically contain component specific test apps that execute component specific unit tests. Component test apps are the recommended way to test components. All the test apps should be located under ``${IDF_PATH}/components//test_apps``, for example: - -.. code:: text - - components/ - └── my_component/ - ├── include/ - │ └── ... - ├── test_apps/ - │ ├── test_app_1 - │ │ ├── main/ - │ │ │ └── ... - │ │ ├── CMakeLists.txt - │ │ └── pytest_my_component_app_1.py - │ ├── test_app_2 - │ │ ├── ... - │ │ └── pytest_my_component_app_2.py - │ └── parent_folder - │ ├── test_app_3 - │ │ ├── ... - │ │ └── pytest_my_component_app_3.py - │ └── ... - ├── my_component.c - └── CMakeLists.txt - -Example Tests -------------- - -The purpose of ESP-IDF examples is to demonstrate parts of ESP-IDF functionality to users (refer to :idf_file:`Examples Readme ` for more information). - -However, to ensure that these examples operate correctly, examples can be treated as test apps and executed automatically by using pytest. All examples should be located under ``${IDF_PATH}/examples``, with tested example including a Python test script, for example: - -.. code:: text - - examples/ - └── parent_folder/ - └── example_1/ - ├── main/ - │ └── ... - ├── CMakeLists.txt - └── pytest_example_1.py - -Custom Tests ------------- - -Custom Tests are tests that aim to test some arbitrary functionality of ESP-IDF, thus are not intended to demonstrate IDF functionality to users in any way. - -All custom test apps are located under ``${IDF_PATH}/tools/test_apps``. For more information please refer to the :idf_file:`Custom Test Readme `. + . + ├── my_app_foo/ + │ ├── main/ + │ │ └── ... + │ └── CMakeLists.txt + ├── my_app_bar/ + │ ├── main/ + │ │ └── ... + │ └── CMakeLists.txt + └── pytest_foo_bar.py pytest in ESP-IDF ================= -.. _pytest-execution-process: +Single DUT Test Cases +--------------------- -pytest Execution Process ------------------------- +Getting Started +^^^^^^^^^^^^^^^ -1. Bootstrapping Phase - - Create session-scoped caches: - - - port-target cache - - port-app cache - -2. Collection Phase - - A. Gather all Python files with the prefix ``pytest_``. - B. Gather all test functions with the prefix ``test_``. - C. Apply the `params `__, and duplicate the test functions. - D. Filter the test cases with CLI options. For the detailed usages, see :ref:`filter-the-test-cases`. - -3. Execution Phase - - A. Construct the `fixtures `__. In ESP-IDF, the common fixtures are initialized in this order: - - a. ``pexpect_proc``: `pexpect `__ instance - - b. ``app``: `IdfApp `__ instance - - The test app's information (e.g., sdkconfig, flash_files, partition_table, etc) would be parsed at this phase. - - c. ``serial``: `IdfSerial `__ instance - - The port of the host to which the target is connected is auto-detected. In the case of multiple targets connected to the host, the test target's type is parsed from the app. The test app binary files are flashed to the test target automatically. - - d. ``dut``: `IdfDut `__ instance - - B. Run the real test function. - - C. Deconstruct the fixtures in this order: - - a. ``dut`` - - i. close the ``serial`` port. - ii. (Only for apps with `Unity test framework `__) generate JUnit report of the Unity test cases. - - b. ``serial`` - c. ``app`` - d. ``pexpect_proc``: Close the file descriptor - - D. (Only for apps with `Unity test framework `__) - - If ``dut.expect_from_unity_output()`` is called, an ``AssertionError`` is raised upon detection of a Unity test failure. - -4. Reporting Phase - - A. Generate JUnit report of the test functions. - B. Modify the JUnit report test case name into ESP-IDF test case ID format: ``..``. - -5. Finalizatoin Phase (Only for apps with `Unity test framework `__) - - Combine the JUnit reports if the JUnit reports of the Unity test cases are generated. - -Basic Example -------------- - -This following Python test script example is taken from :idf_file:`pytest_console_basic.py `. - -.. code:: python +.. code-block:: python @pytest.mark.esp32 - @pytest.mark.esp32c3 + @pytest.mark.esp32s2 @pytest.mark.generic - @pytest.mark.parametrize('config', [ - 'history', - 'nohistory', - ], indirect=True) - def test_console_advanced(config: str, dut: IdfDut) -> None: - if config == 'history': - dut.expect('Command history enabled') - elif config == 'nohistory': - dut.expect('Command history disabled') + def test_hello_world(dut) -> None: + dut.expect('Hello world!') -To demonstrate how pytest is typically used in an ESP-IDF test script, let us go through this simple test script line by line in the following subsections. +This is a simple test script that could run with our getting-started example :example:`get-started/hello_world`. -Target Markers -^^^^^^^^^^^^^^ +First two lines are the target markers: -Pytest markers can be used to indicate which targets (i.e., which ESP chip) a particular test case should should run on. For example: - -.. code:: python - - @pytest.mark.esp32 # <-- support esp32 - @pytest.mark.esp32c3 # <-- support esp32c3 - @pytest.mark.generic # <-- test env "generic" - -The example above indicates that a particular test case is supported on the ESP32 and ESP32-C3. Furthermore, the target's board type should be ``generic``. For more details regarding the ``generic`` type, you may run ``pytest --markers`` to get detailed information regarding all markers. +* The ``@pytest.mark.esp32`` is a marker that indicates that this test case should be run on the ESP32. +* The ``@pytest.mark.esp32s2`` is a marker that indicates that this test case should be run on the ESP32-S2. .. note:: If the test case can be run on all targets officially supported by ESP-IDF (call ``idf.py --list-targets`` for more details), you can use a special marker ``supported_targets`` to apply all of them in one line. -Parameterized Markers -^^^^^^^^^^^^^^^^^^^^^ + We also supports ``preview_targets`` and ``all_targets`` as special target markers (call ``idf.py --list-targets --preview`` for a full targets list including preview targets). -You can use ``pytest.mark.parametrize`` with ``config`` to apply the same test to different apps with different sdkconfig files. For more information about ``sdkconfig.ci.xxx`` files, please refer to the Configuration Files section under :idf_file:`this readme `. +Next, we have the environment marker: -.. code:: python +* The ``@pytest.mark.generic`` is a marker that indicates that this test case should be run on the ``generic`` board type. + +.. note:: + + For the detailed explanation of the environment markers, please refer to :idf_file:`ENV_MARKERS definition ` + +Finally, we have the test function. With a ``dut`` fixture. In single-dut test cases, the ``dut`` fixture is an instance of ``IdfDut`` class, for multi-dut test cases, it is a tuple of ``IdfDut`` instances. For more details regarding the ``IdfDut`` class, please refer to `pytest-embedded IdfDut API reference `__. + +Same App with Different sdkconfig Files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For some test cases, you may need to run the same app with different sdkconfig files. For detailed documentation regarding sdkconfig related concepts, please refer to `idf-build-apps Documentation `__. + +Here's a simple example that demonstrates how to run the same app with different sdkconfig files. Assume we have the following folder structure: + +.. code-block:: text + + . + └── my_app/ + ├── main/ + │ └── ... + ├── CMakeLists.txt + ├── sdkconfig.ci.foo + ├── sdkconfig.ci.bar + └── pytest_foo.py + +If the test case needs to run all supported targets with these two sdkconfig files, you can use the following code: + +.. code-block:: python + + @pytest.mark.esp32 + @pytest.mark.esp32s2 + @pytest.mark.parametrize('config', [ # <-- parameterize the sdkconfig file + 'foo', # <-- run with sdkconfig.ci.foo + 'bar', # <-- run with sdkconfig.ci.bar + ], indirect=True) # <-- `indirect=True` is required, indicates this param is pre-calculated before other fixtures + def test_foo_bar(dut, config) -> None: + if config == 'foo': + dut.expect('This is from sdkconfig.ci.foo') + elif config == 'bar': + dut.expect('This is from sdkconfig.ci.bar') + +All markers will impact the test case simultaneously. Overall, this test function would be replicated to 4 test cases: + +- ``test_foo_bar``, with esp32 target, and sdkconfig.ci.foo as the sdkconfig file +- ``test_foo_bar``, with esp32 target, and sdkconfig.ci.bar as the sdkconfig file +- ``test_foo_bar``, with esp32s2 target, and sdkconfig.ci.foo as the sdkconfig file +- ``test_foo_bar``, with esp32s2 target, and sdkconfig.ci.bar as the sdkconfig file + +Sometimes in the test script or the log file, you may see the following format: + +- ``esp32.foo.test_foo_bar`` +- ``esp32.bar.test_foo_bar`` +- ``esp32s2.foo.test_foo_bar`` +- ``esp32s2.bar.test_foo_bar`` + +We call this format the **test case ID**. The test case ID should be considered as the unique identifier of a test case. It is composed of the following parts: + +- ``esp32``: the target name +- ``foo``: the config name +- ``test_foo_bar``: the test function name + +The test case ID is used to identify the test case in the JUnit report. + +.. note:: + + Nearly all the CLI options of pytest-embedded supports parameterization. To see all supported CLI options, you may run ``pytest --help`` and check the ``embedded-...`` sections for vanilla pytest-embedded ones, and the ``idf`` sections for ESP-IDF specific ones. + +.. note:: + + The target markers, like ``@pytest.mark.esp32`` and ``@pytest.mark.esp32s2``, are actually syntactic sugar for parameterization. In fact they are defined as: + + .. code-block:: python + + @pytest.mark.parametrize('target', [ + 'esp32', + 'esp32s2', + ], indirect=True) + +Same App with Different sdkconfig Files, Different Targets +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For some test cases, you may need to run the same app with different sdkconfig files. These sdkconfig files supports different targets. We may use ``pytest.param`` to achieve this. Let's use the same folder structure as above. + +.. code-block:: python @pytest.mark.parametrize('config', [ - 'history', # <-- run with app built by sdkconfig.ci.history - 'nohistory', # <-- run with app built by sdkconfig.ci.nohistory - ], indirect=True) # <-- `indirect=True` is required + pytest.param('foo', marks=[pytest.mark.esp32]), + pytest.param('bar', marks=[pytest.mark.esp32s2]), + ], indirect=True) -Overall, this test function would be replicated to 4 test cases: +Now this test function would be replicated to 2 test cases (represented as test case IDs): -- ``esp32.history.test_console_advanced`` -- ``esp32.nohistory.test_console_advanced`` -- ``esp32c3.history.test_console_advanced`` -- ``esp32c3.nohistory.test_console_advanced`` +* ``esp32.foo.test_foo_bar`` +* ``esp32s2.bar.test_foo_bar`` -Testing Serial Output -^^^^^^^^^^^^^^^^^^^^^ +Testing Serial Output (Expecting) +--------------------------------- To ensure that test has executed successfully on target, the test script can test that serial output of the target using the ``dut.expect()`` function, for example: -.. code:: python +.. code-block:: python - def test_console_advanced(config: str, dut: IdfDut) -> None: # The value of argument ``config`` is assigned by the parameterization. - if config == 'history': - dut.expect('Command history enabled') - elif config == 'nohistory': - dut.expect('Command history disabled') + def test_hello_world(dut) -> None: + dut.expect('\d+') # <-- `expect`ing from a regex + dut.expect_exact('Hello world!') # <-- `expect_exact`ly the string The ``dut.expect(...)`` will first compile the expected string into regex, which in turn is then used to seek through the serial output until the compiled regex is matched, or until a timeout occurs. @@ -288,115 +219,377 @@ Please pay extra attention to the expected string when it contains regex keyword For more information regarding the different types of ``expect`` functions, please refer to the `pytest-embedded Expecting documentation `__. -Advanced Examples ------------------ +Multi-DUT Test Cases +-------------------- Multi-Target Tests with the Same App ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In some cases a test may involve multiple targets running the same test app. In this case, multiple DUTs can be instantiated using ``parameterize``, for example: +In some cases a test may involve multiple targets running the same test app. Parametrize ``count`` to the number of DUTs you want to test with. -.. code:: python +.. code-block:: python + @pytest.mark.parametrize('count', [ + 2, + ], indirect=True) + @pytest.mark.parametrize('target', [ + 'esp32|esp32s2', + 'esp32s3', + ], indirect=True) + def test_hello_world(dut) -> None: + dut[0].expect('Hello world!') + dut[1].expect('Hello world!') + +The ``|`` symbol in all parametrized items is used for separating the settings for each DUT. In this example, the test case would be tested with: + +* esp32, esp32s2 +* esp32s3, esp32s3 + +After setting the param ``count`` to 2, all the fixtures are changed into tuples. + +.. important:: + + ``count`` is mandatory for multi-DUT tests. + +.. note:: + + For detailed multi-dut parametrization documentation, please refer to `pytest-embedded Multi-DUT documentation `__. + +.. warning:: + + In some test scripts, you may see target markers like ``@pytest.mark.esp32`` and ``@pytest.mark.esp32s2`` used together with multi-DUT test cases. This is deprecated and should be replaced with the ``target`` parametrization. + + For example, + + .. code-block:: python + + @pytest.mark.esp32 @pytest.mark.esp32s2 - @pytest.mark.esp32s3 - @pytest.mark.usb_host @pytest.mark.parametrize('count', [ 2, ], indirect=True) - def test_usb_host(dut: Tuple[IdfDut, IdfDut]) -> None: - device = dut[0] # <-- assume the first dut is the device - host = dut[1] # <-- and the second dut is the host - ... + def test_hello_world(dut) -> None: + dut[0].expect('Hello world!') + dut[1].expect('Hello world!') + + should be replaced with: -After setting the param ``count`` to 2, all these fixtures are changed into tuples. + .. code-block:: python + + @pytest.mark.parametrize('count', [ + 2, + ], indirect=True) + @pytest.mark.parametrize('target', [ + 'esp32', + 'esp32s2', + ], indirect=True) + def test_hello_world(dut) -> None: + dut[0].expect('Hello world!') + dut[1].expect('Hello world!') + + This could help avoid the ambiguity of the target markers when multi-DUT test cases are using different type of targets. Multi-Target Tests with Different Apps ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In some cases (in particular protocol tests), a test may involve multiple targets running different test apps (e.g., separate targets to act as master and slave). In this case, multiple DUTs with different test apps can be instantiated using ``parameterize``. +In some cases, a test may involve multiple targets running different test apps (e.g., separate targets to act as master and slave). Usually in ESP-IDF, the folder structure would be like this: -This code example is taken from :idf_file:`pytest_wifi_getting_started.py `. +.. code-block:: text -.. code:: python + . + ├── master/ + │ ├── main/ + │ │ └── ... + │ └── CMakeLists.txt + ├── slave/ + │ ├── main/ + │ │ └── ... + │ └── CMakeLists.txt + └── pytest_master_slave.py + +In this case, we can parametrize the ``app_path`` to the path of the test apps you want to test with. + +.. code-block:: python - @pytest.mark.esp32 @pytest.mark.multi_dut_generic - @pytest.mark.parametrize( - 'count, app_path', [ - (2, - f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}'), - ], indirect=True - ) - def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None: - softap = dut[0] - station = dut[1] - ... - -Here the first DUT was flashed with the app :idf_file:`softAP `, and the second DUT was flashed with the app :idf_file:`station `. + @pytest.mark.parametrize('count', [ + 2, + ], indirect=True) + @pytest.mark.parametrize('app_path, target', [ + (f'{os.path.join(os.path.dirname(__file__), "master")}|{os.path.join(os.path.dirname(__file__), "slave")}', 'esp32|esp32s2'), + (f'{os.path.join(os.path.dirname(__file__), "master")}|{os.path.join(os.path.dirname(__file__), "slave")}', 'esp32s2|esp32'), + ], indirect=True) + def test_master_slave(dut) -> None: + master = dut[0] + slave = dut[1] + + master.write('Hello world!') + slave.expect_exact('Hello world!') .. note:: - Here the ``app_path`` should be set with absolute path. The ``__file__`` macro in Python would return the absolute path of the test script itself. + When parametrizing two items, like ``app_path, target`` here, make sure you're passing a list of tuples to the ``parametrize`` decorator. Each tuple should contain the values for each item. -Multi-Target Tests with Different Apps and Targets -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The test case here will be replicated to 2 test cases: -This code example is taken from :idf_file:`pytest_wifi_getting_started.py `. As the comment says, for now it is not running in the ESP-IDF CI. +* dut-0, an ESP32, running app ``master``, and dut-1, an ESP32-S2, running app ``slave`` +* dut-0, an ESP32-S2, running app ``master``, and dut-1, an ESP32, running app ``slave`` -.. code:: python +Test Cases with Unity Test Framework +------------------------------------ + +We use `Unity test framework `__ in our unit tests. Overall, we have three types of test cases (`Unity test framework `__): + +* Normal test cases (single DUT) +* Multi-stage test cases (single DUT) +* Multi-device test cases (multi-DUT) + +All single-DUT test cases (including normal test cases and multi-stage test cases) can be run using the following command: + +.. code-block:: python + + def test_unity_single_dut(dut: IdfDut): + dut.run_all_single_board_cases() + +Using this command will skip all the test cases containing the ``[ignore]`` tag. + +If you need to run a group of test cases, you may run: + +.. code-block:: python + + def test_unity_single_dut(dut: IdfDut): + dut.run_all_single_board_cases(group='psram') + +It would trigger all test cases with the ``[psram]`` tag. + +.. warning:: + + You may also see that there are some test scripts with the following statements, which are deprecated. Please use the suggested one as above. + + .. code-block:: python + + def test_unity_single_dut(dut: IdfDut): + dut.expect_exact('Press ENTER to see the list of tests') + dut.write('*') + dut.expect_unity_test_output() + +We also provide a fixture ``case_tester`` to trigger all kinds of test cases easier. For example: + +.. code-block:: python + + def test_unity_single_dut(case_tester): + case_tester.run_all_normal_cases() # to run all normal test cases + case_tester.run_all_multi_dev_cases() # to run all multi-device test cases + case_tester.run_all_multi_stage_cases() # to run all multi-stage test cases + +For a full list of the available functions, please refer to `pytest-embedded case_tester API reference `__. + +Running Target Tests in CI +========================== + +The workflow in CI is as follows: + +.. blockdiag:: + :caption: Target Test Child Pipeline Workflow + :align: center + + blockdiag child-pipeline-workflow { + default_group_color = lightgray; + + group { + label = "build" + + build_test_related_apps; build_non_test_related_apps; + } + + group { + label = "assign_test" + + build_job_report; generate_pytest_child_pipeline; + } + + group { + label = "target_test" + + "Specific Target Test Jobs"; + } + + group { + label = ".post" + + target_test_report; + } + + build_test_related_apps, build_non_test_related_apps -> generate_pytest_child_pipeline, build_job_report -> "Specific Target Test Jobs" -> target_test_report; + } + +All build jobs and target test jobs are generated automatically by our CI script :project:`tools/ci/dynamic_pipelines`. + +Build Jobs +---------- + +In CI, all ESP-IDF projects under ``components``, ``examples``, and ``tools/test_apps``, are built with all supported targets and sdkconfig files. The binaries are built under ``build__``. For example + +.. code-block:: text + + . + ├── build_esp32_history/ + │ └── ... + ├── build_esp32_nohistory/ + │ └── ... + ├── build_esp32s2_history/ + │ └── ... + ├── ... + ├── main/ + ├── CMakeLists.txt + ├── sdkconfig.ci.history + ├── sdkconfig.ci.nohistory + └── ... + +There are two types of build jobs, ``build_test_related_apps`` and ``build_non_test_related_apps``. + +For ``build_test_related_apps``, all the built binaries will be uploaded to our internal MinIO server. You may find the download link in the build report posted in the internal MR. + +For ``build_non_test_related_apps``, all the built binaries will be removed after the build job is finished. Only the build log files will be uploaded to our internal MinIO server. You may also find the download link in the build report posted in the internal MR. + +Target Test Jobs +---------------- + +In CI, all generated target test jobs are named according to the pattern " - ". For example, single-dut test job ``esp32 - generic``, or multi-dut test job ``esp32,esp32 - multi_dut_generic``. + +The binaries in the target test jobs are downloaded from our internal MinIO servers. For most of the test cases, only the files that are required by flash (like .bin files, flash_args files, etc) would be downloaded. For some test cases, like jtag test cases, .elf files are also downloaded. + +Running Tests Locally +===================== + +Installation +------------ + +First you need to install ESP-IDF with additional Python requirements: + +.. code-block:: shell + + $ cd $IDF_PATH + $ bash install.sh --enable-ci --enable-pytest + $ . ./export.sh + +Build Directories +----------------- + +By default, each test case looks for the required binary files in the following directories (in order): + +- ``build__`` +- ``build_`` +- ``build_`` +- ``build`` + +As long as one of the above directories exists, the test case uses that directory to flash the binaries. If non of the above directories exists, the test case fails with an error. + +Test Your Test Script +--------------------- + +Single-DUT Test Cases with ``sdkconfig.defaults`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is the simplest use case. Let's take :project:`examples/get-started/hello_world` as an example. Assume we're testing with a ESP32 board. + +.. code-block:: shell + + $ cd $IDF_PATH/examples/get-started/hello_world + $ idf.py set-target esp32 build + $ pytest --target esp32 + +Single-DUT Test Cases with ``sdkconfig.ci.xxx`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some test cases may need to run with different sdkconfig files. Let's take :project:`examples/system/console/basic` as an example. Assume we're testing with a ESP32 board, and test with ``sdkconfig.ci.history``. + +.. code-block:: shell + + $ cd $IDF_PATH/examples/system/console/basic + $ idf.py -DSDKCONFIG_DEFAULTS='sdkconfig.defaults;sdkconfig.ci.history' -B build_esp32_history set-target esp32 build + $ pytest --target esp32 -k "not nohistory" + +.. note:: + + Here if we use ``pytest --target esp32 -k history``, both test cases will be selected, since ``pytest -k`` will use string matching to filter the test cases. + +If you want to build and test with all sdkconfig files at the same time, you should use our CI script as an helper script: + +.. code-block:: shell + + $ cd $IDF_PATH/examples/system/console/basic + $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -v --pytest-apps + $ pytest --target esp32 + +The app with ``sdkconfig.ci.history`` will be built in ``build_esp32_history``, and the app with ``sdkconfig.ci.nohistory`` will be built in ``build_esp32_nohistory``. ``pytest --target esp32`` will run tests on both apps. + +Multi-DUT Test Cases +^^^^^^^^^^^^^^^^^^^^ + +Some test cases may need to run with multiple DUTs. Let's take :project:`examples/openthread` as an example. The test case function looks like this: + +.. code-block:: python @pytest.mark.parametrize( - 'count, app_path, target', [ - (2, - f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', - 'esp32|esp32s2'), - (2, - f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', - 'esp32s2|esp32'), + 'config, count, app_path, target', [ + ('rcp|cli_h2|br', 3, + f'{os.path.join(os.path.dirname(__file__), "ot_rcp")}' + f'|{os.path.join(os.path.dirname(__file__), "ot_cli")}' + f'|{os.path.join(os.path.dirname(__file__), "ot_br")}', + 'esp32c6|esp32h2|esp32s3'), ], indirect=True, ) - def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None: - softap = dut[0] - station = dut[1] + def test_thread_connect(dut:Tuple[IdfDut, IdfDut, IdfDut]) -> None: ... -Overall, this test function would be replicated to 2 test cases: +The test cases will run with -- softAP with ESP32 target, and station with ESP32-S2 target -- softAP with ESP32-S2 target, and station with ESP32 target +- ESP32-C6, flashed with ``ot_rcp`` +- ESP32-H2, flashed with ``ot_cli`` +- ESP32-S3, flashed with ``ot_br`` -Support Different Targets with Different sdkconfig Files -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Of course we can build the required binaries manually, but we can also use our CI script as an helper script: -This code example is taken from :idf_file:`pytest_panic.py ` as an advanced example. +.. code-block:: shell -.. code:: python + $ cd $IDF_PATH/examples/openthread + $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target all -v --pytest-apps -k test_thread_connect + $ pytest --target esp32c6,esp32h2,esp32s3 -k test_thread_connect - CONFIGS = [ - pytest.param('coredump_flash_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('coredump_flash_elf_sha', marks=[pytest.mark.esp32]), # sha256 only supported on esp32 - pytest.param('coredump_uart_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('coredump_uart_elf_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('gdbstub', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('panic', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - ] +.. important:: - @pytest.mark.parametrize('config', CONFIGS, indirect=True) - ... + It is mandatory to list all the targets for multi-DUT test cases. Otherwise, the test case would fail with an error. + +Debug CI Test Cases +------------------- + +Sometimes you can't reprocude the CI test case failure locally. In this case, you may need to debug the test case with the binaries built in CI. + +Run pytest with ``--pipeline-id `` to force pytest to download the binaries from CI. For example: + +.. code-block:: shell + + $ cd $IDF_PATH/examples/get-started/hello_world + $ pytest --target esp32 --pipeline-id 123456 + +Even if you have ``build_esp32_default``, or ``build`` directory locally, pytest would still download the binaries from pipeline 123456 and place the binaries in ``build_esp32_default``. Then run the test case with this binary. + +Pytest Tips and Tricks +====================== Custom Classes -^^^^^^^^^^^^^^ +-------------- Usually, you may want to write a custom class under these conditions: 1. Add more reusable functions for a certain number of DUTs. -2. Add custom setup and teardown functions in different phases described in Section :ref:`pytest-execution-process`. +2. Add custom setup and teardown functions This code example is taken from :idf_file:`panic/conftest.py `. -.. code:: python +.. code-block:: python class PanicTestDut(IdfDut): ... @@ -417,13 +610,13 @@ This code example is taken from :idf_file:`panic/conftest.py `__ `autouse `__ fixture. This function replaces the ``IdfDut`` class with your custom class. Mark Flaky Tests -^^^^^^^^^^^^^^^^ +---------------- Certain test cases are based on Ethernet or Wi-Fi. However, the test may be flaky due to networking issues. Thus, it is possible to mark a particular test case as flaky. This code example is taken from :idf_file:`pytest_esp_eth.py `. -.. code:: python +.. code-block:: python @pytest.mark.flaky(reruns=3, reruns_delay=5) def test_esp_eth_ip101(dut: IdfDut) -> None: @@ -432,7 +625,7 @@ This code example is taken from :idf_file:`pytest_esp_eth.py ` -.. code:: python +.. code-block:: python @pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None: @@ -451,195 +644,33 @@ This code example is taken from :idf_file:`pytest_panic.py `. - -Running Tests in CI -=================== - -The workflow in CI is simple, build jobs > target test jobs. - -Build Jobs ----------- - -Build Job Names -^^^^^^^^^^^^^^^ - -- Component-based Unit Tests: ``build_pytest_components_`` -- Example Tests: ``build_pytest_examples_`` -- Custom Tests: ``build_pytest_test_apps_`` - -Build Job Commands -^^^^^^^^^^^^^^^^^^ - -The command used by CI to build all the relevant tests is: ``python $IDF_PATH/tools/ci/ci_build_apps.py --target -vv --pytest-apps`` - -All apps which supported the specified target would be built with all supported sdkconfig files under ``build__``. - -For example, If you run ``python $IDF_PATH/tools/ci/ci_build_apps.py $IDF_PATH/examples/system/console/basic --target esp32 --pytest-apps``, the folder structure would be like this: - -.. code:: text - - basic - ├── build_esp32_history/ - │ └── ... - ├── build_esp32_nohistory/ - │ └── ... - ├── main/ - ├── CMakeLists.txt - ├── pytest_console_basic.py - └── ... - -All the build folders would be uploaded as artifacts under the same directories. - -Target Test Jobs ----------------- - -Target Test Job Names -^^^^^^^^^^^^^^^^^^^^^ - -- Component-based Unit Tests: ``component_ut_pytest__`` -- Example Tests: ``example_test_pytest__`` -- Custom Tests: ``test_app_test_pytest__`` - -Target Test Job Commands -^^^^^^^^^^^^^^^^^^^^^^^^ - -The command used by CI to run all the relevant tests is: ``pytest --target -m `` - -All test cases with the specified target marker and the test env marker under the parent folder would be executed. - -The binaries in the target test jobs are downloaded from build jobs. the artifacts would be placed under the same directories. - -Running Tests Locally -===================== - -First you need to install ESP-IDF with additional Python requirements: - -.. code-block:: shell - - $ cd $IDF_PATH - $ bash install.sh --enable-pytest - $ . ./export.sh - -By default, the pytest script will look for the build directory in this order: - -- ``build__`` -- ``build_`` -- ``build_`` -- ``build`` - -Which means, the simplest way to run pytest is calling ``idf.py build``. - -For example, if you want to run all the esp32 tests under the ``$IDF_PATH/examples/get-started/hello_world`` folder, you should run: - -.. code-block:: shell - - $ cd examples/get-started/hello_world - $ idf.py build - $ pytest --target esp32 - -If you have multiple sdkconfig files in your test app, like those ``sdkconfig.ci.*`` files, the simple ``idf.py build`` won't apply the extra sdkconfig files. Let us take ``$IDF_PATH/examples/system/console/basic`` as an example. - -If you want to test this app with config ``history``, and build with ``idf.py build``, you should run - -.. code-block:: shell - - $ cd examples/system/console/basic - $ idf.py -DSDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.ci.history" build - $ pytest --target esp32 --sdkconfig history - -If you want to build and test with all sdkconfig files at the same time, you should use our CI script as an helper script: - -.. code-block:: shell - - $ cd examples/system/console/basic - $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps - $ pytest --target esp32 - -The app with ``sdkconfig.ci.history`` will be built in ``build_esp32_history``, and the app with ``sdkconfig.ci.nohistory`` will be built in ``build_esp32_nohistory``. ``pytest --target esp32`` will run tests on both apps. - -Tips and Tricks -=============== - -.. _filter-the-test-cases: - -Filter the Test Cases ---------------------- - -- Filter by target with ``pytest --target `` - - pytest would run all the test cases that support specified target. - -- Filter by sdkconfig file with ``pytest --sdkconfig `` - - If ```` is ``default``, pytest would run all the test cases with the sdkconfig file ``sdkconfig.defaults``. - - In other cases, pytest would run all the test cases with sdkconfig file ``sdkconfig.ci.``. - -- Filter by test-case name with ``pytest -k `` to run a single test-case, e.g. ``pytest -k test_int_wdt_cache_disabled``. - Add New Markers --------------- We are using two types of custom markers, target markers which indicate that the test cases should support this target, and env markers which indicate that the test cases should be assigned to runners with these tags in CI. -You can add new markers by adding one line under the ``${IDF_PATH}/conftest.py``. If it is a target marker, it should be added into ``TARGET_MARKERS``. If it is a marker that specifies a type of test environment, it should be added into ``ENV_MARKERS``. The syntax should be: ``: ``. - -Generate JUnit Report ---------------------- - -You can call pytest with ``--junitxml `` to generate the JUnit report. In ESP-IDF, the test case name would be unified as ``..: ``. Skip Auto Flash Binary ---------------------- @@ -662,13 +693,12 @@ Sometimes you may need to add some extra logging lines while running the test ca You can use `Python logging module `__ to achieve this. -Useful Logging Functions (as Fixture) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Here are some logging functions provided as fixtures, ``log_performance`` -""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^ -.. code:: python +.. code-block:: python def test_hello_world( dut: IdfDut, @@ -676,9 +706,10 @@ Useful Logging Functions (as Fixture) ) -> None: log_performance('test', 1) + The above example would log the performance item with pre-defined format: ``[performance][test]: 1`` and record it under the ``properties`` tag in the JUnit report if ``--junitxml `` is specified. The JUnit test case node would look like: -.. code:: html +.. code-block:: html @@ -687,11 +718,11 @@ The above example would log the performance item with pre-defined format: ``[per ``check_performance`` -""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^ We provide C macros ``TEST_PERFORMANCE_LESS_THAN`` and ``TEST_PERFORMANCE_GREATER_THAN`` to log the performance item and check if the value is in the valid range. Sometimes the performance item value could not be measured in C code, so we also provide a Python function for the same purpose. Please note that using C macros is the preferred approach, since the Python function could not recognize the threshold values of the same performance item under different ifdef blocks well. -.. code:: python +.. code-block:: python def test_hello_world( dut: IdfDut, @@ -708,4 +739,4 @@ Further Readings ================ - pytest documentation: https://docs.pytest.org/en/latest/contents.html -- pytest-embedded documentation: https://docs.espressif.com/projects/pytest-embedded/en/latest/ \ No newline at end of file +- pytest-embedded documentation: https://docs.espressif.com/projects/pytest-embedded/en/latest/ diff --git a/examples/build_system/cmake/import_prebuilt/main/project_include.cmake b/examples/build_system/cmake/import_prebuilt/main/project_include.cmake index 9bf6e7cb6e..4f262f8e42 100644 --- a/examples/build_system/cmake/import_prebuilt/main/project_include.cmake +++ b/examples/build_system/cmake/import_prebuilt/main/project_include.cmake @@ -1,6 +1,14 @@ # For users checking this example, ignore the following code. This is so that # the prebuilt project is built automatically in ESP-IDF CI. if("$ENV{CI}") + # otherwise these file won't be rebuilt when switching the built target within the same job + file(REMOVE + ${CMAKE_SOURCE_DIR}/prebuilt/sdkconfig + ${CMAKE_SOURCE_DIR}/main/libprebuilt.a + ${CMAKE_SOURCE_DIR}/main/prebuilt.h + ) + file(REMOVE_RECURSE ${CMAKE_SOURCE_DIR}/prebuilt/build) + execute_process(COMMAND ${IDFTOOL} build WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/prebuilt) endif() diff --git a/examples/network/.build-test-rules.yml b/examples/network/.build-test-rules.yml index 93f5b739c1..1285265a53 100644 --- a/examples/network/.build-test-rules.yml +++ b/examples/network/.build-test-rules.yml @@ -18,11 +18,33 @@ examples/network/simple_sniffer: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] temporary: true reason: lack of runners + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - examples/common_components/protocol_examples_common/**/* + - examples/protocols/**/* + - examples/wifi/**/* + - examples/network/simple_sniffer/**/* + - components/mbedtls/port/dynamic/* + - examples/system/ota/**/* + depends_components: + - app_update + - esp_https_ota examples/network/sta2eth: disable: - if: SOC_WIFI_SUPPORTED != 1 + examples/network/vlan_support: disable_test: - if: IDF_TARGET not in ["esp32"] reason: Runner uses esp32 ethernet kit + depends_components: + - esp_eth + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - examples/common_components/**/* + - examples/protocols/**/* + - examples/system/ota/**/* + - examples/ethernet/iperf/**/* + - examples/network/vlan_support/**/* + - components/esp_netif/esp_netif_handlers.c diff --git a/examples/peripherals/.build-test-rules.yml b/examples/peripherals/.build-test-rules.yml index 1941ae6f5a..057af470e4 100644 --- a/examples/peripherals/.build-test-rules.yml +++ b/examples/peripherals/.build-test-rules.yml @@ -1,5 +1,12 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps +.adc_dependencies: &adc_dependencies + depends_components: + - esp_adc + - efuse + - esp_driver_i2s + - esp_driver_spi + .i2c_dependencies: &i2c_dependencies depends_filepatterns: # components @@ -10,10 +17,12 @@ examples/peripherals/adc/continuous_read: disable: - if: SOC_ADC_DMA_SUPPORTED != 1 + <<: *adc_dependencies examples/peripherals/adc/oneshot_read: disable: - if: SOC_ADC_SUPPORTED != 1 + <<: *adc_dependencies examples/peripherals/analog_comparator: disable: @@ -30,6 +39,16 @@ examples/peripherals/dac: disable: - if: SOC_DAC_SUPPORTED != 1 +examples/peripherals/dac/dac_cosine_wave: + disable: + - if: SOC_DAC_SUPPORTED != 1 + depends_components: + - esp_adc + - efuse + - esp_driver_i2s + - esp_driver_spi + - esp_driver_dac + examples/peripherals/gpio: depends_components: - esp_driver_gpio @@ -446,6 +465,21 @@ examples/peripherals/usb: disable: - if: SOC_USB_OTG_SUPPORTED != 1 +examples/peripherals/usb/device: + enable: + - if: SOC_USB_OTG_SUPPORTED == 1 + disable_test: + - if: IDF_TARGET == "esp32s3" + temporary: true + reason: lack of runners + depends_components: + - usb + depends_filepatterns: + - components/hal/usb*.c + - components/hal/esp32*/include/hal/usb*.h + - examples/peripherals/usb/host/**/* + - examples/peripherals/usb/device/**/* + examples/peripherals/usb_serial_jtag/usb_serial_jtag_echo: disable: - if: SOC_USB_SERIAL_JTAG_SUPPORTED != 1 diff --git a/examples/peripherals/usb/device/.build-test-rules.yml b/examples/peripherals/usb/device/.build-test-rules.yml deleted file mode 100644 index 2e88c76571..0000000000 --- a/examples/peripherals/usb/device/.build-test-rules.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps - -examples/peripherals/usb/device: - enable: - - if: SOC_USB_OTG_SUPPORTED == 1 - disable_test: - - if: IDF_TARGET == "esp32s3" - temporary: true - reason: lack of runners diff --git a/examples/protocols/.build-test-rules.yml b/examples/protocols/.build-test-rules.yml index 2b0b52480c..f1ba770c6f 100644 --- a/examples/protocols/.build-test-rules.yml +++ b/examples/protocols/.build-test-rules.yml @@ -1,5 +1,24 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps +.ethernet_dependencies: ðernet_dependencies + # TODO: IDFCI-1821 + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - components/esp_netif/esp_netif_handlers.c + +.wifi_dependencies: &wifi_dependencies + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - examples/common_components/protocol_examples_common/**/* + - examples/protocols/**/* + - examples/wifi/**/* + - examples/network/simple_sniffer/**/* + - components/mbedtls/port/dynamic/* + - examples/system/ota/**/* + depends_components: + - app_update + - esp_https_ota + examples/protocols/esp_http_client: enable: - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux" @@ -11,6 +30,7 @@ examples/protocols/esp_http_client: - if: IDF_TARGET == "esp32p4" temporary: true reason: not supported on p4 + <<: *ethernet_dependencies examples/protocols/esp_local_ctrl: disable: @@ -22,6 +42,7 @@ examples/protocols/esp_local_ctrl: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] temporary: true reason: lack of runners + <<: *wifi_dependencies examples/protocols/http_request: disable: @@ -32,6 +53,7 @@ examples/protocols/http_request: - if: IDF_TARGET != "esp32" temporary: true reason: only test on esp32 + <<: *ethernet_dependencies examples/protocols/http_server: disable: @@ -42,6 +64,7 @@ examples/protocols/http_server: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] temporary: true reason: lack of runners + <<: *wifi_dependencies examples/protocols/http_server/captive_portal: disable: @@ -52,12 +75,14 @@ examples/protocols/http_server/captive_portal: - if: IDF_TARGET != "esp32" temporary: true reason: only test on esp32 + <<: *wifi_dependencies examples/protocols/http_server/restful_server: disable: - if: IDF_TARGET in ["esp32h2", "esp32p4"] temporary: true reason: not supported on p4 # TODO: IDF-8076 + <<: *wifi_dependencies examples/protocols/http_server/ws_echo_server: disable: @@ -68,6 +93,7 @@ examples/protocols/http_server/ws_echo_server: - if: IDF_TARGET != "esp32" temporary: true reason: only test on esp32 + <<: *wifi_dependencies examples/protocols/https_mbedtls: disable: @@ -78,6 +104,7 @@ examples/protocols/https_mbedtls: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/https_request: disable: @@ -88,6 +115,7 @@ examples/protocols/https_request: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/https_server/simple: disable: @@ -98,6 +126,7 @@ examples/protocols/https_server/simple: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] temporary: true reason: lack of runners + <<: *wifi_dependencies examples/protocols/https_server/wss_server: disable: @@ -108,6 +137,7 @@ examples/protocols/https_server/wss_server: - if: IDF_TARGET != "esp32" temporary: true reason: only test on esp32 + <<: *wifi_dependencies examples/protocols/https_x509_bundle: disable: @@ -118,6 +148,7 @@ examples/protocols/https_x509_bundle: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/icmp_echo: disable: @@ -126,6 +157,7 @@ examples/protocols/icmp_echo: reason: not supported on p4 disable_test: - if: SOC_WIFI_SUPPORTED != 1 + <<: *wifi_dependencies examples/protocols/l2tap: disable: @@ -158,6 +190,7 @@ examples/protocols/mqtt/ssl: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/mqtt/ssl_ds: disable: @@ -186,6 +219,7 @@ examples/protocols/mqtt/tcp: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/mqtt/ws: disable: @@ -196,6 +230,7 @@ examples/protocols/mqtt/ws: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/mqtt/wss: disable: @@ -206,6 +241,7 @@ examples/protocols/mqtt/wss: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/mqtt5: disable: @@ -216,6 +252,7 @@ examples/protocols/mqtt5: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/smtp_client: disable: @@ -228,6 +265,7 @@ examples/protocols/sntp: - if: IDF_TARGET == "esp32" temporary: true reason: the other targets are not tested yet + <<: *wifi_dependencies examples/protocols/sockets: disable: @@ -244,12 +282,12 @@ examples/protocols/sockets/non_blocking: examples/protocols/sockets/tcp_client: disable_test: - if: SOC_WIFI_SUPPORTED != 1 - enable: - - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux" + # linux target won't work with CONFIG_EXAMPLE_SOCKET_IP_INPUT_STDIN=y disable: - if: IDF_TARGET == "esp32p4" temporary: true reason: not supported on p4 + <<: *wifi_dependencies examples/protocols/sockets/tcp_server: disable: @@ -258,6 +296,7 @@ examples/protocols/sockets/tcp_server: reason: not supported on p4 disable_test: - if: SOC_WIFI_SUPPORTED != 1 + <<: *wifi_dependencies examples/protocols/sockets/udp_client: disable: @@ -266,6 +305,7 @@ examples/protocols/sockets/udp_client: reason: not supported on p4 disable_test: - if: SOC_WIFI_SUPPORTED != 1 + <<: *wifi_dependencies examples/protocols/sockets/udp_server: disable: @@ -274,6 +314,7 @@ examples/protocols/sockets/udp_server: reason: not supported on p4 disable_test: - if: SOC_WIFI_SUPPORTED != 1 + <<: *wifi_dependencies examples/protocols/static_ip: disable: diff --git a/examples/protocols/l2tap/pytest_example_l2tap_echo.py b/examples/protocols/l2tap/pytest_example_l2tap_echo.py index 7bbf205931..074d9ee383 100644 --- a/examples/protocols/l2tap/pytest_example_l2tap_echo.py +++ b/examples/protocols/l2tap/pytest_example_l2tap_echo.py @@ -101,6 +101,7 @@ def actual_test(dut: Dut) -> None: @pytest.mark.esp32 # internally tested using ESP32 with IP101 but may support all targets with SPI Ethernet @pytest.mark.ip101 +@pytest.mark.temp_skip_ci(targets=['esp32'], reason='runner under maintenance') @pytest.mark.flaky(reruns=3, reruns_delay=5) def test_esp_netif_l2tap_example(dut: Dut) -> None: actual_test(dut) diff --git a/examples/protocols/sockets/tcp_client/README.md b/examples/protocols/sockets/tcp_client/README.md index 89b32f3b0a..59218a487e 100644 --- a/examples/protocols/sockets/tcp_client/README.md +++ b/examples/protocols/sockets/tcp_client/README.md @@ -1,5 +1,5 @@ -| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-S2 | ESP32-S3 | Linux | -| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | ----- | +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | # TCP Client example diff --git a/examples/security/.build-test-rules.yml b/examples/security/.build-test-rules.yml index e24e449691..81f7d70edd 100644 --- a/examples/security/.build-test-rules.yml +++ b/examples/security/.build-test-rules.yml @@ -13,3 +13,8 @@ examples/security/nvs_encryption_hmac: - if: IDF_TARGET not in ["esp32c3"] temporary: true reason: lack of runners + depends_components: + - nvs_flash + - nvs_sec_provider + depends_filepatterns: + - examples/security/nvs_encryption_hmac/**/* diff --git a/examples/storage/littlefs/main/idf_component.yml b/examples/storage/littlefs/main/idf_component.yml index 95c7329ae1..6f83c9dfd6 100644 --- a/examples/storage/littlefs/main/idf_component.yml +++ b/examples/storage/littlefs/main/idf_component.yml @@ -1,6 +1,3 @@ ## IDF Component Manager Manifest File dependencies: - joltwallet/littlefs: "==1.5.5" - ## Required IDF version - idf: - version: ">=5.2.0" + joltwallet/littlefs: "~=1.10.0" diff --git a/examples/system/.build-test-rules.yml b/examples/system/.build-test-rules.yml index 9bc604cda2..bfac822ab9 100644 --- a/examples/system/.build-test-rules.yml +++ b/examples/system/.build-test-rules.yml @@ -1,5 +1,11 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps +.ethernet_dependencies: ðernet_dependencies + # TODO: IDFCI-1821 + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - components/esp_netif/esp_netif_handlers.c + examples/system/app_trace_basic: disable: - if: IDF_TARGET in ["esp32c6", "esp32h2", "esp32p4"] @@ -151,6 +157,21 @@ examples/system/ota/advanced_https_ota: - if: IDF_TARGET == "esp32c2" or IDF_TARGET == "esp32c6" temporary: true reason: lack of runners + depends_filepatterns: + - components/esp_netif/esp_netif_handlers.c + - components/mbedtls/port/dynamic/* + - examples/common_components/**/* + - examples/ethernet/iperf/**/* + - examples/network/simple_sniffer/**/* + - examples/network/vlan_support/**/* + - examples/protocols/**/* + - examples/system/ota/**/* + - examples/wifi/**/* + - tools/ci/python_packages/common_test_methods.py + depends_components: + - app_update + - esp_https_ota + - esp_eth examples/system/ota/native_ota_example: disable: @@ -161,6 +182,7 @@ examples/system/ota/native_ota_example: - if: IDF_TARGET == "esp32c6" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/system/ota/otatool: disable: @@ -173,10 +195,14 @@ examples/system/ota/pre_encrypted_ota: - if: IDF_TARGET in ["esp32h2", "esp32p4"] temporary: true reason: target esp32h2, esp32p4 is not supported yet + - if: CONFIG_NAME == "partial_download" and IDF_TARGET == "esp32c3" + temporary: true + reason: build failed disable_test: - if: IDF_TARGET == "esp32c2" or IDF_TARGET == "esp32c6" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/system/ota/simple_ota_example: disable: @@ -188,6 +214,9 @@ examples/system/ota/simple_ota_example: - if: IDF_TARGET == "esp32c2" or IDF_TARGET == "esp32c6" temporary: true reason: lack of runners + depends_components: + - app_update + - esp_https_ota examples/system/perfmon: enable: diff --git a/examples/zigbee/.build-test-rules.yml b/examples/zigbee/.build-test-rules.yml index d733dc017f..ecffa999bd 100644 --- a/examples/zigbee/.build-test-rules.yml +++ b/examples/zigbee/.build-test-rules.yml @@ -1,8 +1,9 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps .zigbee_dependencies: &zigbee_dependencies + depends_components: + - ieee802154 depends_filepatterns: - - components/ieee802154/**/* - examples/zigbee/light_sample/**/* examples/zigbee/esp_zigbee_gateway: diff --git a/pytest.ini b/pytest.ini index c9c0c43560..08ef25dfe7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,6 +12,7 @@ addopts = --skip-check-coredump y --logfile-extension ".txt" --check-duplicates y + --ignore-glob */managed_components/* # ignore DeprecationWarning filterwarnings = diff --git a/tools/ci/artifacts_handler.py b/tools/ci/artifacts_handler.py index 73930a3a05..c51a69f300 100644 --- a/tools/ci/artifacts_handler.py +++ b/tools/ci/artifacts_handler.py @@ -12,6 +12,7 @@ from pathlib import Path from zipfile import ZipFile import urllib3 +from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME from minio import Minio @@ -33,7 +34,7 @@ TYPE_PATTERNS_DICT = { '**/build*/*.elf', ], ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [ - '**/build*/build_log.txt', + f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}', '**/build*/*.bin', '**/build*/bootloader/*.bin', '**/build*/partition_table/*.bin', @@ -41,17 +42,17 @@ TYPE_PATTERNS_DICT = { '**/build*/flash_project_args', '**/build*/config/sdkconfig.json', '**/build*/project_description.json', - 'list_job_*.txt', + 'list_job*.txt', ], ArtifactType.LOGS: [ - '**/build*/build_log.txt', + f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}', ], ArtifactType.SIZE_REPORTS: [ '**/build*/size.json', 'size_info.txt', ], ArtifactType.JUNIT_REPORTS: [ - 'XUNIT_RESULT.xml', + 'XUNIT_RESULT*.xml', ], ArtifactType.MODIFIED_FILES_AND_COMPONENTS_REPORT: [ 'pipeline.env', @@ -66,6 +67,23 @@ def getenv(env_var: str) -> str: raise Exception(f'Environment variable {env_var} not set') from e +def get_minio_client() -> Minio: + return Minio( + getenv('IDF_S3_SERVER').replace('https://', ''), + access_key=getenv('IDF_S3_ACCESS_KEY'), + secret_key=getenv('IDF_S3_SECRET_KEY'), + http_client=urllib3.PoolManager( + num_pools=10, + timeout=urllib3.Timeout.DEFAULT_TIMEOUT, + retries=urllib3.Retry( + total=5, + backoff_factor=0.2, + status_forcelist=[500, 502, 503, 504], + ), + ), + ) + + def _download_files( pipeline_id: int, *, @@ -131,7 +149,7 @@ def _upload_files( try: if has_file: - obj_name = f'{pipeline_id}/{artifact_type.value}/{job_name.split(" ")[0]}/{job_id}.zip' + obj_name = f'{pipeline_id}/{artifact_type.value}/{job_name.rsplit(" ", maxsplit=1)[0]}/{job_id}.zip' print(f'Created archive file: {job_id}.zip, uploading as {obj_name}') client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, f'{job_id}.zip') @@ -168,19 +186,7 @@ if __name__ == '__main__': args = parser.parse_args() - client = Minio( - getenv('IDF_S3_SERVER').replace('https://', ''), - access_key=getenv('IDF_S3_ACCESS_KEY'), - secret_key=getenv('IDF_S3_SECRET_KEY'), - http_client=urllib3.PoolManager( - timeout=urllib3.Timeout.DEFAULT_TIMEOUT, - retries=urllib3.Retry( - total=5, - backoff_factor=0.2, - status_forcelist=[500, 502, 503, 504], - ), - ), - ) + client = get_minio_client() ci_pipeline_id = args.pipeline_id or getenv('CI_PIPELINE_ID') # required if args.action == 'download': diff --git a/tools/ci/build_template_app.sh b/tools/ci/build_template_app.sh index 9751888cb9..dae77fdd4f 100755 --- a/tools/ci/build_template_app.sh +++ b/tools/ci/build_template_app.sh @@ -60,6 +60,7 @@ build_stage2() { --build-dir ${BUILD_DIR} \ --build-log ${BUILD_LOG_CMAKE} \ --size-file size.json \ + --keep-going \ --collect-size-info size_info.txt \ --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c6 esp32h2 esp32p4 } diff --git a/tools/ci/check_artifacts_expire_time.py b/tools/ci/check_artifacts_expire_time.py deleted file mode 100644 index ff298cac38..0000000000 --- a/tools/ci/check_artifacts_expire_time.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -# internal use only -# check if expire time is set for all artifacts - -import os - -import yaml - -IDF_PATH = os.getenv('IDF_PATH') -if not IDF_PATH: - print('Please set IDF_PATH before running this script') - raise SystemExit(-1) - -GITLAB_CONFIG_FILE = os.path.join(IDF_PATH, '.gitlab-ci.yml') - - -def check_artifacts_expire_time() -> None: - with open(GITLAB_CONFIG_FILE, 'r') as f: - config = yaml.load(f, Loader=yaml.FullLoader) - - # load files listed in `include` - if 'include' in config: - for _file in config['include']: - with open(os.path.join(IDF_PATH or '', _file)) as f: - config.update(yaml.load(f, Loader=yaml.FullLoader)) - - print('expire time for jobs:') - errors = [] - - job_names = list(config.keys()) - job_names.sort() - - for job_name in job_names: - try: - if 'expire_in' not in config[job_name]['artifacts']: - errors.append(job_name) - else: - print('{}: {}'.format(job_name, config[job_name]['artifacts']['expire_in'])) - except (KeyError, TypeError): - # this is not job, or the job does not have artifacts - pass - - if errors: - print('\n\nThe following jobs did not set expire time for its artifacts') - for error in errors: - print(error) - raise SystemExit(-2) - - -if __name__ == '__main__': - check_artifacts_expire_time() diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index f204f0702a..e70fb0e324 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple import yaml -from idf_ci_utils import IDF_PATH +from idf_ci_utils import IDF_PATH, get_all_manifest_files YES = u'\u2713' NO = u'\u2717' @@ -148,9 +148,7 @@ def check_readme( 'all', recursive=True, exclude_list=exclude_dirs or [], - manifest_files=[ - str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml') - ], + manifest_files=get_all_manifest_files(), default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, ) ) @@ -304,9 +302,7 @@ def check_test_scripts( 'all', recursive=True, exclude_list=exclude_dirs or [], - manifest_files=[ - str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml') - ], + manifest_files=get_all_manifest_files(), default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, ) ) @@ -382,7 +378,7 @@ def sort_yaml(files: List[str]) -> None: def check_exist() -> None: exit_code = 0 - config_files = [str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')] + config_files = get_all_manifest_files() for file in config_files: if 'managed_components' in Path(file).parts: continue diff --git a/tools/ci/check_rules_yml.py b/tools/ci/check_rules_yml.py deleted file mode 100755 index 5a55132456..0000000000 --- a/tools/ci/check_rules_yml.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python -# -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -""" -Check if all rules in rules.yml used or not in CI yaml files. -""" - -import argparse -import os -import re -import sys -from copy import deepcopy -from typing import Any, Dict, List, Optional, Set, Union - -import yaml -from idf_ci_utils import IDF_PATH - -ROOT_YML_FP = os.path.join(IDF_PATH, '.gitlab-ci.yml') - - -def load_yaml(file_path: str) -> Any: - return yaml.load(open(file_path), Loader=yaml.FullLoader) - - -class YMLConfig: - def __init__(self, root_yml_file_path: str) -> None: - self._config: Optional[Dict] = None - self._all_extends: Optional[Set] = None - - self.root_yml = load_yaml(root_yml_file_path) - assert self.root_yml - - @staticmethod - def _list(str_or_list: Union[str, List]) -> List: - if isinstance(str_or_list, str): - return [str_or_list] - if isinstance(str_or_list, list): - return str_or_list - raise ValueError( - 'Wrong type: {}. Only supports str or list.'.format(type(str_or_list)) - ) - - @property - def config(self) -> Dict: - if self._config: - return self._config - - all_config = dict() - for item in self.root_yml['include']: - all_config.update(load_yaml(os.path.join(IDF_PATH, item))) - self._config = all_config - return self._config - - @property - def all_extends(self) -> Set: - if self._all_extends: - return self._all_extends - - res = set([]) - for v in self.config.values(): - if 'extends' in v: - for item in self._list(v['extends']): - if item.startswith('.rules:'): - res.add(item) - self._all_extends = res - return self._all_extends - - def exists(self, key: str) -> bool: - if key in self.all_extends: - return True - return False - - -YML_CONFIG = YMLConfig(ROOT_YML_FP) - - -def get_needed_rules() -> Set[str]: - return deepcopy(YML_CONFIG.all_extends) - - -def validate_needed_rules(rules_yml: 'os.PathLike[str]') -> int: - res = 0 - needed_rules = deepcopy(YML_CONFIG.all_extends) - with open(rules_yml) as fr: - for index, line in enumerate(fr): - if line.startswith('.rules:'): - key = line.strip().rsplit(':', 1)[0] - if not YML_CONFIG.exists(key): - print( - '{}:{}:WARNING:rule "{}" unused'.format(rules_yml, index, key) - ) - else: - needed_rules.remove(key) - - if needed_rules: - for item in needed_rules: - print('ERROR: missing rule: "{}"'.format(item)) - res = 1 - - if res == 0: - print('Pass') - return res - - -def parse_submodule_paths( - gitsubmodules: str = os.path.join(IDF_PATH, '.gitmodules') -) -> List[str]: - path_regex = re.compile(r'^\s+path = (.+)$', re.MULTILINE) - with open(gitsubmodules, 'r') as f: - data = f.read() - - res = [] - for item in path_regex.finditer(data): - res.append(item.group(1)) - - return res - - -def validate_submodule_patterns() -> int: - submodule_paths = sorted(['.gitmodules'] + parse_submodule_paths()) - submodule_paths_in_patterns = sorted( - YML_CONFIG.config.get('.patterns-submodule', []) - ) - - res = 0 - if submodule_paths != submodule_paths_in_patterns: - res = 1 - print('please update the pattern ".patterns-submodule"') - should_remove = set(submodule_paths_in_patterns) - set(submodule_paths) - if should_remove: - print(f'- should remove: {should_remove}') - should_add = set(submodule_paths) - set(submodule_paths_in_patterns) - if should_add: - print(f'- should add: {should_add}') - - return res - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - 'rules_yml', - nargs='?', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), - help='rules.yml file path', - ) - args = parser.parse_args() - - exit_code = 0 - if validate_needed_rules(args.rules_yml): - exit_code = 1 - if validate_submodule_patterns(): - exit_code = 1 - - sys.exit(exit_code) diff --git a/tools/ci/check_tools_files_patterns.py b/tools/ci/check_tools_files_patterns.py index 9b6407f451..89e593fa73 100755 --- a/tools/ci/check_tools_files_patterns.py +++ b/tools/ci/check_tools_files_patterns.py @@ -39,7 +39,7 @@ def check(pattern_yml: str, exclude_list: str) -> Tuple[Set, Set]: git_files = get_git_files(os.path.join(IDF_PATH, 'tools'), full_path=True) for f in git_files: f = Path(f) - if f in rules_files_set or f in exclude_files_set: + if f in rules_files_set or f in exclude_files_set or str(f).startswith(os.path.join(IDF_PATH, 'tools', 'test_apps')): continue missing_files.add(os.path.relpath(f, IDF_PATH)) diff --git a/tools/ci/ci_build_apps.py b/tools/ci/ci_build_apps.py index a37de3c34d..d1ef4cec20 100644 --- a/tools/ci/ci_build_apps.py +++ b/tools/ci/ci_build_apps.py @@ -10,13 +10,15 @@ import os import sys import typing as t import unittest -from collections import defaultdict from pathlib import Path import yaml -from idf_build_apps import LOGGER, App, build_apps, find_apps, setup_logging -from idf_build_apps.constants import SUPPORTED_TARGETS -from idf_ci_utils import IDF_PATH +from dynamic_pipelines.constants import DEFAULT_TEST_PATHS +from idf_build_apps import build_apps, setup_logging +from idf_build_apps.utils import semicolon_separated_str_to_list +from idf_pytest.constants import (DEFAULT_BUILD_TEST_RULES_FILEPATH, DEFAULT_CONFIG_RULES_STR, + DEFAULT_FULL_BUILD_TEST_FILEPATTERNS, DEFAULT_IGNORE_WARNING_FILEPATH) +from idf_pytest.script import get_all_apps CI_ENV_VARS = { 'EXTRA_CFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable ' @@ -27,118 +29,6 @@ CI_ENV_VARS = { } -def get_pytest_apps( - paths: t.List[str], - target: str, - config_rules_str: t.List[str], - marker_expr: str, - filter_expr: str, - preserve_all: bool = False, - extra_default_build_targets: t.Optional[t.List[str]] = None, - modified_components: t.Optional[t.List[str]] = None, - modified_files: t.Optional[t.List[str]] = None, - ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, -) -> t.List[App]: - from idf_pytest.script import get_pytest_cases - - pytest_cases = get_pytest_cases(paths, target, marker_expr, filter_expr) - - _paths: t.Set[str] = set() - test_related_app_configs = defaultdict(set) - for case in pytest_cases: - for app in case.apps: - _paths.add(app.path) - test_related_app_configs[app.path].add(app.config) - - if not extra_default_build_targets: - extra_default_build_targets = [] - - app_dirs = list(_paths) - if not app_dirs: - raise RuntimeError('No apps found') - - LOGGER.info(f'Found {len(app_dirs)} apps') - app_dirs.sort() - - apps = find_apps( - app_dirs, - target=target, - build_dir='build_@t_@w', - config_rules_str=config_rules_str, - build_log_path='build_log.txt', - size_json_path='size.json', - check_warnings=True, - manifest_rootpath=IDF_PATH, - manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')], - default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - modified_components=modified_components, - modified_files=modified_files, - ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, - ) - - for app in apps: - is_test_related = app.config_name in test_related_app_configs[app.app_dir] - if not preserve_all and not is_test_related: - app.preserve = False - - if app.target == 'linux': - app._size_json_path = None # no esp_idf_size for linux target - - return apps # type: ignore - - -def get_cmake_apps( - paths: t.List[str], - target: str, - config_rules_str: t.List[str], - preserve_all: bool = False, - extra_default_build_targets: t.Optional[t.List[str]] = None, - modified_components: t.Optional[t.List[str]] = None, - modified_files: t.Optional[t.List[str]] = None, - ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, -) -> t.List[App]: - from idf_pytest.constants import PytestApp - from idf_pytest.script import get_pytest_cases - - apps = find_apps( - paths, - recursive=True, - target=target, - build_dir='build_@t_@w', - config_rules_str=config_rules_str, - build_log_path='build_log.txt', - size_json_path='size.json', - check_warnings=True, - preserve=False, - manifest_rootpath=IDF_PATH, - manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')], - default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - modified_components=modified_components, - modified_files=modified_files, - ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, - ) - - apps_for_build = [] - pytest_cases_apps = [app for case in get_pytest_cases(paths, target) for app in case.apps] - for app in apps: - if preserve_all: # relpath - app.preserve = True - - if PytestApp(os.path.realpath(app.app_dir), app.target, app.config_name) in pytest_cases_apps: - LOGGER.debug('Skipping build app with pytest scripts: %s', app) - continue - - if app.target == 'linux': - app._size_json_path = None # no esp_idf_size for linux target - - apps_for_build.append(app) - - return apps_for_build - - -APPS_BUILD_PER_JOB = 30 - - def main(args: argparse.Namespace) -> None: extra_default_build_targets: t.List[str] = [] if args.default_build_test_rules: @@ -148,39 +38,24 @@ def main(args: argparse.Namespace) -> None: if configs: extra_default_build_targets = configs.get('extra_default_build_targets') or [] - if args.pytest_apps: - LOGGER.info('Only build apps with pytest scripts') - apps = get_pytest_apps( - args.paths, - args.target, - args.config, - args.marker_expr, - args.filter_expr, - args.preserve_all, - extra_default_build_targets, - args.modified_components, - args.modified_files, - args.ignore_app_dependencies_filepatterns, - ) - else: - LOGGER.info('build apps. will skip pytest apps with pytest scripts') - apps = get_cmake_apps( - args.paths, - args.target, - args.config, - args.preserve_all, - extra_default_build_targets, - args.modified_components, - args.modified_files, - args.ignore_app_dependencies_filepatterns, - ) - - LOGGER.info('Found %d apps after filtering', len(apps)) - LOGGER.info( - 'Suggest setting the parallel count to %d for this build job', - len(apps) // APPS_BUILD_PER_JOB + 1, + test_related_apps, non_test_related_apps = get_all_apps( + args.paths, + args.target, + config_rules_str=args.config, + marker_expr=args.marker_expr, + filter_expr=args.filter_expr, + preserve_all=args.preserve_all, + extra_default_build_targets=extra_default_build_targets, + modified_files=args.modified_components, + modified_components=args.modified_files, + ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns, ) + if args.pytest_apps: + apps = test_related_apps + else: + apps = non_test_related_apps + if args.extra_preserve_dirs: for app in apps: if app.preserve: @@ -192,7 +67,7 @@ def main(args: argparse.Namespace) -> None: app.preserve = True res = build_apps( - apps, + sorted(apps), parallel_count=args.parallel_count, parallel_index=args.parallel_index, dry_run=False, @@ -206,12 +81,10 @@ def main(args: argparse.Namespace) -> None: modified_components=args.modified_components, modified_files=args.modified_files, ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns, + junitxml=args.junitxml, ) - if isinstance(res, tuple): - sys.exit(res[0]) - else: - sys.exit(res) + sys.exit(res) if __name__ == '__main__': @@ -219,7 +92,7 @@ if __name__ == '__main__': description='Build all the apps for different test types. Will auto remove those non-test apps binaries', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument('paths', nargs='+', help='Paths to the apps to build.') + parser.add_argument('paths', nargs='*', help='Paths to the apps to build.') parser.add_argument( '-t', '--target', @@ -228,7 +101,7 @@ if __name__ == '__main__': ) parser.add_argument( '--config', - default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], + default=DEFAULT_CONFIG_RULES_STR, nargs='+', help='Adds configurations (sdkconfig file names) to build. This can either be ' 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' @@ -272,7 +145,7 @@ if __name__ == '__main__': ) parser.add_argument( '--ignore-warning-file', - default=os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt'), + default=DEFAULT_IGNORE_WARNING_FILEPATH, type=argparse.FileType('r'), help='Ignore the warning strings in the specified file. Each line should be a regex string.', ) @@ -290,7 +163,8 @@ if __name__ == '__main__': parser.add_argument( '--pytest-apps', action='store_true', - help='Only build apps with pytest scripts. Will build apps without pytest scripts if this flag is unspecified.', + help='Only build apps required by pytest scripts. ' + 'Will build non-test-related apps if this flag is unspecified.', ) parser.add_argument( '-m', @@ -307,7 +181,7 @@ if __name__ == '__main__': ) parser.add_argument( '--default-build-test-rules', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), + default=DEFAULT_BUILD_TEST_RULES_FILEPATH, help='default build test rules config file', ) parser.add_argument( @@ -318,69 +192,64 @@ if __name__ == '__main__': ) parser.add_argument( '--modified-components', - nargs='*', - default=None, - help='space-separated list which specifies the modified components. app with `depends_components` set in the ' - 'corresponding manifest files would only be built if depends on any of the specified components.', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified components. ' + 'app with `depends_components` set in the corresponding manifest files would only be built ' + 'if depends on any of the specified components. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', ) parser.add_argument( '--modified-files', - nargs='*', - default=None, - help='space-separated list which specifies the modified files. app with `depends_filepatterns` set in the ' - 'corresponding manifest files would only be built if any of the specified file pattern matches any of the ' - 'specified modified files.', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified files. ' + 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' + 'if any of the specified file pattern matches any of the specified modified files. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', ) parser.add_argument( '-if', '--ignore-app-dependencies-filepatterns', - nargs='*', - default=None, - help='space-separated list which specifies the file patterns used for ignoring checking the app dependencies. ' - 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored when any of the ' - 'specified file patterns matches any of the modified files. Must be used together with --modified-files', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the file patterns used for ' + 'ignoring checking the app dependencies. ' + 'The `depends_components` and `depends_filepatterns` set in the manifest files ' + 'will be ignored when any of the specified file patterns matches any of the modified files. ' + 'Must be used together with --modified-files. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + parser.add_argument( + '--junitxml', + default='build_summary_@p.xml', + help='Path to the junitxml file. If specified, the junitxml file will be generated', ) arguments = parser.parse_args() setup_logging(arguments.verbose) + # set default paths + if not arguments.paths: + arguments.paths = DEFAULT_TEST_PATHS + # skip setting flags in CI if not arguments.skip_setting_flags and not os.getenv('CI_JOB_ID'): for _k, _v in CI_ENV_VARS.items(): os.environ[_k] = _v - LOGGER.info(f'env var {_k} set to "{_v}"') + print(f'env var {_k} set to "{_v}"') if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': # if it's not MR pipeline or env var BUILD_AND_TEST_ALL_APPS=1, # remove component dependency related arguments - if 'modified_components' in arguments: - arguments.modified_components = None - if 'modified_files' in arguments: - arguments.modified_files = None + arguments.modified_components = None + arguments.modified_files = None + arguments.ignore_app_dependencies_filepatterns = None - # file patterns to tigger full build - if 'modified_components' in arguments and not arguments.ignore_app_dependencies_filepatterns: - arguments.ignore_app_dependencies_filepatterns = [ - # tools - 'tools/cmake/**/*', - 'tools/tools.json', - # components - 'components/cxx/**/*', - 'components/esp_common/**/*', - 'components/esp_hw_support/**/*', - 'components/esp_rom/**/*', - 'components/esp_system/**/*', - 'components/esp_timer/**/*', - 'components/freertos/**/*', - 'components/hal/**/*', - 'components/heap/**/*', - 'components/log/**/*', - 'components/newlib/**/*', - 'components/riscv/**/*', - 'components/soc/**/*', - 'components/xtensa/**/*', - ] + # default file patterns to tigger full build + if arguments.modified_files is not None and arguments.ignore_app_dependencies_filepatterns is None: + arguments.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS main(arguments) diff --git a/tools/ci/dynamic_pipelines/__init__.py b/tools/ci/dynamic_pipelines/__init__.py new file mode 100644 index 0000000000..28b2a3c5e3 --- /dev/null +++ b/tools/ci/dynamic_pipelines/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +tools_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..')) +if tools_dir not in sys.path: + sys.path.append(tools_dir) diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py new file mode 100644 index 0000000000..a90e6769f2 --- /dev/null +++ b/tools/ci/dynamic_pipelines/constants.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os + +from idf_ci_utils import IDF_PATH + +# use relative path to avoid absolute path in pipeline +DEFAULT_TEST_PATHS = [ + 'examples', + os.path.join('tools', 'test_apps'), + 'components', +] + +DEFAULT_APPS_BUILD_PER_JOB = 60 +DEFAULT_CASES_TEST_PER_JOB = 60 + +DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'build_child_pipeline.yml') +DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'target_test_child_pipeline.yml') + +TEST_RELATED_BUILD_JOB_NAME = 'build_test_related_apps' +NON_TEST_RELATED_BUILD_JOB_NAME = 'build_non_test_related_apps' + +COMMENT_START_MARKER = '### Dynamic Pipeline Report' +TEST_RELATED_APPS_FILENAME = 'test_related_apps.txt' +NON_TEST_RELATED_APPS_FILENAME = 'non_test_related_apps.txt' + +TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME = 'test_related_apps_download_urls.yml' +REPORT_TEMPLATE_FILEPATH = os.path.join( + IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html' +) diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py new file mode 100644 index 0000000000..a71fa2830c --- /dev/null +++ b/tools/ci/dynamic_pipelines/models.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import inspect +import typing as t +from dataclasses import dataclass +from xml.etree.ElementTree import Element + +import yaml + + +class Job: + def __init__( + self, + *, + name: str, + extends: t.Optional[t.List[str]] = None, + tags: t.Optional[t.List[str]] = None, + stage: t.Optional[str] = None, + parallel: int = 1, + variables: t.Dict[str, str] = None, + script: t.Optional[t.List[str]] = None, + before_script: t.Optional[t.List[str]] = None, + after_script: t.Optional[t.List[str]] = None, + needs: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + self.name = name + self.extends = extends + self.tags = tags + self.stage = stage + self.parallel = parallel + self.variables = variables or {} + self.script = script + self.before_script = before_script + self.after_script = after_script + self.needs = needs + + for k, v in kwargs.items(): + setattr(self, k, v) + + def __str__(self) -> str: + return yaml.dump(self.to_dict()) # type: ignore + + def set_variable(self, key: str, value: str) -> None: + self.variables[key] = value + + def to_dict(self) -> t.Dict[str, t.Any]: + res = {} + for k, v in inspect.getmembers(self): + if k.startswith('_'): + continue + + # name is the dict key + if k == 'name': + continue + + # parallel 1 is not allowed + if k == 'parallel' and v == 1: + continue + + if v is None: + continue + + if inspect.ismethod(v) or inspect.isfunction(v): + continue + + res[k] = v + + return {self.name: res} + + +class EmptyJob(Job): + def __init__( + self, + *, + name: t.Optional[str] = None, + tags: t.Optional[t.List[str]] = None, + stage: t.Optional[str] = None, + before_script: t.Optional[t.List[str]] = None, + after_script: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + super().__init__( + name=name or 'fake_pass_job', + tags=tags or ['build', 'shiny'], + stage=stage or 'build', + script=['echo "This is a fake job to pass the pipeline"'], + before_script=before_script or [], + after_script=after_script or [], + **kwargs, + ) + + +class BuildJob(Job): + def __init__( + self, + *, + extends: t.Optional[t.List[str]] = None, + tags: t.Optional[t.List[str]] = None, + stage: t.Optional[str] = None, + **kwargs: t.Any, + ) -> None: + super().__init__( + extends=extends or ['.dynamic_build_template'], + tags=tags or ['build', 'shiny'], + stage=stage or 'build', + **kwargs, + ) + + +class TargetTestJob(Job): + def __init__( + self, + *, + extends: t.Optional[t.List[str]] = None, + stage: t.Optional[str] = None, + **kwargs: t.Any, + ) -> None: + super().__init__( + extends=extends or ['.dynamic_target_test_template'], + stage=stage or 'target_test', + **kwargs, + ) + + +@dataclass +class TestCase: + name: str + file: str + time: float + failure: t.Optional[str] = None + skipped: t.Optional[str] = None + ci_job_url: t.Optional[str] = None + + @property + def is_failure(self) -> bool: + return self.failure is not None + + @property + def is_skipped(self) -> bool: + return self.skipped is not None + + @property + def is_success(self) -> bool: + return not self.is_failure and not self.is_skipped + + @classmethod + def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']: + if 'name' not in node.attrib: + print('WARNING: Node Invalid: ', node) + return None + + kwargs = { + 'name': node.attrib['name'], + 'file': node.attrib.get('file'), + 'time': float(node.attrib.get('time') or 0), + 'ci_job_url': node.attrib.get('ci_job_url') or '', + } + + failure_node = node.find('failure') + if failure_node is not None: + kwargs['failure'] = failure_node.attrib['message'] + + skipped_node = node.find('skipped') + if skipped_node is not None: + kwargs['skipped'] = skipped_node.attrib['message'] + + return cls(**kwargs) # type: ignore diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py new file mode 100644 index 0000000000..5ed2ab2605 --- /dev/null +++ b/tools/ci/dynamic_pipelines/report.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import abc +import html +import os +import re +import typing as t + +import yaml +from artifacts_handler import ArtifactType +from gitlab_api import Gitlab +from idf_build_apps import App +from idf_build_apps.constants import BuildStatus +from idf_ci.uploader import AppUploader +from prettytable import PrettyTable + +from .constants import COMMENT_START_MARKER, REPORT_TEMPLATE_FILEPATH, TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME +from .models import TestCase + + +class ReportGenerator: + REGEX_PATTERN = '#### {}[^####]+' + + def __init__(self, project_id: int, mr_iid: int, pipeline_id: int, *, title: str): + gl_project = Gitlab(project_id).project + if mr_iid is not None: + self.mr = gl_project.mergerequests.get(mr_iid) + else: + self.mr = None + self.pipeline_id = pipeline_id + + self.title = title + self.output_filepath = self.title.lower().replace(' ', '_') + '.html' + + @staticmethod + def get_download_link_for_url(url: str) -> str: + if url: + return f'Download' + + return '' + + def generate_html_report(self, table_str: str) -> str: + # we're using bootstrap table + table_str = table_str.replace('', '
') + + with open(REPORT_TEMPLATE_FILEPATH) as fr: + template = fr.read() + + return template.replace('{{title}}', self.title).replace('{{table}}', table_str) + + @staticmethod + def table_to_html_str(table: PrettyTable) -> str: + return html.unescape(table.get_html_string()) # type: ignore + + @abc.abstractmethod + def _get_report_str(self) -> str: + raise NotImplementedError + + def post_report(self, job_id: int, commit_id: str) -> None: + # report in html format, otherwise will exceed the limit + with open(self.output_filepath, 'w') as fw: + fw.write(self._get_report_str()) + + # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt + # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` + url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf') + + comment = f'''#### {self.title} + +Full {self.title} here: {url}/-/jobs/{job_id}/artifacts/{self.output_filepath} (with commit {commit_id}) + +''' + + if self.mr is None: + print('No MR found, skip posting comment') + return + + for note in self.mr.notes.list(iterator=True): + if note.body.startswith(COMMENT_START_MARKER): + updated_str = re.sub(self.REGEX_PATTERN.format(self.title), comment, note.body) + if updated_str == note.body: # not updated + updated_str = f'{note.body.strip()}\n\n{comment}' + + note.body = updated_str + note.save() + break + else: + new_comment = f'''{COMMENT_START_MARKER} + +{comment}''' + self.mr.notes.create({'body': new_comment}) + + +class BuildReportGenerator(ReportGenerator): + def __init__( + self, + project_id: int, + mr_iid: int, + pipeline_id: int, + *, + title: str = 'Build Report', + apps: t.List[App], + ): + super().__init__(project_id, mr_iid, pipeline_id, title=title) + self.apps = apps + + self.apps_presigned_url_filepath = TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME + + def _get_report_str(self) -> str: + if not self.apps: + print('No apps found, skip generating build report') + return 'No Apps Built' + + uploader = AppUploader(self.pipeline_id) + + table_str = '' + + failed_apps = [app for app in self.apps if app.build_status == BuildStatus.FAILED] + if failed_apps: + table_str += '

Failed Apps

' + + failed_apps_table = PrettyTable() + failed_apps_table.field_names = [ + 'App Dir', + 'Build Dir', + 'Failed Reason', + 'Build Log', + ] + for app in failed_apps: + failed_apps_table.add_row( + [ + app.app_dir, + app.build_dir, + app.build_comment or '', + self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), + ] + ) + + table_str += self.table_to_html_str(failed_apps_table) + + built_test_related_apps = [app for app in self.apps if app.build_status == BuildStatus.SUCCESS and app.preserve] + if built_test_related_apps: + table_str += '

Built Apps (Test Related)

' + + built_apps_table = PrettyTable() + built_apps_table.field_names = [ + 'App Dir', + 'Build Dir', + 'Bin Files with Build Log (without map and elf)', + 'Map and Elf Files', + ] + app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = {} + for app in built_test_related_apps: + _d = { + ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES.value: uploader.get_app_presigned_url( + app, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES + ), + ArtifactType.MAP_AND_ELF_FILES.value: uploader.get_app_presigned_url( + app, ArtifactType.MAP_AND_ELF_FILES + ), + } + + built_apps_table.add_row( + [ + app.app_dir, + app.build_dir, + self.get_download_link_for_url(_d[ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES]), + self.get_download_link_for_url(_d[ArtifactType.MAP_AND_ELF_FILES]), + ] + ) + + app_presigned_urls_dict[app.build_path] = _d + + # also generate a yaml file that includes the apps and the presigned urls + # for helping debugging locally + with open(self.apps_presigned_url_filepath, 'w') as fw: + yaml.dump(app_presigned_urls_dict, fw) + + table_str += self.table_to_html_str(built_apps_table) + + built_non_test_related_apps = [ + app for app in self.apps if app.build_status == BuildStatus.SUCCESS and not app.preserve + ] + if built_non_test_related_apps: + table_str += '

Built Apps (Non Test Related)

' + + built_apps_table = PrettyTable() + built_apps_table.field_names = [ + 'App Dir', + 'Build Dir', + 'Build Log', + ] + for app in built_non_test_related_apps: + built_apps_table.add_row( + [ + app.app_dir, + app.build_dir, + self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), + ] + ) + + table_str += self.table_to_html_str(built_apps_table) + + skipped_apps = [app for app in self.apps if app.build_status == BuildStatus.SKIPPED] + if skipped_apps: + table_str += '

Skipped Apps

' + + skipped_apps_table = PrettyTable() + skipped_apps_table.field_names = ['App Dir', 'Build Dir', 'Skipped Reason', 'Build Log'] + for app in skipped_apps: + skipped_apps_table.add_row( + [ + app.app_dir, + app.build_dir, + app.build_comment or '', + self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), + ] + ) + + table_str += self.table_to_html_str(skipped_apps_table) + + return self.generate_html_report(table_str) + + +class TargetTestReportGenerator(ReportGenerator): + def __init__( + self, + project_id: int, + mr_iid: int, + pipeline_id: int, + *, + title: str = 'Target Test Report', + test_cases: t.List[TestCase], + ): + super().__init__(project_id, mr_iid, pipeline_id, title=title) + + self.test_cases = test_cases + + def _get_report_str(self) -> str: + table_str = '' + + failed_test_cases = [tc for tc in self.test_cases if tc.is_failure] + if failed_test_cases: + table_str += '

Failed Test Cases

' + + failed_test_cases_table = PrettyTable() + failed_test_cases_table.field_names = ['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL'] + for tc in failed_test_cases: + failed_test_cases_table.add_row([tc.name, tc.file, tc.failure, tc.ci_job_url]) + + table_str += self.table_to_html_str(failed_test_cases_table) + + skipped_test_cases = [tc for tc in self.test_cases if tc.is_skipped] + if skipped_test_cases: + table_str += '

Skipped Test Cases

' + + skipped_test_cases_table = PrettyTable() + skipped_test_cases_table.field_names = ['Test Case', 'Test Script File Path', 'Skipped Reason'] + for tc in skipped_test_cases: + skipped_test_cases_table.add_row([tc.name, tc.file, tc.skipped]) + + table_str += self.table_to_html_str(skipped_test_cases_table) + + successful_test_cases = [tc for tc in self.test_cases if tc.is_success] + if successful_test_cases: + table_str += '

Succeeded Test Cases

' + + successful_test_cases_table = PrettyTable() + successful_test_cases_table.field_names = ['Test Case', 'Test Script File Path', 'Job URL'] + for tc in successful_test_cases: + successful_test_cases_table.add_row([tc.name, tc.file, tc.ci_job_url]) + + table_str += self.table_to_html_str(successful_test_cases_table) + + return self.generate_html_report(table_str) diff --git a/tools/ci/dynamic_pipelines/scripts/__init__.py b/tools/ci/dynamic_pipelines/scripts/__init__.py new file mode 100644 index 0000000000..2ef7c132ff --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/__init__.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +IDF_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) + +# run this scripts only in idf path, to ensure the relative path is the same +os.chdir(IDF_PATH) + +if 'IDF_PATH' not in os.environ: + os.environ['IDF_PATH'] = IDF_PATH + +tools_path = os.path.join(os.path.dirname(__file__), '..', '..', '..') +if tools_path not in sys.path: + sys.path.append(tools_path) + +tools_ci_path = os.path.join(os.path.dirname(__file__), '..', '..') +if tools_ci_path not in sys.path: + sys.path.append(tools_ci_path) + +tools_ci_python_packages_path = os.path.join(os.path.dirname(__file__), '..', '..', 'python_packages') +if tools_ci_python_packages_path not in sys.path: + sys.path.append(tools_ci_python_packages_path) diff --git a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py new file mode 100644 index 0000000000..cf4d3ebe9c --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import sys + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME +from idf_build_apps import build_apps, setup_logging +from idf_build_apps.utils import semicolon_separated_str_to_list +from idf_ci.app import import_apps_from_txt +from idf_pytest.constants import DEFAULT_IGNORE_WARNING_FILEPATH + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Build Apps for Dynamic Pipeline') + parser.add_argument('app_list_file', default=TEST_RELATED_APPS_FILENAME, help='List of apps to build') + parser.add_argument( + '--build-verbose', + action='store_true', + help='Enable verbose output from build system.', + ) + parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.') + parser.add_argument( + '--parallel-index', + default=1, + type=int, + help='Index (1-based) of the job, out of the number specified by --parallel-count.', + ) + parser.add_argument( + '--ignore-warning-file', + default=DEFAULT_IGNORE_WARNING_FILEPATH, + type=argparse.FileType('r'), + help='Ignore the warning strings in the specified file. Each line should be a regex string.', + ) + parser.add_argument( + '--modified-components', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified components. ' + 'app with `depends_components` set in the corresponding manifest files would only be built ' + 'if depends on any of the specified components. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + parser.add_argument( + '--collect-app-info', + default='list_job_@p.txt', + help='If specified, the test case name and app info json will be written to this file', + ) + parser.add_argument( + '--junitxml', + default='build_summary_@p.xml', + help='Path to the junitxml file. If specified, the junitxml file will be generated', + ) + + args = parser.parse_args() + + setup_logging(verbose=1) + + sys.exit( + build_apps( + import_apps_from_txt(args.app_list_file), + build_verbose=args.build_verbose, + keep_going=True, + ignore_warning_file=args.ignore_warning_file, + modified_components=args.modified_components, + check_app_dependencies=True, + parallel_count=args.parallel_count, + parallel_index=args.parallel_index, + collect_size_info='size_info_@p.txt', + collect_app_info=args.collect_app_info, + junitxml=args.junitxml, + ) + ) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py new file mode 100644 index 0000000000..5194d73671 --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +"""This file is used for generating the child pipeline for build jobs.""" + +import argparse +import os +import typing as t + +import __init__ # noqa: F401 # inject the system path +import yaml +from dynamic_pipelines.constants import (DEFAULT_APPS_BUILD_PER_JOB, DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, + DEFAULT_TEST_PATHS, NON_TEST_RELATED_APPS_FILENAME, + NON_TEST_RELATED_BUILD_JOB_NAME, TEST_RELATED_APPS_FILENAME, + TEST_RELATED_BUILD_JOB_NAME) +from dynamic_pipelines.models import BuildJob, EmptyJob +from dynamic_pipelines.utils import dump_jobs_to_yaml +from idf_build_apps.utils import semicolon_separated_str_to_list +from idf_ci.app import dump_apps_to_txt +from idf_ci_utils import IDF_PATH +from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR, DEFAULT_FULL_BUILD_TEST_FILEPATTERNS, CollectMode +from idf_pytest.script import get_all_apps + + +def main(arguments: argparse.Namespace) -> None: + # load from default build test rules config file + extra_default_build_targets: t.List[str] = [] + if arguments.default_build_test_rules: + with open(arguments.default_build_test_rules) as fr: + configs = yaml.safe_load(fr) + + if configs: + extra_default_build_targets = configs.get('extra_default_build_targets') or [] + + build_jobs = [] + ########################################### + # special case with -k, ignore other args # + ########################################### + if arguments.filter_expr: + # build only test related apps + test_related_apps, _ = get_all_apps( + arguments.paths, + target=CollectMode.ALL, + config_rules_str=DEFAULT_CONFIG_RULES_STR, + filter_expr=arguments.filter_expr, + marker_expr='not host_test', + extra_default_build_targets=extra_default_build_targets, + ) + dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) + print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') + + test_apps_build_job = BuildJob( + name=TEST_RELATED_BUILD_JOB_NAME, + parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, + variables={ + 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, + }, + ) + + build_jobs.append(test_apps_build_job) + else: + ############# + # all cases # + ############# + test_related_apps, non_test_related_apps = get_all_apps( + arguments.paths, + CollectMode.ALL, + marker_expr='not host_test', + config_rules_str=DEFAULT_CONFIG_RULES_STR, + extra_default_build_targets=extra_default_build_targets, + modified_components=arguments.modified_components, + modified_files=arguments.modified_files, + ignore_app_dependencies_filepatterns=arguments.ignore_app_dependencies_filepatterns, + ) + + dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) + print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') + dump_apps_to_txt(sorted(non_test_related_apps), NON_TEST_RELATED_APPS_FILENAME) + print( + f'Generate non-test related apps file {NON_TEST_RELATED_APPS_FILENAME} with {len(non_test_related_apps)} apps' + ) + + if test_related_apps: + test_apps_build_job = BuildJob( + name=TEST_RELATED_BUILD_JOB_NAME, + parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, + variables={ + 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, + }, + ) + build_jobs.append(test_apps_build_job) + + if non_test_related_apps: + non_test_apps_build_job = BuildJob( + name=NON_TEST_RELATED_BUILD_JOB_NAME, + parallel=len(non_test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, + variables={ + 'APP_LIST_FILE': NON_TEST_RELATED_APPS_FILENAME, + }, + ) + build_jobs.append(non_test_apps_build_job) + + # check if there's no jobs + if not build_jobs: + print('No apps need to be built. Create one empty job instead') + build_jobs.append(EmptyJob()) + extra_include_yml = [] + else: + extra_include_yml = ['tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml'] + + dump_jobs_to_yaml(build_jobs, arguments.yaml_output, extra_include_yml) + print(f'Generate child pipeline yaml file {arguments.yaml_output} with {sum(j.parallel for j in build_jobs)} jobs') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Generate build child pipeline', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '-o', + '--yaml-output', + default=DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, + help='Output YAML path', + ) + parser.add_argument( + '-p', + '--paths', + nargs='+', + default=DEFAULT_TEST_PATHS, + help='Paths to the apps to build.', + ) + parser.add_argument( + '-k', + '--filter-expr', + help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only' + 'for pytest', + ) + parser.add_argument( + '--default-build-test-rules', + default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), + help='default build test rules config file', + ) + parser.add_argument( + '--modified-components', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified components. ' + 'app with `depends_components` set in the corresponding manifest files would only be built ' + 'if depends on any of the specified components. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + parser.add_argument( + '--modified-files', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified files. ' + 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' + 'if any of the specified file pattern matches any of the specified modified files. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + parser.add_argument( + '-if', + '--ignore-app-dependencies-filepatterns', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the file patterns used for ' + 'ignoring checking the app dependencies. ' + 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' + 'when any of the specified file patterns matches any of the modified files. ' + 'Must be used together with --modified-files. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + + args = parser.parse_args() + + if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': + print('Build and run all test cases, and compile all cmake apps') + args.modified_components = None + args.modified_files = None + args.ignore_app_dependencies_filepatterns = None + elif args.filter_expr is not None: + print('Build and run only test cases matching "%s"' % args.filter_expr) + args.modified_components = None + args.modified_files = None + args.ignore_app_dependencies_filepatterns = None + else: + print('Build and run only test cases matching the modified components and files') + if args.modified_files and not args.ignore_app_dependencies_filepatterns: + # setting default values + args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS + + main(args) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_report.py b/tools/ci/dynamic_pipelines/scripts/generate_build_report.py new file mode 100644 index 0000000000..cef16271fe --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_report.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import glob +import os + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.report import BuildReportGenerator +from idf_ci.app import import_apps_from_txt + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Update Build Report in MR pipelines', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '--project-id', + type=int, + default=os.getenv('CI_PROJECT_ID'), + help='Project ID', + ) + parser.add_argument( + '--mr-iid', + type=int, + default=os.getenv('CI_MERGE_REQUEST_IID'), + help='Merge Request IID', + ) + parser.add_argument( + '--pipeline-id', + type=int, + default=os.getenv('PARENT_PIPELINE_ID'), + help='Pipeline ID', + ) + parser.add_argument( + '--job-id', + type=int, + default=os.getenv('CI_JOB_ID'), + help='Job ID', + ) + parser.add_argument( + '--commit-id', + default=os.getenv('CI_COMMIT_SHORT_SHA'), + help='MR commit ID', + ) + parser.add_argument( + '--app-list-filepattern', + default='list_job_*.txt', + help='App list file pattern', + ) + + args = parser.parse_args() + + apps = [] + for f in glob.glob(args.app_list_filepattern): + apps.extend(import_apps_from_txt(f)) + + report_generator = BuildReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, apps=apps) + report_generator.post_report(args.job_id, args.commit_id) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py new file mode 100644 index 0000000000..28e8d335cd --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +"""This file is used for generating the child pipeline for target test jobs. + +1. Check the build jobs' artifacts to get the built apps' information. +2. Post the Build Report if it's running in an MR pipeline. +3. Generate the child pipeline for target test jobs. +""" + +import argparse +import glob +import os +import typing as t +from collections import Counter, defaultdict + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.constants import (DEFAULT_CASES_TEST_PER_JOB, DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH, + DEFAULT_TEST_PATHS) +from dynamic_pipelines.models import EmptyJob, Job, TargetTestJob +from dynamic_pipelines.utils import dump_jobs_to_yaml +from gitlab.v4.objects import Project +from gitlab_api import Gitlab +from idf_build_apps import App +from idf_ci.app import import_apps_from_txt +from idf_pytest.script import get_pytest_cases + + +def get_tags_with_amount(s: str) -> t.List[str]: + c: Counter = Counter() + for _t in s.split(','): + c[_t] += 1 + + res = set() + for target, amount in c.items(): + if amount > 1: + res.add(f'{target}_{amount}') + else: + res.add(target) + + return sorted(res) + + +def generate_target_test_child_pipeline(project: Project, paths: str, apps: t.List[App], output_filepath: str) -> None: + pytest_cases = get_pytest_cases( + paths, + apps=apps, + marker_expr='not host_test', # since it's generating target-test child pipeline + ) + + res = defaultdict(list) + for case in pytest_cases: + if not case.env_markers: + print(f'No env markers found for {case.item.originalname} in {case.path}. Ignoring...') + continue + + res[(case.target_selector, tuple(sorted(case.env_markers)))].append(case) + + target_test_jobs: t.List[Job] = [] + for (target_selector, env_markers), cases in res.items(): + runner_tags = get_tags_with_amount(target_selector) + list(env_markers) + # we don't need to get all runner, as long as we get one runner, it's fine + runner_list = project.runners.list(status='online', tag_list=','.join(runner_tags), get_all=False) + if not runner_list: + print(f'WARNING: No runner found with tag {",".join(runner_tags)}, ignoring the following test cases:') + for case in cases: + print(f' - {case.name}') + continue + + target_test_job = TargetTestJob( + name=f'{target_selector} - {",".join(env_markers)}', + tags=runner_tags, + parallel=len(cases) // DEFAULT_CASES_TEST_PER_JOB + 1, + ) + target_test_job.set_variable('TARGET_SELECTOR', f"'{target_selector}'") + target_test_job.set_variable('ENV_MARKERS', "'" + ' and '.join(env_markers) + "'") + target_test_job.set_variable('PYTEST_NODES', ' '.join([f"'{case.item.nodeid}'" for case in cases])) + + target_test_jobs.append(target_test_job) + + if not target_test_jobs: + print('No target test cases required, create one empty job instead') + target_test_jobs.append(EmptyJob()) + extra_include_yml = [] + else: + extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml'] + + dump_jobs_to_yaml(target_test_jobs, output_filepath, extra_include_yml) + print(f'Generate child pipeline yaml file {output_filepath} with {sum(j.parallel for j in target_test_jobs)} jobs') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Generate Target Test Child Pipeline. Update Build Report in MR pipelines', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '-p', + '--paths', + nargs='+', + default=DEFAULT_TEST_PATHS, + help='Paths to the apps to build.', + ) + parser.add_argument( + '--project-id', + type=int, + default=os.getenv('CI_PROJECT_ID'), + help='Project ID', + ) + parser.add_argument( + '--pipeline-id', + type=int, + default=os.getenv('PARENT_PIPELINE_ID'), + help='Pipeline ID', + ) + parser.add_argument( + '-o', + '--output', + default=DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH, + help='Output child pipeline file path', + ) + + args = parser.parse_args() + + app_list_filepattern = 'list_job_*.txt' + apps = [] + for f in glob.glob(app_list_filepattern): + apps.extend(import_apps_from_txt(f)) + + gl_project = Gitlab(args.project_id).project + generate_target_test_child_pipeline(gl_project, args.paths, apps, args.output) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py new file mode 100644 index 0000000000..cd736ad9f1 --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import glob +import os +import xml.etree.ElementTree as ET + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.models import TestCase +from dynamic_pipelines.report import TargetTestReportGenerator + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Update Build Report in MR pipelines', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '--project-id', + type=int, + default=os.getenv('CI_PROJECT_ID'), + help='Project ID', + ) + parser.add_argument( + '--mr-iid', + type=int, + default=os.getenv('CI_MERGE_REQUEST_IID'), + help='Merge Request IID', + ) + parser.add_argument( + '--pipeline-id', + type=int, + default=os.getenv('PARENT_PIPELINE_ID'), + help='Pipeline ID', + ) + parser.add_argument( + '--job-id', + type=int, + default=os.getenv('CI_JOB_ID'), + help='Job ID', + ) + parser.add_argument( + '--commit-id', + default=os.getenv('CI_COMMIT_SHORT_SHA'), + help='MR commit ID', + ) + parser.add_argument( + '--junit-report-filepattern', + default='XUNIT_RESULT*.xml', + help='Junit Report file pattern', + ) + + args = parser.parse_args() + + test_cases = [] + for f in glob.glob(args.junit_report_filepattern): + root = ET.parse(f).getroot() + for tc in root.findall('.//testcase'): + test_cases.append(TestCase.from_test_case_node(tc)) + + report_generator = TargetTestReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, test_cases=test_cases) + report_generator.post_report(args.job_id, args.commit_id) diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml new file mode 100644 index 0000000000..781a97490d --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -0,0 +1,85 @@ +# This file is used to generate build jobs for pytest case dynamic pipeline +# don't add real jobs in this file + +######################## +# Build Jobs Templates # +######################## +.dynamic_build_template: + extends: + - .before_script:build + - .after_script:build:ccache:upload-when-fail + image: $ESP_ENV_IMAGE + stage: build + variables: + # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. + IDF_CCACHE_ENABLE: "1" + needs: + - pipeline: $PARENT_PIPELINE_ID + job: generate_build_child_pipeline + artifacts: + paths: + # The other artifacts patterns are defined under tools/ci/artifacts_handler.py + # Now we're uploading/downloading the binary files from our internal storage server + # + # keep the log file to help debug + - "**/build*/build_log.txt" + # build spec files + - build_summary_*.xml + # list of built apps + - list_job_*.txt + when: always + expire_in: 1 week + script: + # CI specific options start from "--parallel-count xxx". could ignore when running locally + - run_cmd python tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py $APP_LIST_FILE + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" + --modified-components ${MR_MODIFIED_COMPONENTS} + --junitxml "build_summary_${CI_JOB_NAME_SLUG}.xml" + +.dynamic_target_test_template: + extends: + - .before_script:fetch:target_test + image: $TARGET_TEST_ENV_IMAGE + stage: target_test + timeout: 1 hour + variables: + SUBMODULES_TO_FETCH: "none" + # set while generating the pipeline + PYTEST_NODES: "" + TARGET_SELECTOR: "" + ENV_MARKERS: "" + cache: + # Usually do not need submodule-cache in target_test + - key: pip-cache-${LATEST_GIT_TAG} + paths: + - .cache/pip + policy: pull + artifacts: + paths: + - XUNIT_RESULT*.xml + - pytest_embedded_log/ +# Child pipeline reports won't be collected in the main one +# https://gitlab.com/groups/gitlab-org/-/epics/8205 +# reports: +# junit: XUNIT_RESULT.xml + script: + # get known failure cases + - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases + # get runner env config file + - retry_failed git clone $TEST_ENV_CONFIG_REPO + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs + # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally + - run_cmd pytest ${PYTEST_NODES} + --target ${TARGET_SELECTOR} + -m ${ENV_MARKERS} + --pipeline-id $PARENT_PIPELINE_ID + --junitxml=XUNIT_RESULT_${CI_JOB_NAME_SLUG}.xml + --ignore-result-files known_failure_cases/known_failure_cases.txt + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + ${PYTEST_EXTRA_FLAGS} + --app-info-filepattern "list_job_*.txt" + after_script: + - python tools/ci/artifacts_handler.py upload --type logs junit_reports diff --git a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml new file mode 100644 index 0000000000..62cba4d77f --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml @@ -0,0 +1,10 @@ +generate_pytest_report: + stage: .post + tags: [build, shiny] + image: $ESP_ENV_IMAGE + when: always + artifacts: + paths: + - target_test_report.html + script: + - python tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html new file mode 100644 index 0000000000..f8da79ff7e --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/report.template.html @@ -0,0 +1,23 @@ + + + + + {{title}} + + + + + + +
{{table}}
+ + + + + diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml new file mode 100644 index 0000000000..1b531790eb --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -0,0 +1,41 @@ +generate_pytest_build_report: + stage: assign_test + image: $ESP_ENV_IMAGE + tags: + - build + - shiny + when: always + artifacts: + paths: + - build_report.html + - test_related_apps_download_urls.yml + script: + - python tools/ci/dynamic_pipelines/scripts/generate_build_report.py + +generate_pytest_child_pipeline: + # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage + stage: assign_test + image: $ESP_ENV_IMAGE + tags: + - build + - shiny + artifacts: + paths: + - target_test_child_pipeline.yml + script: + - python tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py + +Pytest Target Test Jobs: + stage: target_test + needs: + - generate_pytest_child_pipeline + variables: + PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID + # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 + inherit: + variables: false + trigger: + include: + - artifact: target_test_child_pipeline.yml + job: generate_pytest_child_pipeline + strategy: depend diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py new file mode 100644 index 0000000000..25fee44caa --- /dev/null +++ b/tools/ci/dynamic_pipelines/utils.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import typing as t + +import yaml + +from .models import Job + + +def dump_jobs_to_yaml( + jobs: t.List[Job], output_filepath: str, extra_include_yml: t.Optional[t.List[str]] = None +) -> None: + yaml_dict = {} + for job in jobs: + yaml_dict.update(job.to_dict()) + + # global stuffs + yaml_dict.update( + { + 'include': [ + 'tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml', + '.gitlab/ci/common.yml', + ], + # https://gitlab.com/gitlab-org/gitlab/-/issues/222370#note_662695503 + 'workflow': { + 'rules': [ + {'if': '$CI_MERGE_REQUEST_IID'}, + {'if': '$CI_COMMIT_BRANCH'}, + ], + }, + } + ) + yaml_dict['include'].extend(extra_include_yml or []) + + with open(output_filepath, 'w') as fw: + yaml.dump(yaml_dict, fw, indent=2) diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index c801f3f43c..f8986d11aa 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -40,3 +40,12 @@ tools/templates/sample_component/main.c tools/ci/cleanup_ignore_lists.py tools/ci/artifacts_handler.py tools/unit-test-app/**/* +tools/ci/gitlab_yaml_linter.py +tools/ci/dynamic_pipelines/**/* +tools/ci/idf_ci/**/* +tools/ci/get_supported_examples.sh +tools/ci/python_packages/common_test_methods.py +tools/ci/python_packages/gitlab_api.py +tools/ci/python_packages/idf_http_server_test/**/* +tools/ci/python_packages/idf_iperf_test_util/**/* +tools/esp_prov/**/* diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 04d372832e..2fce0de3c0 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -62,7 +62,6 @@ tools/ci/check_kconfigs.py tools/ci/check_readme_links.py tools/ci/check_requirement_files.py tools/ci/check_rules_components_patterns.py -tools/ci/check_rules_yml.py tools/ci/check_soc_struct_headers.py tools/ci/check_tools_files_patterns.py tools/ci/check_type_comments.py @@ -74,6 +73,7 @@ tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py tools/ci/get-full-sources.sh tools/ci/get_supported_examples.sh +tools/ci/gitlab_yaml_linter.py tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh tools/ci/push_to_github.sh diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index 1d37ac69c6..072c22f974 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse @@ -11,8 +11,7 @@ from collections import defaultdict from itertools import product import yaml -from check_rules_yml import get_needed_rules -from idf_ci_utils import IDF_PATH +from idf_ci_utils import IDF_PATH, GitlabYmlConfig try: import pygraphviz as pgv @@ -100,6 +99,7 @@ class RulesWriter: self.cfg = self.expand_matrices() self.rules = self.expand_rules() + self.yml_config = GitlabYmlConfig() self.graph = None def expand_matrices(self): # type: () -> dict @@ -201,7 +201,7 @@ class RulesWriter: def new_rules_str(self): # type: () -> str res = [] for k, v in sorted(self.rules.items()): - if '.rules:' + k not in get_needed_rules(): + if '.rules:' + k not in self.yml_config.used_rules: print(f'WARNING: unused rule: {k}, skipping...') continue res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) diff --git a/tools/ci/gitlab_yaml_linter.py b/tools/ci/gitlab_yaml_linter.py new file mode 100755 index 0000000000..0f9750f0d5 --- /dev/null +++ b/tools/ci/gitlab_yaml_linter.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +""" +Check gitlab ci yaml files +""" + +import argparse +import os +import typing as t +from functools import cached_property + +from idf_ci_utils import IDF_PATH, GitlabYmlConfig, get_submodule_dirs + + +class YmlLinter: + def __init__(self, yml_config: GitlabYmlConfig) -> None: + self.yml_config = yml_config + + self._errors: t.List[str] = [] + + @cached_property + def lint_functions(self) -> t.List[str]: + funcs = [] + for func in dir(self): + if func.startswith('_lint_'): + funcs.append(func) + + return funcs + + def lint(self) -> None: + exit_code = 0 + + for func in self.lint_functions: + getattr(self, func)() + + if self._errors: + print(f'Errors found while running {func}:') + exit_code = 1 + print('\t- ' + '\n\t- '.join(self._errors)) + self._errors = [] # reset + + exit(exit_code) + + # name it like _1_ to make it run first + def _lint_1_yml_parser(self) -> None: + for k, v in self.yml_config.config.items(): + if ( + k not in self.yml_config.global_keys + and k not in self.yml_config.anchors + and k not in self.yml_config.jobs + ): + raise SystemExit(f'Parser incorrect. Key {k} not in global keys, rules or jobs') + + def _lint_default_values_artifacts(self) -> None: + defaults_artifacts = self.yml_config.default.get('artifacts', {}) + + for job_name, d in self.yml_config.jobs.items(): + for k, v in d.get('artifacts', {}).items(): + if k not in defaults_artifacts: + continue + + if v == defaults_artifacts[k]: + self._errors.append(f'job {job_name} key {k} has same value as default value {v}') + + def _lint_submodule_patterns(self) -> None: + submodule_paths = sorted(['.gitmodules'] + get_submodule_dirs()) + submodule_paths_in_patterns = sorted(self.yml_config.config.get('.patterns-submodule', [])) + + if submodule_paths != submodule_paths_in_patterns: + unused_patterns = set(submodule_paths_in_patterns) - set(submodule_paths) + if unused_patterns: + for item in unused_patterns: + self._errors.append(f'non-exist pattern {item}. Please remove {item} from .patterns-submodule') + undefined_patterns = set(submodule_paths) - set(submodule_paths_in_patterns) + if undefined_patterns: + for item in undefined_patterns: + self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule') + + def _lint_gitlab_yml_rules(self) -> None: + unused_rules = self.yml_config.rules - self.yml_config.used_rules + for item in unused_rules: + self._errors.append(f'Unused rule: {item}, please remove it') + undefined_rules = self.yml_config.used_rules - self.yml_config.rules + for item in undefined_rules: + self._errors.append(f'Undefined rule: {item}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--root-yml-filepath', help='root yml file path', default=os.path.join(IDF_PATH, '.gitlab-ci.yml') + ) + args = parser.parse_args() + + config = GitlabYmlConfig(args.root_yml_filepath) + linter = YmlLinter(config) + linter.lint() diff --git a/tools/ci/idf_ci/__init__.py b/tools/ci/idf_ci/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci/idf_ci/app.py b/tools/ci/idf_ci/app.py new file mode 100644 index 0000000000..7ade06a817 --- /dev/null +++ b/tools/ci/idf_ci/app.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import sys +import typing as t +from typing import Literal + +from idf_build_apps import App, CMakeApp, json_to_app +from idf_ci.uploader import AppUploader, get_app_uploader + + +class IdfCMakeApp(CMakeApp): + uploader: t.ClassVar[t.Optional['AppUploader']] = get_app_uploader() + build_system: Literal['idf_cmake'] = 'idf_cmake' + + def _post_build(self) -> None: + super()._post_build() + + if self.uploader: + self.uploader.upload_app(self.build_path) + + +def dump_apps_to_txt(apps: t.List[App], output_filepath: str) -> None: + with open(output_filepath, 'w') as fw: + for app in apps: + fw.write(app.model_dump_json() + '\n') + + +def import_apps_from_txt(input_filepath: str) -> t.List[App]: + apps: t.List[App] = [] + with open(input_filepath) as fr: + for line in fr: + if line := line.strip(): + try: + apps.append(json_to_app(line, extra_classes=[IdfCMakeApp])) + except Exception: # noqa + print('Failed to deserialize app from line: %s' % line) + sys.exit(1) + + return apps diff --git a/tools/ci/idf_ci/uploader.py b/tools/ci/idf_ci/uploader.py new file mode 100644 index 0000000000..95b6819d57 --- /dev/null +++ b/tools/ci/idf_ci/uploader.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import glob +import os +import typing as t +from datetime import timedelta +from zipfile import ZIP_DEFLATED, ZipFile + +import minio +from artifacts_handler import ArtifactType, get_minio_client, getenv +from idf_build_apps import App +from idf_build_apps.utils import rmdir +from idf_ci_utils import IDF_PATH +from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME + + +class AppUploader: + TYPE_PATTERNS_DICT = { + ArtifactType.MAP_AND_ELF_FILES: [ + 'bootloader/*.map', + 'bootloader/*.elf', + '*.map', + '*.elf', + ], + ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [ + '*.bin', + 'bootloader/*.bin', + 'partition_table/*.bin', + 'flasher_args.json', + 'flash_project_args', + 'config/sdkconfig.json', + 'project_description.json', + ], + ArtifactType.LOGS: [ + DEFAULT_BUILD_LOG_FILENAME, + ], + } + + def __init__(self, pipeline_id: t.Union[str, int, None] = None) -> None: + self.pipeline_id = str(pipeline_id or '1') + + self._client = get_minio_client() + + def get_app_object_name(self, app_path: str, zip_name: str, artifact_type: ArtifactType) -> str: + return f'{self.pipeline_id}/{artifact_type.value}/{app_path}/{zip_name}' + + def _upload_app(self, app_build_path: str, artifact_type: ArtifactType) -> bool: + app_path, build_dir = os.path.split(app_build_path) + zip_filename = f'{build_dir}.zip' + + has_file = False + with ZipFile( + zip_filename, + 'w', + compression=ZIP_DEFLATED, + # 1 is the fastest compression level + # the size differs not much between 1 and 9 + compresslevel=1, + ) as zw: + for pattern in self.TYPE_PATTERNS_DICT[artifact_type]: + for file in glob.glob(os.path.join(app_build_path, pattern), recursive=True): + zw.write(file) + has_file = True + + uploaded = False + try: + if has_file: + obj_name = self.get_app_object_name(app_path, zip_filename, artifact_type) + print(f'Created archive file: {zip_filename}, uploading as {obj_name}') + self._client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, zip_filename) + uploaded = True + finally: + os.remove(zip_filename) + return uploaded + + def upload_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: + uploaded = False + if not artifact_type: + for _artifact_type in [ + ArtifactType.MAP_AND_ELF_FILES, + ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES, + ArtifactType.LOGS, + ]: + uploaded |= self._upload_app(app_build_path, _artifact_type) + else: + uploaded = self._upload_app(app_build_path, artifact_type) + + if uploaded: + rmdir(app_build_path, exclude_file_patterns=DEFAULT_BUILD_LOG_FILENAME) + + def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: + app_path, build_dir = os.path.split(app_build_path) + zip_filename = f'{build_dir}.zip' + + # path are relative to IDF_PATH + current_dir = os.getcwd() + os.chdir(IDF_PATH) + try: + obj_name = self.get_app_object_name(app_path, zip_filename, artifact_type) + print(f'Downloading {obj_name}') + try: + try: + self._client.stat_object(getenv('IDF_S3_BUCKET'), obj_name) + except minio.error.S3Error as e: + raise SystemExit( + f'No such file on minio server: {obj_name}. ' + f'Probably the build failed or the artifacts got expired. ' + f'Full error message: {str(e)}' + ) + else: + self._client.fget_object(getenv('IDF_S3_BUCKET'), obj_name, zip_filename) + print(f'Downloaded to {zip_filename}') + except minio.error.S3Error as e: + raise SystemExit('Shouldn\'t happen, please report this bug in the CI channel' + str(e)) + + with ZipFile(zip_filename, 'r') as zr: + zr.extractall() + + os.remove(zip_filename) + finally: + os.chdir(current_dir) + + def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: + if not artifact_type: + for _artifact_type in [ArtifactType.MAP_AND_ELF_FILES, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES]: + self._download_app(app_build_path, _artifact_type) + else: + self._download_app(app_build_path, artifact_type) + + def get_app_presigned_url(self, app: App, artifact_type: ArtifactType) -> str: + obj_name = self.get_app_object_name(app.app_dir, f'{app.build_dir}.zip', artifact_type) + try: + self._client.stat_object( + getenv('IDF_S3_BUCKET'), + obj_name, + ) + except minio.error.S3Error: + return '' + else: + return self._client.get_presigned_url( # type: ignore + 'GET', getenv('IDF_S3_BUCKET'), obj_name, expires=timedelta(days=4) + ) + + +def get_app_uploader() -> t.Optional['AppUploader']: + if parent_pipeline_id := os.getenv('PARENT_PIPELINE_ID'): + return AppUploader(parent_pipeline_id) + + return None diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 1776766eaf..9ddc40ded5 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -8,12 +8,14 @@ import logging import os import subprocess import sys -from typing import Any, List +import typing as t +from functools import cached_property +from pathlib import Path IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) -def get_submodule_dirs(full_path: bool = False) -> List[str]: +def get_submodule_dirs(full_path: bool = False) -> t.List[str]: """ To avoid issue could be introduced by multi-os or additional dependency, we use python and git to get this output @@ -71,7 +73,7 @@ def is_executable(full_path: str) -> bool: return os.access(full_path, os.X_OK) -def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]: +def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: """ Get the result of git ls-files :param path: path to run git ls-files @@ -98,11 +100,10 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]: return [os.path.join(path, f) for f in files] if full_path else files -def is_in_directory(file_path: str, folder: str) -> bool: - return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep) +def to_list(s: t.Any) -> t.List[t.Any]: + if not s: + return [] - -def to_list(s: Any) -> List[Any]: if isinstance(s, (set, tuple)): return list(s) @@ -110,3 +111,83 @@ def to_list(s: Any) -> List[Any]: return s return [s] + + +class GitlabYmlConfig: + def __init__(self, root_yml_filepath: str = os.path.join(IDF_PATH, '.gitlab-ci.yml')) -> None: + self._config: t.Dict[str, t.Any] = {} + self._defaults: t.Dict[str, t.Any] = {} + + self._load(root_yml_filepath) + + def _load(self, root_yml_filepath: str) -> None: + # avoid unused import in other pre-commit hooks + import yaml + + all_config = dict() + root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader) + for item in root_yml['include']: + all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) + + if 'default' in all_config: + self._defaults = all_config.pop('default') + + self._config = all_config + + @property + def default(self) -> t.Dict[str, t.Any]: + return self._defaults + + @property + def config(self) -> t.Dict[str, t.Any]: + return self._config + + @cached_property + def global_keys(self) -> t.List[str]: + return ['default', 'include', 'workflow', 'variables', 'stages'] + + @cached_property + def anchors(self) -> t.Dict[str, t.Any]: + return {k: v for k, v in self.config.items() if k.startswith('.')} + + @cached_property + def jobs(self) -> t.Dict[str, t.Any]: + return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys} + + @cached_property + def rules(self) -> t.Set[str]: + return {k for k, _ in self.anchors.items() if self._is_rule_key(k)} + + @cached_property + def used_rules(self) -> t.Set[str]: + res = set() + + for v in self.config.values(): + if not isinstance(v, dict): + continue + + for item in to_list(v.get('extends')): + if self._is_rule_key(item): + res.add(item) + + return res + + @staticmethod + def _is_rule_key(key: str) -> bool: + return key.startswith('.rules:') or key.endswith('template') + + +def get_all_manifest_files() -> t.List[str]: + """ + + :rtype: object + """ + paths: t.List[str] = [] + + for p in Path(IDF_PATH).glob('**/.build-test-rules.yml'): + if 'managed_components' in p.parts: + continue + + paths.append(str(p)) + + return paths diff --git a/tools/ci/idf_pytest/constants.py b/tools/ci/idf_pytest/constants.py index f5bb8743ae..2eec0efda1 100644 --- a/tools/ci/idf_pytest/constants.py +++ b/tools/ci/idf_pytest/constants.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """ @@ -7,8 +7,12 @@ Pytest Related Constants. Don't import third-party packages here. import os import typing as t from dataclasses import dataclass +from enum import Enum +from functools import cached_property +from pathlib import Path from _pytest.python import Function +from idf_ci_utils import IDF_PATH from pytest_embedded.utils import to_list SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2', 'esp32c6', 'esp32h2', 'esp32p4'] @@ -35,10 +39,11 @@ SPECIAL_MARKERS = { 'temp_skip': 'temp skip tests for specified targets both in ci and locally', 'nightly_run': 'tests should be executed as part of the nightly trigger pipeline', 'host_test': 'tests which should not be built at the build stage, and instead built in host_test stage', - 'qemu': 'build and test using qemu-system-xtensa, not real target', } ENV_MARKERS = { + # special markers + 'qemu': 'build and test using qemu, not real target', # single-dut markers 'generic': 'tests should be run on generic runners', 'flash_suspend': 'support flash suspend feature', @@ -89,7 +94,6 @@ ENV_MARKERS = { 'adc': 'ADC related tests should run on adc runners', 'xtal32k': 'Runner with external 32k crystal connected', 'no32kXtal': 'Runner with no external 32k crystal connected', - 'multi_dut_modbus_rs485': 'a pair of runners connected by RS485 bus', 'psramv0': 'Runner with PSRAM version 0', 'esp32eco3': 'Runner with esp32 eco3 connected', 'ecdsa_efuse': 'Runner with test ECDSA private keys programmed in efuse', @@ -98,6 +102,7 @@ ENV_MARKERS = { 'i2c_oled': 'Runner with ssd1306 I2C oled connected', 'httpbin': 'runner for tests that need to access the httpbin service', # multi-dut markers + 'multi_dut_modbus_rs485': 'a pair of runners connected by RS485 bus', 'ieee802154': 'ieee802154 related tests should run on ieee802154 runners.', 'openthread_br': 'tests should be used for openthread border router.', 'openthread_bbr': 'tests should be used for openthread border router linked to Internet.', @@ -109,9 +114,41 @@ ENV_MARKERS = { 'sdio_master_slave': 'Test sdio multi board, esp32+esp32', 'sdio_multidev_32_c6': 'Test sdio multi board, esp32+esp32c6', 'usj_device': 'Test usb_serial_jtag and usb_serial_jtag is used as serial only (not console)', - 'twai_std': 'twai runner with all twai supported targets connect to usb-can adapter' + 'twai_std': 'twai runner with all twai supported targets connect to usb-can adapter', } +DEFAULT_CONFIG_RULES_STR = ['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'] +DEFAULT_IGNORE_WARNING_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt') +DEFAULT_BUILD_TEST_RULES_FILEPATH = os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml') +DEFAULT_FULL_BUILD_TEST_FILEPATTERNS = [ + # tools + 'tools/cmake/**/*', + 'tools/tools.json', + # components + 'components/cxx/**/*', + 'components/esp_common/**/*', + 'components/esp_hw_support/**/*', + 'components/esp_rom/**/*', + 'components/esp_system/**/*', + 'components/esp_timer/**/*', + 'components/freertos/**/*', + 'components/hal/**/*', + 'components/heap/**/*', + 'components/log/**/*', + 'components/newlib/**/*', + 'components/riscv/**/*', + 'components/soc/**/*', + 'components/xtensa/**/*', +] +DEFAULT_BUILD_LOG_FILENAME = 'build_log.txt' + + +class CollectMode(str, Enum): + SINGLE_SPECIFIC = 'single_specific' + MULTI_SPECIFIC = 'multi_specific' + MULTI_ALL_WITH_PARAM = 'multi_all_with_param' + ALL = 'all' + @dataclass class PytestApp: @@ -122,38 +159,47 @@ class PytestApp: def __hash__(self) -> int: return hash((self.path, self.target, self.config)) + @cached_property + def build_dir(self) -> str: + return os.path.join(self.path, f'build_{self.target}_{self.config}') + @dataclass class PytestCase: - path: str - name: str - - apps: t.Set[PytestApp] - target: str + apps: t.List[PytestApp] item: Function def __hash__(self) -> int: return hash((self.path, self.name, self.apps, self.all_markers)) + @cached_property + def path(self) -> str: + return str(self.item.path) + + @cached_property + def name(self) -> str: + return self.item.originalname # type: ignore + + @cached_property + def targets(self) -> t.List[str]: + return [app.target for app in self.apps] + + @cached_property + def is_single_dut_test_case(self) -> bool: + return True if len(self.apps) == 1 else False + + @cached_property + def is_host_test(self) -> bool: + return 'host_test' in self.all_markers or 'linux' in self.targets + + # the following markers could be changed dynamically, don't use cached_property @property def all_markers(self) -> t.Set[str]: return {marker.name for marker in self.item.iter_markers()} - @property - def is_nightly_run(self) -> bool: - return 'nightly_run' in self.all_markers - @property def target_markers(self) -> t.Set[str]: - return {marker for marker in self.all_markers if marker in TARGET_MARKERS} - - @property - def env_markers(self) -> t.Set[str]: - return {marker for marker in self.all_markers if marker in ENV_MARKERS} - - @property - def skipped_targets(self) -> t.Set[str]: def _get_temp_markers_disabled_targets(marker_name: str) -> t.Set[str]: temp_marker = self.item.get_closest_marker(marker_name) @@ -179,4 +225,65 @@ class PytestCase: else: # we use `temp_skip` locally skip_targets = temp_skip_targets - return skip_targets + return {marker for marker in self.all_markers if marker in TARGET_MARKERS} - skip_targets + + @property + def env_markers(self) -> t.Set[str]: + return {marker for marker in self.all_markers if marker in ENV_MARKERS} + + @property + def target_selector(self) -> str: + return ','.join(app.target for app in self.apps) + + @property + def requires_elf_or_map(self) -> bool: + """ + This property determines whether the test case requires elf or map file. By default, one app in the test case + only requires .bin files. + + :return: True if the test case requires elf or map file, False otherwise + """ + if 'jtag' in self.env_markers or 'usb_serial_jtag' in self.env_markers: + return True + + if any('panic' in Path(app.path).parts for app in self.apps): + return True + + return False + + def all_built_in_app_lists(self, app_lists: t.Optional[t.List[str]] = None) -> t.Optional[str]: + """ + Check if all binaries of the test case are built in the app lists. + + :param app_lists: app lists to check + :return: debug string if not all binaries are built in the app lists, None otherwise + """ + if app_lists is None: + # ignore this feature + return None + + bin_found = [0] * len(self.apps) + for i, app in enumerate(self.apps): + if app.build_dir in app_lists: + bin_found[i] = 1 + + if sum(bin_found) == 0: + msg = f'Skip test case {self.name} because all following binaries are not listed in the app lists: ' + for app in self.apps: + msg += f'\n - {app.build_dir}' + + print(msg) + return msg + + if sum(bin_found) == len(self.apps): + return None + + # some found, some not, looks suspicious + msg = f'Found some binaries of test case {self.name} are not listed in the app lists.' + for i, app in enumerate(self.apps): + if bin_found[i] == 0: + msg += f'\n - {app.build_dir}' + + msg += '\nMight be a issue of .build-test-rules.yml files' + print(msg) + return msg diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index c5c5576a05..af9e61e247 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,9 +1,10 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import logging import os import typing as t +from collections import defaultdict +from functools import cached_property from xml.etree import ElementTree as ET import pytest @@ -11,17 +12,20 @@ from _pytest.config import ExitCode from _pytest.main import Session from _pytest.python import Function from _pytest.runner import CallInfo +from idf_build_apps import App +from idf_build_apps.constants import BuildStatus from pytest_embedded import Dut from pytest_embedded.plugin import parse_multi_dut_args from pytest_embedded.utils import find_by_suffix, to_list from pytest_ignore_test_results.ignore_results import ChildCase, ChildCasesStashKey -from .constants import DEFAULT_SDKCONFIG, PREVIEW_TARGETS, SUPPORTED_TARGETS, PytestApp, PytestCase -from .utils import format_case_id, merge_junit_files +from .constants import DEFAULT_SDKCONFIG, PREVIEW_TARGETS, SUPPORTED_TARGETS, CollectMode, PytestApp, PytestCase +from .utils import comma_sep_str_to_list, format_case_id, merge_junit_files IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']() ITEM_FAILED_CASES_KEY = pytest.StashKey[list]() ITEM_FAILED_KEY = pytest.StashKey[bool]() +ITEM_PYTEST_CASE_KEY = pytest.StashKey[PytestCase]() class IdfPytestEmbedded: @@ -33,80 +37,133 @@ class IdfPytestEmbedded: def __init__( self, - target: str, - sdkconfig: t.Optional[str] = None, - apps_list: t.Optional[t.List[str]] = None, + target: t.Union[t.List[str], str], + *, + single_target_duplicate_mode: bool = False, + apps: t.Optional[t.List[App]] = None, ): - # CLI options to filter the test cases - self.target = target.lower() - self.sdkconfig = sdkconfig - self.apps_list = apps_list + if isinstance(target, str): + self.target = sorted(comma_sep_str_to_list(target)) + else: + self.target = sorted(target) + + if not self.target: + raise ValueError('`target` should not be empty') + + # these are useful while gathering all the multi-dut test cases + # when this mode is activated, + # + # pytest.mark.esp32 + # pytest.mark.parametrize('count', [2], indirect=True) + # def test_foo(dut): + # pass + # + # should be collected when running `pytest --target esp32` + # + # otherwise, it should be collected when running `pytest --target esp32,esp32` + self._single_target_duplicate_mode = single_target_duplicate_mode + + self.apps_list = ( + [os.path.join(app.app_dir, app.build_dir) for app in apps if app.build_status == BuildStatus.SUCCESS] + if apps + else None + ) self.cases: t.List[PytestCase] = [] + # record the additional info + # test case id: {key: value} + self.additional_info: t.Dict[str, t.Dict[str, t.Any]] = defaultdict(dict) + + @cached_property + def collect_mode(self) -> CollectMode: + if len(self.target) == 1: + if self.target[0] == CollectMode.MULTI_ALL_WITH_PARAM: + return CollectMode.MULTI_ALL_WITH_PARAM + else: + return CollectMode.SINGLE_SPECIFIC + else: + return CollectMode.MULTI_SPECIFIC + @staticmethod def get_param(item: Function, key: str, default: t.Any = None) -> t.Any: - # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting - # https://github.com/pytest-dev/pytest/discussions/9689 + # funcargs is not calculated while collection + # callspec is something defined in parametrize if not hasattr(item, 'callspec'): return default return item.callspec.params.get(key, default) or default def item_to_pytest_case(self, item: Function) -> PytestCase: - count = 1 - case_path = str(item.path) - case_name = item.originalname - target = self.target + """ + Turn pytest item to PytestCase + """ + count = self.get_param(item, 'count', 1) - # funcargs is not calculated while collection - if hasattr(item, 'callspec'): - count = item.callspec.params.get('count', 1) - app_paths = to_list( - parse_multi_dut_args( - count, - self.get_param(item, 'app_path', os.path.dirname(case_path)), - ) - ) - configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', 'default'))) - targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target', target))) - else: - app_paths = [os.path.dirname(case_path)] - configs = ['default'] - targets = [target] + # default app_path is where the test script locates + app_paths = to_list(parse_multi_dut_args(count, self.get_param(item, 'app_path', os.path.dirname(item.path)))) + configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', DEFAULT_SDKCONFIG))) + targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target', self.target[0]))) - case_apps = set() - for i in range(count): - case_apps.add(PytestApp(app_paths[i], targets[i], configs[i])) + def abspath_or_relpath(s: str) -> str: + if os.path.abspath(s) and s.startswith(os.getcwd()): + return os.path.relpath(s) + + return s return PytestCase( - case_path, - case_name, - case_apps, - self.target, - item, + [PytestApp(abspath_or_relpath(app_paths[i]), targets[i], configs[i]) for i in range(count)], item ) - @pytest.hookimpl(tryfirst=True) - def pytest_sessionstart(self, session: Session) -> None: - # same behavior for vanilla pytest-embedded '--target' - session.config.option.target = self.target - @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(self, items: t.List[Function]) -> None: - item_to_case: t.Dict[Function, PytestCase] = {} + """ + Background info: - # Add Markers to the test cases + We're using `pytest.mark.[TARGET]` as a syntactic sugar to indicate that they are actually supported by all + the listed targets. For example, + + >>> @pytest.mark.esp32 + >>> @pytest.mark.esp32s2 + + should be treated as + + >>> @pytest.mark.parametrize('target', [ + >>> 'esp32', + >>> 'esp32s2', + >>> ], indirect=True) + + All single-dut test cases, and some of the multi-dut test cases with the same targets, are using this + way to indicate the supported targets. + + To avoid ambiguity, + + - when we're collecting single-dut test cases with esp32, we call + + `pytest --collect-only --target esp32` + + - when we're collecting multi-dut test cases, we list all the targets, even when they're the same + + `pytest --collect-only --target esp32,esp32` for two esp32 connected + `pytest --collect-only --target esp32,esp32s2` for esp32 and esp32s2 connected + + therefore, we have two different logic for searching test cases, explained in 2.1 and 2.2 + """ + # 1. Filter according to nighty_run related markers + if os.getenv('INCLUDE_NIGHTLY_RUN') == '1': + # nightly_run and non-nightly_run cases are both included + pass + elif os.getenv('NIGHTLY_RUN') == '1': + # only nightly_run cases are included + items[:] = [_item for _item in items if _item.get_closest_marker('nightly_run') is not None] + else: + # only non-nightly_run cases are included + items[:] = [_item for _item in items if _item.get_closest_marker('nightly_run') is None] + + # 2. Add markers according to special markers + item_to_case_dict: t.Dict[Function, PytestCase] = {} for item in items: - # generate PytestCase for each item - case = self.item_to_pytest_case(item) - item_to_case[item] = case - - # set default timeout 10 minutes for each case - if 'timeout' not in item.keywords: - item.add_marker(pytest.mark.timeout(10 * 60)) - - # add markers for special markers + item.stash[ITEM_PYTEST_CASE_KEY] = item_to_case_dict[item] = self.item_to_pytest_case(item) if 'supported_targets' in item.keywords: for _target in SUPPORTED_TARGETS: item.add_marker(_target) @@ -117,75 +174,63 @@ class IdfPytestEmbedded: for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]: item.add_marker(_target) + # 3.1. CollectMode.SINGLE_SPECIFIC, like `pytest --target esp32` + if self.collect_mode == CollectMode.SINGLE_SPECIFIC: + filtered_items = [] + for item in items: + case = item_to_case_dict[item] + + # single-dut one + if case.is_single_dut_test_case and self.target[0] in case.target_markers: + filtered_items.append(item) + + # multi-dut ones and in single_target_duplicate_mode + elif self._single_target_duplicate_mode and not case.is_single_dut_test_case: + # ignore those test cases with `target` defined in parametrize, since these will be covered in 3.3 + if self.get_param(item, 'target', None) is None and self.target[0] in case.target_markers: + filtered_items.append(item) + + items[:] = filtered_items + # 3.2. CollectMode.MULTI_SPECIFIC, like `pytest --target esp32,esp32` + elif self.collect_mode == CollectMode.MULTI_SPECIFIC: + items[:] = [_item for _item in items if item_to_case_dict[_item].targets == self.target] + + # 3.3. CollectMode.MULTI_ALL_WITH_PARAM, intended to be used by `get_pytest_cases` + else: + items[:] = [ + _item + for _item in items + if not item_to_case_dict[_item].is_single_dut_test_case + and self.get_param(_item, 'target', None) is not None + ] + + # 4. filter by `self.apps_list`, skip the test case if not listed + # should only be used in CI + _items = [] + for item in items: + case = item_to_case_dict[item] + if msg := case.all_built_in_app_lists(self.apps_list): + self.additional_info[case.name]['skip_reason'] = msg + else: + _items.append(item) + + # OKAY!!! All left ones will be executed, sort it and add more markers + items[:] = sorted( + _items, key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG)) + ) + for item in items: + case = item_to_case_dict[item] + # set default timeout 10 minutes for each case + if 'timeout' not in item.keywords: + item.add_marker(pytest.mark.timeout(10 * 60)) + # add 'xtal_40mhz' tag as a default tag for esp32c2 target # only add this marker for esp32c2 cases - if self.target == 'esp32c2' and 'esp32c2' in case.target_markers and 'xtal_26mhz' not in case.all_markers: + if 'esp32c2' in self.target and 'esp32c2' in case.targets and 'xtal_26mhz' not in case.all_markers: item.add_marker('xtal_40mhz') - # Filter the test cases - filtered_items = [] - for item in items: - case = item_to_case[item] - # filter by "nightly_run" marker - if os.getenv('INCLUDE_NIGHTLY_RUN') == '1': - # Do not filter nightly_run cases - pass - elif os.getenv('NIGHTLY_RUN') == '1': - if not case.is_nightly_run: - logging.debug( - 'Skipping test case %s because of this test case is not a nightly run test case', item.name - ) - continue - else: - if case.is_nightly_run: - logging.debug( - 'Skipping test case %s because of this test case is a nightly run test case', item.name - ) - continue - - # filter by target - if self.target not in case.target_markers: - continue - - if self.target in case.skipped_targets: - continue - - # filter by sdkconfig - if self.sdkconfig: - if self.get_param(item, 'config', DEFAULT_SDKCONFIG) != self.sdkconfig: - continue - - # filter by apps_list, skip the test case if not listed - # should only be used in CI - if self.apps_list is not None: - bin_not_found = False - for case_app in case.apps: - # in ci, always use build__ as build dir - binary_path = os.path.join(case_app.path, f'build_{case_app.target}_{case_app.config}') - if binary_path not in self.apps_list: - logging.info( - 'Skipping test case %s because binary path %s is not listed in app info list files', - item.name, - binary_path, - ) - bin_not_found = True - break - - if bin_not_found: - continue - - # finally! - filtered_items.append(item) - - # sort the test cases with (app folder, config) - items[:] = sorted( - filtered_items, - key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG)) - ) - def pytest_report_collectionfinish(self, items: t.List[Function]) -> None: - for item in items: - self.cases.append(self.item_to_pytest_case(item)) + self.cases = [item.stash[ITEM_PYTEST_CASE_KEY] for item in items] def pytest_custom_test_case_name(self, item: Function) -> str: return item.funcargs.get('test_case_name', item.nodeid) # type: ignore @@ -252,6 +297,9 @@ class IdfPytestEmbedded: if 'file' in case.attrib: case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework + if ci_job_url := os.getenv('CI_JOB_URL'): + case.attrib['ci_job_url'] = ci_job_url + xml.write(junit) def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: diff --git a/tools/ci/idf_pytest/pytest.ini b/tools/ci/idf_pytest/pytest.ini new file mode 100644 index 0000000000..0ee949b898 --- /dev/null +++ b/tools/ci/idf_pytest/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files = test_*.py diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py index a970864478..7e3232c0c7 100644 --- a/tools/ci/idf_pytest/script.py +++ b/tools/ci/idf_pytest/script.py @@ -1,18 +1,24 @@ # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +import fnmatch import io +import logging +import os.path import typing as t from contextlib import redirect_stdout from pathlib import Path import pytest from _pytest.config import ExitCode +from idf_build_apps import App, find_apps +from idf_build_apps.constants import SUPPORTED_TARGETS, BuildStatus +from idf_ci.app import IdfCMakeApp +from idf_ci_utils import IDF_PATH, get_all_manifest_files, to_list from idf_py_actions.constants import PREVIEW_TARGETS as TOOLS_PREVIEW_TARGETS from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGETS -from pytest_embedded.utils import to_list -from .constants import PytestCase +from .constants import DEFAULT_BUILD_LOG_FILENAME, DEFAULT_CONFIG_RULES_STR, CollectMode, PytestCase from .plugin import IdfPytestEmbedded @@ -35,15 +41,28 @@ def get_pytest_files(paths: t.List[str]) -> t.List[str]: def get_pytest_cases( paths: t.Union[str, t.List[str]], - target: str = 'all', + target: str = CollectMode.ALL, + *, marker_expr: t.Optional[str] = None, filter_expr: t.Optional[str] = None, + apps: t.Optional[t.List[App]] = None, ) -> t.List[PytestCase]: - if target == 'all': - targets = TOOLS_SUPPORTED_TARGETS + TOOLS_PREVIEW_TARGETS - else: - targets = [target] + """ + For single-dut test cases, `target` could be + - [TARGET], e.g. `esp32`, to get the test cases for the given target + - or `single_all`, to get all single-dut test cases + For multi-dut test cases, `target` could be + - [TARGET,[TARGET...]], e.g. `esp32,esp32s2`, to get the test cases for the given targets + - or `multi_all`, to get all multi-dut test cases + + :param paths: paths to search for pytest scripts + :param target: target or keywords to get test cases for, detailed above + :param marker_expr: pytest marker expression, `-m` + :param filter_expr: pytest filter expression, `-k` + :param apps: built app list, skip the tests required by apps not in the list + :return: list of test cases + """ paths = to_list(paths) cases: t.List[PytestCase] = [] @@ -52,12 +71,12 @@ def get_pytest_cases( print(f'WARNING: no pytest scripts found for target {target} under paths {", ".join(paths)}') return cases - for target in targets: - collector = IdfPytestEmbedded(target) + def _get_pytest_cases(_target: str, _single_target_duplicate_mode: bool = False) -> t.List[PytestCase]: + collector = IdfPytestEmbedded(_target, single_target_duplicate_mode=_single_target_duplicate_mode, apps=apps) with io.StringIO() as buf: with redirect_stdout(buf): - cmd = ['--collect-only', *pytest_scripts, '--target', target, '-q'] + cmd = ['--collect-only', *pytest_scripts, '--target', _target, '-q'] if marker_expr: cmd.extend(['-m', marker_expr]) if filter_expr: @@ -66,11 +85,129 @@ def get_pytest_cases( if res.value != ExitCode.OK: if res.value == ExitCode.NO_TESTS_COLLECTED: - print(f'WARNING: no pytest app found for target {target} under paths {", ".join(paths)}') + print(f'WARNING: no pytest app found for target {_target} under paths {", ".join(paths)}') else: print(buf.getvalue()) - raise RuntimeError(f'pytest collection failed at {", ".join(paths)} with command \"{" ".join(cmd)}\"') + raise RuntimeError( + f'pytest collection failed at {", ".join(paths)} with command \"{" ".join(cmd)}\"' + ) - cases.extend(collector.cases) + return collector.cases # type: ignore - return cases + if target == CollectMode.ALL: + targets = TOOLS_SUPPORTED_TARGETS + TOOLS_PREVIEW_TARGETS + [CollectMode.MULTI_ALL_WITH_PARAM] + else: + targets = [target] + + for _target in targets: + if target == CollectMode.ALL: + cases.extend(_get_pytest_cases(_target, _single_target_duplicate_mode=True)) + else: + cases.extend(_get_pytest_cases(_target)) + + return sorted(cases, key=lambda x: (x.path, x.name, str(x.targets))) + + +def get_all_apps( + paths: t.List[str], + target: str = CollectMode.ALL, + *, + marker_expr: t.Optional[str] = None, + filter_expr: t.Optional[str] = None, + config_rules_str: t.Optional[t.List[str]] = None, + preserve_all: bool = False, + extra_default_build_targets: t.Optional[t.List[str]] = None, + modified_components: t.Optional[t.List[str]] = None, + modified_files: t.Optional[t.List[str]] = None, + ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, +) -> t.Tuple[t.Set[App], t.Set[App]]: + """ + Return the tuple of test-required apps and non-test-related apps + + :param paths: paths to search for pytest scripts + :param target: target or keywords to get test cases for, explained in `get_pytest_cases` + :param marker_expr: pytest marker expression, `-m` + :param filter_expr: pytest filter expression, `-k` + :param config_rules_str: config rules string + :param preserve_all: preserve all apps + :param extra_default_build_targets: extra default build targets + :param modified_components: modified components + :param modified_files: modified files + :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns + :return: tuple of test-required apps and non-test-related apps + """ + all_apps = find_apps( + paths, + target, + build_system=IdfCMakeApp, + recursive=True, + build_dir='build_@t_@w', + config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR, + build_log_filename=DEFAULT_BUILD_LOG_FILENAME, + size_json_filename='size.json', + check_warnings=True, + manifest_rootpath=IDF_PATH, + manifest_files=get_all_manifest_files(), + default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []), + modified_components=modified_components, + modified_files=modified_files, + ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, + include_skipped_apps=True, + ) + + pytest_cases = get_pytest_cases( + paths, + target, + marker_expr=marker_expr, + filter_expr=filter_expr, + ) + + modified_pytest_cases = [] + if modified_files: + modified_pytest_scripts = [ + os.path.dirname(f) for f in modified_files if fnmatch.fnmatch(os.path.basename(f), 'pytest_*.py') + ] + if modified_pytest_scripts: + modified_pytest_cases = get_pytest_cases( + modified_pytest_scripts, + target, + marker_expr=marker_expr, + filter_expr=filter_expr, + ) + + # app_path, target, config + pytest_app_path_tuple_dict: t.Dict[t.Tuple[Path, str, str], PytestCase] = {} + for case in pytest_cases: + for app in case.apps: + pytest_app_path_tuple_dict[(Path(app.path), app.target, app.config)] = case + + modified_pytest_app_path_tuple_dict: t.Dict[t.Tuple[Path, str, str], PytestCase] = {} + for case in modified_pytest_cases: + for app in case.apps: + modified_pytest_app_path_tuple_dict[(Path(app.path), app.target, app.config)] = case + + test_related_apps: t.Set[App] = set() + non_test_related_apps: t.Set[App] = set() + for app in all_apps: + # override build_status if test script got modified + if case := modified_pytest_app_path_tuple_dict.get((Path(app.app_dir), app.target, app.config_name)): + test_related_apps.add(app) + app.build_status = BuildStatus.SHOULD_BE_BUILT + app.preserve = True + logging.debug('Found app: %s - required by modified test case %s', app, case.path) + elif app.build_status != BuildStatus.SKIPPED: + if case := pytest_app_path_tuple_dict.get((Path(app.app_dir), app.target, app.config_name)): + test_related_apps.add(app) + # should be built if + app.build_status = BuildStatus.SHOULD_BE_BUILT + app.preserve = True + logging.debug('Found test-related app: %s - required by %s', app, case.path) + else: + non_test_related_apps.add(app) + app.preserve = preserve_all + logging.debug('Found non-test-related app: %s', app) + + print(f'Found {len(test_related_apps)} test-related apps') + print(f'Found {len(non_test_related_apps)} non-test-related apps') + + return test_related_apps, non_test_related_apps diff --git a/tools/ci/idf_pytest/tests/conftest.py b/tools/ci/idf_pytest/tests/conftest.py new file mode 100644 index 0000000000..bfd05d8dfb --- /dev/null +++ b/tools/ci/idf_pytest/tests/conftest.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +from pathlib import Path + +tools_ci_dir = os.path.join(os.path.dirname(__file__), '..', '..') +if tools_ci_dir not in sys.path: + sys.path.append(tools_ci_dir) + +tools_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..') +if tools_dir not in sys.path: + sys.path.append(tools_dir) + + +def create_project(name: str, folder: Path) -> Path: + p = folder / name + p.mkdir(parents=True, exist_ok=True) + (p / 'main').mkdir(parents=True, exist_ok=True) + + with open(p / 'CMakeLists.txt', 'w') as fw: + fw.write( + """cmake_minimum_required(VERSION 3.16) +include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) +project({}) +""".format( + name + ) + ) + + with open(p / 'main' / 'CMakeLists.txt', 'w') as fw: + fw.write( + """idf_component_register(SRCS "{}.c" +INCLUDE_DIRS ".") +""".format( + name + ) + ) + + with open(p / 'main' / f'{name}.c', 'w') as fw: + fw.write( + """#include +void app_main(void) {} +""" + ) + + return p diff --git a/tools/ci/idf_pytest/tests/test_get_all_apps.py b/tools/ci/idf_pytest/tests/test_get_all_apps.py new file mode 100644 index 0000000000..e2aa2c95b2 --- /dev/null +++ b/tools/ci/idf_pytest/tests/test_get_all_apps.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +from idf_pytest.script import SUPPORTED_TARGETS, get_all_apps + +from conftest import create_project + + +def test_get_all_apps_non(tmp_path: Path) -> None: + create_project('foo', tmp_path) + create_project('bar', tmp_path) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)]) + + assert test_related_apps == set() + assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) + + +def test_get_all_apps_single_dut_test_script(tmp_path: Path) -> None: + create_project('foo', tmp_path) + with open(tmp_path / 'foo' / 'pytest_get_all_apps_single_dut_test_script.py', 'w') as fw: + fw.write( + """import pytest + +@pytest.mark.esp32 +@pytest.mark.esp32s2 +def test_foo(dut): + pass +""" + ) + create_project('bar', tmp_path) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all') + + assert len(test_related_apps) == 2 + assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2 + + +def test_get_all_apps_multi_dut_test_script(tmp_path: Path) -> None: + create_project('foo', tmp_path) + with open(tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_test_script.py', 'w') as fw: + fw.write( + """import pytest + +@pytest.mark.parametrize( + 'count, target', [ + (2, 'esp32s2|esp32s3'), + (3, 'esp32|esp32s3|esp32'), + ], indirect=True +) +def test_foo(dut): + pass +""" + ) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all') + + assert len(test_related_apps) == 3 # 32, s2, s3 + assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 3 + + +def test_get_all_apps_modified_pytest_script(tmp_path: Path) -> None: + create_project('foo', tmp_path) + create_project('bar', tmp_path) + + (tmp_path / 'pytest_get_all_apps_modified_pytest_script.py').write_text( + """import pytest +import os + +@pytest.mark.parametrize('count, target', [(2, 'esp32')], indirect=True) +@pytest.mark.parametrize('app_path', [ + '{}|{}'.format(os.path.join(os.path.dirname(__file__), 'foo'), os.path.join(os.path.dirname(__file__), 'bar')), + ], indirect=True +) +def test_multi_foo_bar(dut): + pass +""", + encoding='utf-8', + ) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all') + assert len(test_related_apps) == 2 # foo-esp32, bar-esp32 + assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2 + + test_related_apps, non_test_related_apps = get_all_apps( + [str(tmp_path)], target='all', modified_files=[], modified_components=[] + ) + assert len(test_related_apps) == 0 + assert len(non_test_related_apps) == 0 + + test_related_apps, non_test_related_apps = get_all_apps( + [str(tmp_path)], + target='all', + modified_files=[str(tmp_path / 'pytest_get_all_apps_modified_pytest_script.py')], + modified_components=[], + ) + assert len(test_related_apps) == 2 + assert len(non_test_related_apps) == 0 diff --git a/tools/ci/idf_pytest/tests/test_get_pytest_cases.py b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py new file mode 100644 index 0000000000..e078146ce4 --- /dev/null +++ b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +from idf_pytest.constants import CollectMode +from idf_pytest.script import get_pytest_cases + +TEMPLATE_SCRIPT = ''' +import pytest + +@pytest.mark.esp32 +@pytest.mark.esp32s2 +def test_foo_single(dut): + pass + +@pytest.mark.parametrize( + 'count, target', [ + (2, 'esp32|esp32s2'), + (3, 'esp32s2|esp32s2|esp32s3'), + ], indirect=True +) +def test_foo_multi(dut): + pass + +@pytest.mark.esp32 +@pytest.mark.esp32s2 +@pytest.mark.parametrize( + 'count', [2], indirect=True +) +def test_foo_multi_with_marker(dut): + pass +''' + + +def test_get_pytest_cases_single_specific(tmp_path: Path) -> None: + script = tmp_path / 'pytest_get_pytest_cases_single_specific.py' + script.write_text(TEMPLATE_SCRIPT) + cases = get_pytest_cases([str(tmp_path)], 'esp32') + + assert len(cases) == 1 + assert cases[0].targets == ['esp32'] + + +def test_get_pytest_cases_multi_specific(tmp_path: Path) -> None: + script = tmp_path / 'pytest_get_pytest_cases_multi_specific.py' + script.write_text(TEMPLATE_SCRIPT) + cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2, esp32s2') + + assert len(cases) == 1 + assert cases[0].targets == ['esp32s2', 'esp32s2', 'esp32s3'] + + +def test_get_pytest_cases_multi_all(tmp_path: Path) -> None: + script = tmp_path / 'pytest_get_pytest_cases_multi_all.py' + script.write_text(TEMPLATE_SCRIPT) + cases = get_pytest_cases([str(tmp_path)], CollectMode.MULTI_ALL_WITH_PARAM) + + assert len(cases) == 2 + assert cases[0].targets == ['esp32', 'esp32s2'] + assert cases[1].targets == ['esp32s2', 'esp32s2', 'esp32s3'] + + +def test_get_pytest_cases_all(tmp_path: Path) -> None: + script = tmp_path / 'pytest_get_pytest_cases_all.py' + script.write_text(TEMPLATE_SCRIPT) + cases = get_pytest_cases([str(tmp_path)], CollectMode.ALL) + + assert len(cases) == 6 + assert cases[0].targets == ['esp32', 'esp32s2'] + assert cases[0].name == 'test_foo_multi' + + assert cases[1].targets == ['esp32s2', 'esp32s2', 'esp32s3'] + assert cases[1].name == 'test_foo_multi' + + assert cases[2].targets == ['esp32', 'esp32'] + assert cases[2].name == 'test_foo_multi_with_marker' + + assert cases[3].targets == ['esp32s2', 'esp32s2'] + assert cases[3].name == 'test_foo_multi_with_marker' + + assert cases[4].targets == ['esp32'] + assert cases[4].name == 'test_foo_single' + + assert cases[5].targets == ['esp32s2'] + assert cases[5].name == 'test_foo_single' diff --git a/tools/ci/idf_pytest/utils.py b/tools/ci/idf_pytest/utils.py index aef1a776c5..1e06354668 100644 --- a/tools/ci/idf_pytest/utils.py +++ b/tools/ci/idf_pytest/utils.py @@ -6,10 +6,10 @@ import os import typing as t from xml.etree import ElementTree as ET -from .constants import TARGET_MARKERS - -def format_case_id(target: t.Optional[str], config: t.Optional[str], case: str, is_qemu: bool = False, params: t.Optional[dict] = None) -> str: +def format_case_id( + target: t.Optional[str], config: t.Optional[str], case: str, is_qemu: bool = False, params: t.Optional[dict] = None +) -> str: parts = [] if target: parts.append((str(target) + '_qemu') if is_qemu else str(target)) @@ -23,23 +23,6 @@ def format_case_id(target: t.Optional[str], config: t.Optional[str], case: str, return '.'.join(parts) -def get_target_marker_from_expr(markexpr: str) -> str: - candidates = set() - # we use `-m "esp32 and generic"` in our CI to filter the test cases - # this doesn't cover all use cases, but fit what we do in CI. - for marker in markexpr.split('and'): - marker = marker.strip() - if marker in TARGET_MARKERS: - candidates.add(marker) - - if len(candidates) > 1: - raise ValueError(f'Specified more than one target markers: {candidates}. Please specify no more than one.') - elif len(candidates) == 1: - return candidates.pop() - else: - raise ValueError('Please specify one target marker via "--target [TARGET]" or via "-m [TARGET]"') - - def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: if len(junit_files) <= 1: return @@ -78,3 +61,7 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: with open(target_path, 'wb') as fw: fw.write(ET.tostring(merged_testsuite)) + + +def comma_sep_str_to_list(s: str) -> t.List[str]: + return [s.strip() for s in s.split(',') if s.strip()] diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 1482ddd52f..32652ec3b4 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 + import argparse import logging import os @@ -10,7 +11,7 @@ import tempfile import time import zipfile from functools import wraps -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Union import gitlab @@ -63,7 +64,7 @@ class Gitlab(object): DOWNLOAD_ERROR_MAX_RETRIES = 3 - def __init__(self, project_id: Optional[int] = None): + def __init__(self, project_id: Union[int, str, None] = None): config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') if config_data_from_env: # prefer to load config from env variable @@ -129,7 +130,7 @@ class Gitlab(object): archive_file.extractall(destination) @retry - def download_artifact(self, job_id: int, artifact_path: str, destination: Optional[str] = None) -> List[bytes]: + def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]: """ download specific path of job artifacts and extract to destination. diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index 0e5c231133..4ab65cf374 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -3,7 +3,7 @@ # ci coverage -idf-build-apps +idf-build-apps~=2.0.0rc1 jsonschema junit_xml python-gitlab diff --git a/tools/requirements/requirements.pytest.txt b/tools/requirements/requirements.pytest.txt index 8ca65461b5..3c6dcbf223 100644 --- a/tools/requirements/requirements.pytest.txt +++ b/tools/requirements/requirements.pytest.txt @@ -10,7 +10,7 @@ pytest-timeout pytest-ignore-test-results # build -idf-build-apps +idf-build-apps~=2.0.0rc1 # dependencies in pytest test scripts scapy diff --git a/tools/test_apps/.build-test-rules.yml b/tools/test_apps/.build-test-rules.yml index 47177fff01..22f992ed66 100644 --- a/tools/test_apps/.build-test-rules.yml +++ b/tools/test_apps/.build-test-rules.yml @@ -52,6 +52,16 @@ tools/test_apps/protocols/mqtt/publish_connect_test: - if: IDF_TARGET == "esp32s2" or IDF_TARGET == "esp32c3" temporary: true reason: lack of runners + depends_components: + - esp_eth + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - examples/common_components/**/* + - examples/protocols/**/* + - examples/system/ota/**/* + - examples/ethernet/iperf/**/* + - examples/network/vlan_support/**/* + - components/esp_netif/esp_netif_handlers.c tools/test_apps/protocols/netif_components: enable: