From f33469dd638163a29df49c679d39001b17072192 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 13 Jun 2025 14:28:26 +0200 Subject: [PATCH] ci: apply `idf-ci build run` removed script: - tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py -> idf-ci build run - tools/ci/ci_build_apps.py -> idf-build-apps build moved from remove from idf_pytest/constants.py to .idf_build_apps.toml - DEFAULT_FULL_BUILD_TEST_COMPONENTS - DEFAULT_FULL_BUILD_TEST_FILEPATTERNS - DEFAULT_IGNORE_WARNING_FILEPATH - DEFAULT_BUILD_LOG_FILENAME - DEFAULT_SIZE_JSON_FILENAME --- .gitignore | 2 +- .gitlab/ci/README.md | 1 + .gitlab/ci/build.yml | 7 +- .gitlab/ci/host-test.yml | 28 +- .gitlab/ci/pre_check.yml | 9 - .gitlab/ci/rules.yml | 1 - .idf_build_apps.toml | 64 ++++ .idf_ci.toml | 5 + .pre-commit-config.yaml | 4 +- .../esp_tee/test_apps/tee_test_fw/README.md | 2 +- .../contribute/esp-idf-tests-with-pytest.rst | 6 +- .../contribute/esp-idf-tests-with-pytest.rst | 6 +- examples/protocols/sockets/README.md | 8 +- pytest.ini | 4 +- tools/ci/artifacts_handler.py | 9 +- tools/ci/build_template_app.sh | 11 +- tools/ci/ci_build_apps.py | 294 ------------------ tools/ci/dynamic_pipelines/report.py | 2 +- .../scripts/child_pipeline_build_apps.py | 75 ----- .../scripts/generate_report.py | 2 +- .../generate_target_test_child_pipeline.py | 2 +- .../templates/.dynamic_jobs.yml | 10 +- tools/ci/idf_ci_local/app.py | 7 +- tools/ci/idf_ci_local/uploader.py | 8 +- tools/ci/idf_pytest/constants.py | 26 -- tools/ci/idf_pytest/script.py | 8 +- 26 files changed, 124 insertions(+), 477 deletions(-) create mode 100644 .idf_build_apps.toml create mode 100644 .idf_ci.toml delete mode 100644 tools/ci/ci_build_apps.py delete mode 100644 tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py diff --git a/.gitignore b/.gitignore index 0db298aab7..0e2e4e1fd4 100644 --- a/.gitignore +++ b/.gitignore @@ -100,7 +100,7 @@ managed_components pytest-embedded/ # legacy one pytest_embedded_log/ -list_job*.txt +app_info_*.txt size_info*.txt XUNIT_RESULT*.xml .manifest_sha diff --git a/.gitlab/ci/README.md b/.gitlab/ci/README.md index d51a4f5673..6e3fb43a8a 100644 --- a/.gitlab/ci/README.md +++ b/.gitlab/ci/README.md @@ -250,6 +250,7 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature: - Add MR labels `BUILD_AND_TEST_ALL_APPS` +- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1` - Run in protected branches ## Upload/Download Artifacts to Internal Minio Server diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 7d3e52ae57..618620fef4 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -34,13 +34,12 @@ variables: IDF_TOOLCHAIN: clang TEST_BUILD_OPTS_EXTRA: "" - TEST_DIR: tools/test_apps/system/clang_build_test PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" script: # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v + - run_cmd idf-build-apps build + -p tools/test_apps/system/clang_build_test -t $IDF_TARGET - --copy-sdkconfig --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} --modified-components ${MR_MODIFIED_COMPONENTS} @@ -105,7 +104,7 @@ gcc_static_analyzer: ANALYZING_APP: "examples/get-started/hello_world" script: - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults - - python -m idf_build_apps build -v -p ${ANALYZING_APP} -t all + - idf-build-apps build -p ${ANALYZING_APP} ######################################## # Clang Build Apps Without Tests Cases # diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index e6ea1e16bc..2a73b72017 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -297,12 +297,11 @@ test_pytest_qemu: INSTALL_EXTRA_TOOLS: "qemu-riscv32" IDF_TOOLCHAIN: [gcc, clang] script: - - run_cmd python tools/ci/ci_build_apps.py . -v + - run_cmd idf-ci build run + --build-system cmake --target $IDF_TARGET - --pytest-apps + --only-test-related -m qemu - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} --modified-files ${MR_MODIFIED_FILES} - python tools/ci/get_known_failure_cases_file.py - run_cmd pytest @@ -312,7 +311,6 @@ test_pytest_qemu: --embedded-services idf,qemu --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" test_pytest_linux: @@ -327,21 +325,18 @@ test_pytest_linux: reports: junit: XUNIT_RESULT.xml script: - - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v + - run_cmd idf-ci build run + --build-system cmake + -p components -p examples -p tools/test_apps --target linux - --pytest-apps - -m host_test - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} + --only-test-related --modified-files ${MR_MODIFIED_FILES} - python tools/ci/get_known_failure_cases_file.py - run_cmd pytest --target linux - -m host_test --embedded-services idf --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" test_pytest_macos: extends: @@ -364,12 +359,12 @@ test_pytest_macos: # GitLab sets the project dir to this template `//` IDF_PATH: "/Users/espressif/builds/espressif/esp-idf" script: - - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v + - run_cmd idf-ci build run + -p components -p examples -p tools/test_apps + --build-system cmake --target linux - --pytest-apps + --only-test-related -m \"host_test and macos\" - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} --modified-files ${MR_MODIFIED_FILES} - python tools/ci/get_known_failure_cases_file.py - run_cmd pytest @@ -377,7 +372,6 @@ test_pytest_macos: -m \"host_test and macos\" --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" test_idf_pytest_plugin: extends: diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 7f70b47ba1..4885f0c22a 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -122,15 +122,6 @@ check_test_scripts_build_test_rules: # requires basic pytest dependencies - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components -check_configure_ci_environment_parsing: - extends: - - .pre_check_template - - .before_script:build - - .rules:build - script: - - cd tools/ci - - python -m unittest ci_build_apps.py - pipeline_variables: extends: - .pre_check_template diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 96e19867e2..88097051a7 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -53,7 +53,6 @@ - "tools/ci/ignore_build_warnings.txt" - "tools/ci/test_build_system*.sh" - "tools/ci/test_build_system*.py" - - "tools/ci/ci_build_apps.py" - "tools/test_build_system/**/*" .patterns-build_system_win: &patterns-build_system_win diff --git a/.idf_build_apps.toml b/.idf_build_apps.toml new file mode 100644 index 0000000000..473db19d8b --- /dev/null +++ b/.idf_build_apps.toml @@ -0,0 +1,64 @@ +config_rules = [ + 'sdkconfig.ci=default', + 'sdkconfig.ci.*=', + '=default', +] + +extra_pythonpaths = [ + '$IDF_PATH/tools/ci/python_packages', + '$IDF_PATH/tools/ci', + '$IDF_PATH/tools', +] +build_system = "idf_ci_local.app:IdfCMakeApp" + +recursive = true +check_warnings = true +keep_going = true +copy_sdkconfig = true +ignore_warning_files = [ + '$IDF_PATH/tools/ci/ignore_build_warnings.txt', +] + +build_dir = "build_@t_@w" +build_log_filename = "build.log" +size_json_filename = "size.json" + +verbose = 1 # INFO + +# collect +collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt" +collect_size_info_filename = "size_info_${CI_JOB_NAME_SLUG}.txt" # TODO remove this file when ci-dashboard is ready +junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml" + +# manifest +# check_manifest_rules = true # FIXME +manifest_rootpath = "$IDF_PATH" +manifest_filepatterns = [ + '**/.build-test-rules.yml', +] + +# dependency-driven build +deactivate_dependency_driven_build_by_components = [ + 'cxx', + 'esp_common', + 'esp_hw_support', + 'esp_rom', + 'esp_system', + 'esp_timer', + 'freertos', + 'hal', + 'heap', + 'log', + 'newlib', + 'riscv', + 'soc', + 'xtensa', +] + +deactivate_dependency_driven_build_by_filepatterns = [ + # tools + 'tools/cmake/**/*', + 'tools/tools.json', + # ci + 'tools/ci/ignore_build_warnings.txt', +] diff --git a/.idf_ci.toml b/.idf_ci.toml new file mode 100644 index 0000000000..f09999413f --- /dev/null +++ b/.idf_ci.toml @@ -0,0 +1,5 @@ +[local_runtime_envs] +EXTRA_CFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes" +EXTRA_CXXFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function" +LDGEN_CHECK_MAPPING = "1" +IDF_CI_BUILD = "1" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d506aa269a..9a5d883815 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: - pytest-ignore-test-results~=0.3 - pytest-rerunfailures - pytest-timeout - - idf-build-apps~=2.8 + - idf-build-apps~=2.11 - python-gitlab - minio - click @@ -163,7 +163,7 @@ repos: require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf-build-apps>=2.8,<3 + - idf-build-apps~=2.11 - id: sort-yaml-files name: sort yaml files entry: tools/ci/sort_yaml.py diff --git a/components/esp_tee/test_apps/tee_test_fw/README.md b/components/esp_tee/test_apps/tee_test_fw/README.md index 9a8031229a..5396aaf280 100644 --- a/components/esp_tee/test_apps/tee_test_fw/README.md +++ b/components/esp_tee/test_apps/tee_test_fw/README.md @@ -23,6 +23,6 @@ bash install.sh --enable-ci - For example, to execute the TEE test suite for ESP32-C6 with all the available `sdkconfig` files, run the following steps. The required test applications will be built and flashed automatically onto the DUT by the `pytest` framework. ```bash -python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32c6 -v --pytest-apps +idf-ci build run --target esp32c6 --only-test-related pytest --target esp32c6 ``` diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index 7ce23fe048..97f1686a5b 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -544,7 +544,7 @@ If you want to build and test with all sdkconfig files at the same time, you sho .. code-block:: shell $ cd $IDF_PATH/examples/system/console/basic - $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -v --pytest-apps + $ idf-ci build run --target esp32 --only-test-related $ pytest --target esp32 The app with ``sdkconfig.ci.history`` will be built in ``build_esp32_history``, and the app with ``sdkconfig.ci.nohistory`` will be built in ``build_esp32_nohistory``. ``pytest --target esp32`` will run tests on both apps. @@ -580,8 +580,8 @@ Of course we can build the required binaries manually, but we can also use our C .. code-block:: shell $ cd $IDF_PATH/examples/openthread - $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target all -v --pytest-apps -k test_thread_connect - $ pytest --target esp32c6,esp32h2,esp32s3 -k test_thread_connect + $ idf-ci build run --only-test-related -k test_thread_connect + $ pytest -k test_thread_connect .. important:: diff --git a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst index 1d4d4c8dfc..ea2803c7bd 100644 --- a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst @@ -544,7 +544,7 @@ CI 的工作流程如下所示： .. code-block:: shell $ cd $IDF_PATH/examples/system/console/basic - $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -v --pytest-apps + $ idf-ci build run --target esp32 --only-test-related $ pytest --target esp32 包含 ``sdkconfig.ci.history`` 配置的应用程序会编译到 ``build_esp32_history`` 中，而包含 ``sdkconfig.ci.nohistory`` 配置的应用程序会编译到 ``build_esp32_nohistory`` 中。 ``pytest --target esp32`` 命令会在这两个应用程序上运行测试。 @@ -580,8 +580,8 @@ CI 的工作流程如下所示： .. code-block:: shell $ cd $IDF_PATH/examples/openthread - $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target all -v --pytest-apps -k test_thread_connect - $ pytest --target esp32c6,esp32h2,esp32s3 -k test_thread_connect + $ idf-ci build run --only-test-related -k test_thread_connect + $ pytest -k test_thread_connect .. important:: diff --git a/examples/protocols/sockets/README.md b/examples/protocols/sockets/README.md index ccf28036b7..7a8985d6fb 100644 --- a/examples/protocols/sockets/README.md +++ b/examples/protocols/sockets/README.md @@ -2,8 +2,8 @@ # BSD Socket API Examples This directory contains simple examples demonstrating BSD Socket API. -Each example, contains README.md file with mode detailed informations about that particular example. -For more general informations about all examples, see the README.md file in the upper level 'examples' directory. +Each example, contains README.md file with mode detailed information about that particular example. +For more general information about all examples, see the README.md file in the upper level 'examples' directory. Examples: * UDP Client - The application creates UDP socket and sends message to the predefined port and IP address. After the server's reply, the application prints received reply as ASCII text, waits for 2 seconds and sends another message. @@ -59,7 +59,7 @@ $ cd $IDF_PATH $ bash install.sh --enable-ci $ . ./export.sh $ cd examples/protocols/sockets/tcp_client -$ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps +$ idf-ci build run --target esp32 --only-test-related $ pytest --target esp32 ``` @@ -112,7 +112,7 @@ Please make sure that when using the Local Link address, an interface id is incl * On the host - Interface name suffix is present when passing the address as a string, for example `fe80::260a:XXX:XXX:XXX%en0` - - The interface id is present when passing the endpoint as tupple, for example `socket.connect(('fd00::260a:XXXX:XXXX:XXXX', 3333, 0, 3))` + - The interface id is present when passing the endpoint as tuple, for example `socket.connect(('fd00::260a:XXXX:XXXX:XXXX', 3333, 0, 3))` ## Hardware Required diff --git a/pytest.ini b/pytest.ini index 1fe5e78cc6..7f8c1f0953 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,12 +15,14 @@ addopts = --check-duplicates y --ignore-glob */managed_components/* --ignore pytest-embedded + --unity-test-report-mode merge + --ignore-no-tests-collected-error # ignore DeprecationWarning filterwarnings = ignore::DeprecationWarning:matplotlib.*: ignore::DeprecationWarning:google.protobuf.*: - ignore::_pytest.warning_types.PytestExperimentalApiWarning + ignore::FutureWarning # log related log_cli = True diff --git a/tools/ci/artifacts_handler.py b/tools/ci/artifacts_handler.py index 3a2cd429e6..ceb4ff1044 100644 --- a/tools/ci/artifacts_handler.py +++ b/tools/ci/artifacts_handler.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import fnmatch @@ -12,7 +12,6 @@ from zipfile import ZipFile import urllib3 from idf_ci_utils import sanitize_job_name -from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME from minio import Minio @@ -36,7 +35,7 @@ TYPE_PATTERNS_DICT = { '**/build*/*.elf', ], ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [ - f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}', + '**/build*/build_log.txt', '**/build*/*.bin', '**/build*/bootloader/*.bin', '**/build*/esp_tee/*.bin', @@ -46,10 +45,10 @@ TYPE_PATTERNS_DICT = { '**/build*/config/sdkconfig.json', '**/build*/sdkconfig', '**/build*/project_description.json', - 'list_job*.txt', + 'app_info_*.txt', ], ArtifactType.LOGS: [ - f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}', + '**/build*/build_log.txt', ], ArtifactType.SIZE_REPORTS: [ '**/build*/size.json', diff --git a/tools/ci/build_template_app.sh b/tools/ci/build_template_app.sh index e4a68f45e5..e4a5e9cff1 100755 --- a/tools/ci/build_template_app.sh +++ b/tools/ci/build_template_app.sh @@ -54,30 +54,23 @@ build_stage2() { # Override EXTRA_CFLAGS and EXTRA_CXXFLAGS in the environment export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} - python -m idf_build_apps build -v \ + idf-build-apps build \ -p ${TEMPLATE_APP_PATH} \ - -t all \ ${CONFIG_STR} \ --work-dir ${BUILD_PATH}/cmake \ --build-dir ${BUILD_DIR} \ --build-log ${BUILD_LOG_CMAKE} \ - --size-file size.json \ - --keep-going \ - --collect-size-info size_info.txt \ --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c5 esp32c6 esp32h2 esp32p4 esp32c61 esp32h21 esp32h4 } build_stage1() { CONFIG_STR=$(get_config_str sdkconfig.ci2.*=) - python -m idf_build_apps build -v \ + idf-build-apps build \ -p ${TEMPLATE_APP_PATH} \ - -t all \ ${CONFIG_STR} \ --work-dir ${BUILD_PATH}/cmake \ --build-dir ${BUILD_DIR} \ --build-log ${BUILD_LOG_CMAKE} \ - --size-file size.json \ - --collect-size-info size_info.txt \ --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c5 esp32c6 esp32h2 esp32p4 esp32c61 esp32h21 esp32h4 } diff --git a/tools/ci/ci_build_apps.py b/tools/ci/ci_build_apps.py deleted file mode 100644 index 77bbc46857..0000000000 --- a/tools/ci/ci_build_apps.py +++ /dev/null @@ -1,294 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -""" -This file is used in CI generate binary files for different kinds of apps -""" - -import argparse -import os -import sys -import typing as t -import unittest -from pathlib import Path - -import yaml -from dynamic_pipelines.constants import DEFAULT_TEST_PATHS -from idf_build_apps import build_apps -from idf_build_apps import setup_logging -from idf_build_apps.utils import semicolon_separated_str_to_list -from idf_pytest.constants import DEFAULT_BUILD_TEST_RULES_FILEPATH -from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR -from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_COMPONENTS -from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_FILEPATTERNS -from idf_pytest.constants import DEFAULT_IGNORE_WARNING_FILEPATH -from idf_pytest.script import get_all_apps - -CI_ENV_VARS = { - 'EXTRA_CFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable ' - '-Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes', - 'EXTRA_CXXFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable ' - '-Werror=unused-but-set-variable -Werror=unused-function', - 'LDGEN_CHECK_MAPPING': '1', - 'IDF_CI_BUILD': '1', -} - - -def main(args: argparse.Namespace) -> None: - extra_default_build_targets: t.List[str] = [] - if args.default_build_test_rules: - with open(args.default_build_test_rules) as fr: - configs = yaml.safe_load(fr) - - if configs: - extra_default_build_targets = configs.get('extra_default_build_targets') or [] - - test_related_apps, non_test_related_apps = get_all_apps( - args.paths, - args.target, - config_rules_str=args.config, - marker_expr=args.marker_expr, - filter_expr=args.filter_expr, - preserve_all=args.preserve_all, - extra_default_build_targets=extra_default_build_targets, - modified_files=args.modified_files, - modified_components=args.modified_components, - ignore_app_dependencies_components=args.ignore_app_dependencies_components, - ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns, - ) - - if args.pytest_apps: - apps = test_related_apps - else: - apps = non_test_related_apps - - if args.extra_preserve_dirs: - for app in apps: - if app.preserve: - continue - for extra_preserve_dir in args.extra_preserve_dirs: - abs_extra_preserve_dir = Path(extra_preserve_dir).resolve() - abs_app_dir = Path(app.app_dir).resolve() - if abs_extra_preserve_dir == abs_app_dir or abs_extra_preserve_dir in abs_app_dir.parents: - app.preserve = True - - res = build_apps( - sorted(apps), - parallel_count=args.parallel_count, - parallel_index=args.parallel_index, - dry_run=False, - build_verbose=args.build_verbose, - keep_going=True, - collect_size_info='size_info.txt', - collect_app_info=args.collect_app_info, - ignore_warning_strs=args.ignore_warning_str, - ignore_warning_file=args.ignore_warning_file, - copy_sdkconfig=args.copy_sdkconfig, - modified_components=args.modified_components, - modified_files=args.modified_files, - ignore_app_dependencies_components=args.ignore_app_dependencies_components, - ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns, - junitxml=args.junitxml, - ) - - sys.exit(res) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Build all the apps for different test types. Will auto remove those non-test apps binaries', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument('paths', nargs='*', help='Paths to the apps to build.') - parser.add_argument( - '-t', - '--target', - default='all', - help='Build apps for given target', - ) - parser.add_argument( - '--config', - default=DEFAULT_CONFIG_RULES_STR, - nargs='+', - help='Adds configurations (sdkconfig file names) to build. This can either be ' - 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' - 'relative to the project directory, to be used. Optional NAME can be specified, ' - 'which can be used as a name of this configuration. FILEPATTERN is the name of ' - 'the sdkconfig file, relative to the project directory, with at most one wildcard. ' - 'The part captured by the wildcard is used as the name of the configuration.', - ) - parser.add_argument( - '-v', - '--verbose', - action='count', - help='Increase the LOGGER level of the script. Can be specified multiple times.', - ) - parser.add_argument( - '--build-verbose', - action='store_true', - help='Enable verbose output from build system.', - ) - parser.add_argument( - '--preserve-all', - action='store_true', - help='Preserve the binaries for all apps when specified.', - ) - parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.') - parser.add_argument( - '--parallel-index', - default=1, - type=int, - help='Index (1-based) of the job, out of the number specified by --parallel-count.', - ) - parser.add_argument( - '--collect-app-info', - default='list_job_@p.txt', - help='If specified, the test case name and app info json will be written to this file', - ) - parser.add_argument( - '--ignore-warning-str', - nargs='+', - help='Ignore the warning string that match the specified regex in the build output. space-separated list', - ) - parser.add_argument( - '--ignore-warning-file', - default=DEFAULT_IGNORE_WARNING_FILEPATH, - type=argparse.FileType('r'), - help='Ignore the warning strings in the specified file. Each line should be a regex string.', - ) - parser.add_argument( - '--copy-sdkconfig', - action='store_true', - help='Copy the sdkconfig file to the build directory.', - ) - parser.add_argument( - '--extra-preserve-dirs', - nargs='+', - help='also preserve binaries of the apps under the specified dirs', - ) - - parser.add_argument( - '--pytest-apps', - action='store_true', - help='Only build apps required by pytest scripts. ' - 'Will build non-test-related apps if this flag is unspecified.', - ) - parser.add_argument( - '-m', - '--marker-expr', - default='not host_test', # host_test apps would be built and tested under the same job - help='only build tests matching given mark expression. For example: -m "host_test and generic". Works only' - 'for pytest', - ) - parser.add_argument( - '-k', - '--filter-expr', - help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only' - 'for pytest', - ) - parser.add_argument( - '--default-build-test-rules', - default=DEFAULT_BUILD_TEST_RULES_FILEPATH, - help='default build test rules config file', - ) - parser.add_argument( - '--skip-setting-flags', - action='store_true', - help='by default this script would set the build flags exactly the same as the CI ones. ' - 'Set this flag to use your local build flags.', - ) - parser.add_argument( - '--modified-components', - type=semicolon_separated_str_to_list, - default=os.getenv('MR_MODIFIED_COMPONENTS'), - help='semicolon-separated string which specifies the modified components. ' - 'app with `depends_components` set in the corresponding manifest files would only be built ' - 'if depends on any of the specified components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '--modified-files', - type=semicolon_separated_str_to_list, - default=os.getenv('MR_MODIFIED_FILES'), - help='semicolon-separated string which specifies the modified files. ' - 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' - 'if any of the specified file pattern matches any of the specified modified files. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '-ic', - '--ignore-app-dependencies-components', - type=semicolon_separated_str_to_list, - help='semicolon-separated string which specifies the modified components used for ' - 'ignoring checking the app dependencies. ' - 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' - 'when any of the specified components matches any of the modified components. ' - 'Must be used together with --modified-components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '-if', - '--ignore-app-dependencies-filepatterns', - type=semicolon_separated_str_to_list, - help='semicolon-separated string which specifies the file patterns used for ' - 'ignoring checking the app dependencies. ' - 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' - 'when any of the specified file patterns matches any of the modified files. ' - 'Must be used together with --modified-files. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '--junitxml', - default='build_summary_@p.xml', - help='Path to the junitxml file. If specified, the junitxml file will be generated', - ) - - arguments = parser.parse_args() - - setup_logging(arguments.verbose) - - # set default paths - if not arguments.paths: - arguments.paths = DEFAULT_TEST_PATHS - - # skip setting flags in CI - if not arguments.skip_setting_flags and not os.getenv('CI_JOB_ID'): - for _k, _v in CI_ENV_VARS.items(): - os.environ[_k] = _v # type: ignore - print(f'env var {_k} set to "{_v}"') - - if not os.getenv('CI_MERGE_REQUEST_IID') or os.getenv('IDF_CI_SELECT_ALL_PYTEST_CASES') == '1': - print('Build and run all test cases, and compile all cmake apps') - arguments.modified_components = None - arguments.modified_files = None - arguments.ignore_app_dependencies_components = None - arguments.ignore_app_dependencies_filepatterns = None - else: - print( - f'Build and run only test cases matching:\n' - f'- modified components: {arguments.modified_components}\n' - f'- modified files: {arguments.modified_files}' - ) - - if arguments.modified_components is not None and not arguments.ignore_app_dependencies_components: - # setting default values - arguments.ignore_app_dependencies_components = DEFAULT_FULL_BUILD_TEST_COMPONENTS - - if arguments.modified_files is not None and not arguments.ignore_app_dependencies_filepatterns: - # setting default values - arguments.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS - - main(arguments) - - -class TestParsingShellScript(unittest.TestCase): - """ - This test case is run in CI jobs to make sure the CI build flags is the same as the ones recorded in CI_ENV_VARS - """ - - def test_parse_result(self) -> None: - for k, v in CI_ENV_VARS.items(): - self.assertEqual(os.getenv(k), v) diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py index 2f51a6e6c2..d6250fc100 100644 --- a/tools/ci/dynamic_pipelines/report.py +++ b/tools/ci/dynamic_pipelines/report.py @@ -89,7 +89,7 @@ class ReportGenerator: with open(output_filepath, 'w') as file: file.write(report_str) - # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt + # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/app_info_84.txt # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` report_url: str = get_artifacts_url(job_id, output_filepath) return report_url diff --git a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py deleted file mode 100644 index 4d00717270..0000000000 --- a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import sys - -import __init__ # noqa: F401 # inject the system path -from idf_build_apps import build_apps -from idf_build_apps import setup_logging -from idf_build_apps.utils import semicolon_separated_str_to_list -from idf_ci_local.app import import_apps_from_txt -from idf_pytest.constants import DEFAULT_IGNORE_WARNING_FILEPATH - -from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Build Apps for Dynamic Pipeline') - parser.add_argument('app_list_file', default=TEST_RELATED_APPS_FILENAME, help='List of apps to build') - parser.add_argument( - '--build-verbose', - action='store_true', - help='Enable verbose output from build system.', - ) - parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.') - parser.add_argument( - '--parallel-index', - default=1, - type=int, - help='Index (1-based) of the job, out of the number specified by --parallel-count.', - ) - parser.add_argument( - '--ignore-warning-file', - default=DEFAULT_IGNORE_WARNING_FILEPATH, - type=argparse.FileType('r'), - help='Ignore the warning strings in the specified file. Each line should be a regex string.', - ) - parser.add_argument( - '--modified-components', - type=semicolon_separated_str_to_list, - help='semicolon-separated string which specifies the modified components. ' - 'app with `depends_components` set in the corresponding manifest files would only be built ' - 'if depends on any of the specified components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '--collect-app-info', - default='list_job_@p.txt', - help='If specified, the test case name and app info json will be written to this file', - ) - parser.add_argument( - '--junitxml', - default='build_summary_@p.xml', - help='Path to the junitxml file. If specified, the junitxml file will be generated', - ) - - args = parser.parse_args() - - setup_logging(verbose=1) - - sys.exit( - build_apps( - import_apps_from_txt(args.app_list_file), - build_verbose=args.build_verbose, - keep_going=True, - ignore_warning_file=args.ignore_warning_file, - modified_components=args.modified_components, - check_app_dependencies=True, - parallel_count=args.parallel_count, - parallel_index=args.parallel_index, - collect_size_info='size_info.txt', - collect_app_info=args.collect_app_info, - junitxml=args.junitxml, - copy_sdkconfig=True, - ) - ) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_report.py b/tools/ci/dynamic_pipelines/scripts/generate_report.py index a224ac7f76..ea67356f16 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_report.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_report.py @@ -65,7 +65,7 @@ def common_arguments(parser: argparse.ArgumentParser) -> None: def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: if report_type_args.report_type == 'build': - parser.add_argument('--app-list-filepattern', default='list_job_*.txt', help='Pattern to match app list files') + parser.add_argument('--app-list-filepattern', default='app_info_*.txt', help='Pattern to match app list files') elif report_type_args.report_type == 'target_test': parser.add_argument( '--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files' diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py index f4875f9f5d..d8c09b427e 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -183,7 +183,7 @@ if __name__ == '__main__': ) parser.add_argument( '--app-info-filepattern', - default='list_job_*.txt', + default='app_info_*.txt', help='glob pattern to specify the files that include built app info generated by ' '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' 'paths not exist in local file system if not listed recorded in the app info.', diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 8166cb9ad9..822f04a6eb 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -26,23 +26,21 @@ paths: # The other artifacts patterns are defined under tools/ci/artifacts_handler.py # Now we're uploading/downloading the binary files from our internal storage server - # + # keep the log file to help debug - "**/build*/build_log.txt" # build spec files - build_summary_*.xml # list of built apps - - list_job_*.txt + - app_info_*.txt when: always expire_in: 1 week script: # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py $APP_LIST_FILE + - run_cmd idf-ci build run --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --junitxml "build_summary_${CI_JOB_NAME_SLUG}.xml" + --modified-files ${MR_MODIFIED_FILES} - run_cmd python tools/ci/artifacts_handler.py upload --type size_reports .dynamic_target_test_template: diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index cec367ca91..601b2b6a19 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -3,20 +3,21 @@ import os import sys import typing as t -from typing import Literal from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME from idf_build_apps import App from idf_build_apps import CMakeApp from idf_build_apps import json_to_app -from .uploader import AppUploader from .uploader import get_app_uploader +if t.TYPE_CHECKING: + from .uploader import AppUploader + class IdfCMakeApp(CMakeApp): uploader: t.ClassVar[t.Optional['AppUploader']] = get_app_uploader() - build_system: Literal['idf_cmake'] = 'idf_cmake' + build_system: t.Literal['idf_cmake'] = 'idf_cmake' def _initialize_hook(self, **kwargs: t.Any) -> None: # ensure this env var exists diff --git a/tools/ci/idf_ci_local/uploader.py b/tools/ci/idf_ci_local/uploader.py index 66c4fdfa8f..6b58846c43 100644 --- a/tools/ci/idf_ci_local/uploader.py +++ b/tools/ci/idf_ci_local/uploader.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import abc import glob @@ -15,8 +15,6 @@ from artifacts_handler import getenv from idf_build_apps import App from idf_build_apps.utils import rmdir from idf_ci_utils import IDF_PATH -from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME -from idf_pytest.constants import DEFAULT_SIZE_JSON_FILENAME class AppDownloader: @@ -63,7 +61,7 @@ class AppUploader(AppDownloader): 'project_description.json', ], ArtifactType.LOGS: [ - DEFAULT_BUILD_LOG_FILENAME, + 'build_log.txt', ], } @@ -116,7 +114,7 @@ class AppUploader(AppDownloader): uploaded |= self._upload_app(app_build_path, upload_type) if uploaded: - rmdir(app_build_path, exclude_file_patterns=[DEFAULT_BUILD_LOG_FILENAME, DEFAULT_SIZE_JSON_FILENAME]) + rmdir(app_build_path, exclude_file_patterns=['build_log.txt', 'size.json']) def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: app_path, build_dir = os.path.split(app_build_path) diff --git a/tools/ci/idf_pytest/constants.py b/tools/ci/idf_pytest/constants.py index 3cba84121a..c43df8dcd8 100644 --- a/tools/ci/idf_pytest/constants.py +++ b/tools/ci/idf_pytest/constants.py @@ -156,33 +156,7 @@ TIMEOUT_4H_MARKERS = [ ] DEFAULT_CONFIG_RULES_STR = ['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'] -DEFAULT_IGNORE_WARNING_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt') DEFAULT_BUILD_TEST_RULES_FILEPATH = os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml') -DEFAULT_FULL_BUILD_TEST_COMPONENTS = [ - 'cxx', - 'esp_common', - 'esp_hw_support', - 'esp_rom', - 'esp_system', - 'esp_timer', - 'freertos', - 'hal', - 'heap', - 'log', - 'newlib', - 'riscv', - 'soc', - 'xtensa', -] -DEFAULT_FULL_BUILD_TEST_FILEPATTERNS = [ - # tools - 'tools/cmake/**/*', - 'tools/tools.json', - # ci - 'tools/ci/ignore_build_warnings.txt', -] -DEFAULT_BUILD_LOG_FILENAME = 'build_log.txt' -DEFAULT_SIZE_JSON_FILENAME = 'size.json' class CollectMode(str, Enum): diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py index 1c79aeecd1..e70b8ad164 100644 --- a/tools/ci/idf_pytest/script.py +++ b/tools/ci/idf_pytest/script.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import fnmatch import io @@ -22,9 +22,7 @@ from idf_ci_utils import to_list from idf_py_actions.constants import PREVIEW_TARGETS as TOOLS_PREVIEW_TARGETS from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGETS -from .constants import DEFAULT_BUILD_LOG_FILENAME from .constants import DEFAULT_CONFIG_RULES_STR -from .constants import DEFAULT_SIZE_JSON_FILENAME from .constants import CollectMode from .constants import PytestCase from .plugin import IdfPytestEmbedded @@ -163,8 +161,8 @@ def get_all_apps( recursive=True, build_dir='build_@t_@w', config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR, - build_log_filename=DEFAULT_BUILD_LOG_FILENAME, - size_json_filename=DEFAULT_SIZE_JSON_FILENAME, + build_log_filename='build_log.txt', + size_json_filename='size.json', check_warnings=True, manifest_rootpath=IDF_PATH, compare_manifest_sha_filepath=compare_manifest_sha_filepath,