ci: apply idf-ci build run

removed script:

- tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py -> idf-ci build run
- tools/ci/ci_build_apps.py -> idf-build-apps build

moved from remove from idf_pytest/constants.py to .idf_build_apps.toml

- DEFAULT_FULL_BUILD_TEST_COMPONENTS
- DEFAULT_FULL_BUILD_TEST_FILEPATTERNS
- DEFAULT_IGNORE_WARNING_FILEPATH
- DEFAULT_BUILD_LOG_FILENAME
- DEFAULT_SIZE_JSON_FILENAME

.gitignore

@@ -100,7 +100,7 @@ managed_components
 pytest-embedded/
 # legacy one
 pytest_embedded_log/
-list_job*.txt
+app_info_*.txt
 size_info*.txt
 XUNIT_RESULT*.xml
 .manifest_sha

.gitlab/ci/README.md

@@ -250,6 +250,7 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
 In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
 
 - Add MR labels `BUILD_AND_TEST_ALL_APPS`
+- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1`
 - Run in protected branches
 
 ## Upload/Download Artifacts to Internal Minio Server

.gitlab/ci/build.yml

@@ -34,13 +34,12 @@
   variables:
     IDF_TOOLCHAIN: clang
     TEST_BUILD_OPTS_EXTRA: ""
-    TEST_DIR: tools/test_apps/system/clang_build_test
     PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
     # CI specific options start from "--parallel-count xxx". could ignore when running locally
-    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+    - run_cmd idf-build-apps build
+      -p tools/test_apps/system/clang_build_test
       -t $IDF_TARGET
-      --copy-sdkconfig
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
       --modified-components ${MR_MODIFIED_COMPONENTS}
@@ -105,7 +104,7 @@ gcc_static_analyzer:
     ANALYZING_APP: "examples/get-started/hello_world"
   script:
     - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
-    - python -m idf_build_apps build -v -p ${ANALYZING_APP} -t all
+    - idf-build-apps build -p ${ANALYZING_APP}
 
 ########################################
 # Clang Build Apps Without Tests Cases #

.gitlab/ci/host-test.yml

@@ -297,12 +297,11 @@ test_pytest_qemu:
         INSTALL_EXTRA_TOOLS: "qemu-riscv32"
         IDF_TOOLCHAIN: [gcc, clang]
   script:
-    - run_cmd python tools/ci/ci_build_apps.py . -v
+    - run_cmd idf-ci build run
+      --build-system cmake
       --target $IDF_TARGET
-      --pytest-apps
+      --only-test-related
       -m qemu
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
@@ -312,7 +311,6 @@ test_pytest_qemu:
       --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
       --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\"
 
 test_pytest_linux:
@@ -327,21 +325,18 @@ test_pytest_linux:
     reports:
       junit: XUNIT_RESULT.xml
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
+    - run_cmd idf-ci build run
+      --build-system cmake
+      -p components -p examples -p tools/test_apps
       --target linux
-      --pytest-apps
+      --only-test-related
-      -m host_test
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
       --target linux
-      -m host_test
       --embedded-services idf
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
 
 test_pytest_macos:
   extends:
@@ -364,12 +359,12 @@ test_pytest_macos:
     # GitLab sets the project dir to this template `<builds_dir>/<namespace>/<project_name>`
     IDF_PATH: "/Users/espressif/builds/espressif/esp-idf"
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
+    - run_cmd idf-ci build run
+      -p components -p examples -p tools/test_apps
+      --build-system cmake
       --target linux
-      --pytest-apps
+      --only-test-related
       -m \"host_test and macos\"
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
@@ -377,7 +372,6 @@ test_pytest_macos:
       -m \"host_test and macos\"
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
 
 test_idf_pytest_plugin:
   extends:

.gitlab/ci/pre_check.yml

@@ -122,15 +122,6 @@ check_test_scripts_build_test_rules:
     # requires basic pytest dependencies
     - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components
 
-check_configure_ci_environment_parsing:
-  extends:
-    - .pre_check_template
-    - .before_script:build
-    - .rules:build
-  script:
-    - cd tools/ci
-    - python -m unittest ci_build_apps.py
-
 pipeline_variables:
   extends:
     - .pre_check_template

.gitlab/ci/rules.yml

@@ -53,7 +53,6 @@
   - "tools/ci/ignore_build_warnings.txt"
   - "tools/ci/test_build_system*.sh"
   - "tools/ci/test_build_system*.py"
-  - "tools/ci/ci_build_apps.py"
   - "tools/test_build_system/**/*"
 
 .patterns-build_system_win: &patterns-build_system_win

.idf_build_apps.toml

@@ -0,0 +1,64 @@
+config_rules = [
+    'sdkconfig.ci=default',
+    'sdkconfig.ci.*=',
+    '=default',
+]
+
+extra_pythonpaths = [
+    '$IDF_PATH/tools/ci/python_packages',
+    '$IDF_PATH/tools/ci',
+    '$IDF_PATH/tools',
+]
+build_system = "idf_ci_local.app:IdfCMakeApp"
+
+recursive = true
+check_warnings = true
+keep_going = true
+copy_sdkconfig = true
+ignore_warning_files = [
+    '$IDF_PATH/tools/ci/ignore_build_warnings.txt',
+]
+
+build_dir = "build_@t_@w"
+build_log_filename = "build.log"
+size_json_filename = "size.json"
+
+verbose = 1  # INFO
+
+# collect
+collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt"
+collect_size_info_filename = "size_info_${CI_JOB_NAME_SLUG}.txt"  # TODO remove this file when ci-dashboard is ready
+junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml"
+
+# manifest
+# check_manifest_rules = true  # FIXME
+manifest_rootpath = "$IDF_PATH"
+manifest_filepatterns = [
+    '**/.build-test-rules.yml',
+]
+
+# dependency-driven build
+deactivate_dependency_driven_build_by_components = [
+    'cxx',
+    'esp_common',
+    'esp_hw_support',
+    'esp_rom',
+    'esp_system',
+    'esp_timer',
+    'freertos',
+    'hal',
+    'heap',
+    'log',
+    'newlib',
+    'riscv',
+    'soc',
+    'xtensa',
+]
+
+deactivate_dependency_driven_build_by_filepatterns = [
+    # tools
+    'tools/cmake/**/*',
+    'tools/tools.json',
+    # ci
+    'tools/ci/ignore_build_warnings.txt',
+]

.idf_ci.toml

@@ -0,0 +1,5 @@
+[local_runtime_envs]
+EXTRA_CFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes"
+EXTRA_CXXFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function"
+LDGEN_CHECK_MAPPING = "1"
+IDF_CI_BUILD = "1"

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
           - pytest-ignore-test-results~=0.3
           - pytest-rerunfailures
           - pytest-timeout
-          - idf-build-apps~=2.8
+          - idf-build-apps~=2.11
           - python-gitlab
           - minio
           - click
@@ -163,7 +163,7 @@ repos:
         require_serial: true
         additional_dependencies:
           - PyYAML == 5.3.1
-          - idf-build-apps>=2.8,<3
+          - idf-build-apps~=2.11
       - id: sort-yaml-files
         name: sort yaml files
         entry: tools/ci/sort_yaml.py

components/esp_tee/test_apps/tee_test_fw/README.md

@@ -23,6 +23,6 @@ bash install.sh --enable-ci
 - For example, to execute the TEE test suite for ESP32-C6 with all the available `sdkconfig` files, run the following steps. The required test applications will be built and flashed automatically onto the DUT by the `pytest` framework.
 
 ```bash
-python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32c6 -v --pytest-apps
+idf-ci build run --target esp32c6 --only-test-related
 pytest --target esp32c6
 ```

docs/en/contribute/esp-idf-tests-with-pytest.rst

@@ -544,7 +544,7 @@ If you want to build and test with all sdkconfig files at the same time, you sho
 .. code-block:: shell
 
     $ cd $IDF_PATH/examples/system/console/basic
-    $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -v --pytest-apps
+    $ idf-ci build run --target esp32 --only-test-related
     $ pytest --target esp32
 
 The app with ``sdkconfig.ci.history`` will be built in ``build_esp32_history``, and the app with ``sdkconfig.ci.nohistory`` will be built in ``build_esp32_nohistory``. ``pytest --target esp32`` will run tests on both apps.
@@ -580,8 +580,8 @@ Of course we can build the required binaries manually, but we can also use our C
 .. code-block:: shell
 
     $ cd $IDF_PATH/examples/openthread
-    $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target all -v --pytest-apps -k test_thread_connect
+    $ idf-ci build run --only-test-related -k test_thread_connect
-    $ pytest --target esp32c6,esp32h2,esp32s3 -k test_thread_connect
+    $ pytest -k test_thread_connect
 
 .. important::
 

docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst

@@ -544,7 +544,7 @@ CI 的工作流程如下所示：
 .. code-block:: shell
 
     $ cd $IDF_PATH/examples/system/console/basic
-    $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -v --pytest-apps
+    $ idf-ci build run --target esp32 --only-test-related
     $ pytest --target esp32
 
 包含 ``sdkconfig.ci.history`` 配置的应用程序会编译到 ``build_esp32_history`` 中，而包含 ``sdkconfig.ci.nohistory`` 配置的应用程序会编译到 ``build_esp32_nohistory`` 中。 ``pytest --target esp32`` 命令会在这两个应用程序上运行测试。
@@ -580,8 +580,8 @@ CI 的工作流程如下所示：
 .. code-block:: shell
 
     $ cd $IDF_PATH/examples/openthread
-    $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target all -v --pytest-apps -k test_thread_connect
+    $ idf-ci build run --only-test-related -k test_thread_connect
-    $ pytest --target esp32c6,esp32h2,esp32s3 -k test_thread_connect
+    $ pytest -k test_thread_connect
 
 .. important::
 

examples/protocols/sockets/README.md

@@ -2,8 +2,8 @@
 # BSD Socket API Examples
 
 This directory contains simple examples demonstrating BSD Socket API. 
-Each example, contains README.md file with mode detailed informations about that particular example.
+Each example, contains README.md file with mode detailed information about that particular example.
-For more general informations about all examples, see the README.md file in the upper level 'examples' directory.
+For more general information about all examples, see the README.md file in the upper level 'examples' directory.
 Examples:
 
 * UDP Client - The application creates UDP socket and sends message to the predefined port and IP address. After the server's reply, the application prints received reply as ASCII text, waits for 2 seconds and sends another message.
@@ -59,7 +59,7 @@ $ cd $IDF_PATH
 $ bash install.sh --enable-ci
 $ . ./export.sh
 $ cd examples/protocols/sockets/tcp_client
-$ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps
+$ idf-ci build run --target esp32 --only-test-related
 $ pytest --target esp32
 ```
 
@@ -112,7 +112,7 @@ Please make sure that when using the Local Link address, an interface id is incl
 * On the host
 
    - Interface name suffix is present when passing the address as a string, for example `fe80::260a:XXX:XXX:XXX%en0`
-   - The interface id is present when passing the endpoint as tupple, for example `socket.connect(('fd00::260a:XXXX:XXXX:XXXX', 3333, 0, 3))`
+   - The interface id is present when passing the endpoint as tuple, for example `socket.connect(('fd00::260a:XXXX:XXXX:XXXX', 3333, 0, 3))`
 
 ## Hardware Required
 

pytest.ini

@@ -15,12 +15,14 @@ addopts =
   --check-duplicates y
   --ignore-glob */managed_components/*
   --ignore pytest-embedded
+  --unity-test-report-mode merge
+  --ignore-no-tests-collected-error
 
 # ignore DeprecationWarning
 filterwarnings =
   ignore::DeprecationWarning:matplotlib.*:
   ignore::DeprecationWarning:google.protobuf.*:
-  ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  ignore::FutureWarning
 
 # log related
 log_cli = True

tools/ci/artifacts_handler.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import fnmatch
@@ -12,7 +12,6 @@ from zipfile import ZipFile
 
 import urllib3
 from idf_ci_utils import sanitize_job_name
-from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME
 from minio import Minio
 
 
@@ -36,7 +35,7 @@ TYPE_PATTERNS_DICT = {
         '**/build*/*.elf',
     ],
     ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [
-        f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}',
+        '**/build*/build_log.txt',
         '**/build*/*.bin',
         '**/build*/bootloader/*.bin',
         '**/build*/esp_tee/*.bin',
@@ -46,10 +45,10 @@ TYPE_PATTERNS_DICT = {
         '**/build*/config/sdkconfig.json',
         '**/build*/sdkconfig',
         '**/build*/project_description.json',
-        'list_job*.txt',
+        'app_info_*.txt',
     ],
     ArtifactType.LOGS: [
-        f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}',
+        '**/build*/build_log.txt',
     ],
     ArtifactType.SIZE_REPORTS: [
         '**/build*/size.json',

tools/ci/build_template_app.sh

@@ -54,30 +54,23 @@ build_stage2() {
     # Override EXTRA_CFLAGS and EXTRA_CXXFLAGS in the environment
     export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
     export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
-    python -m idf_build_apps build -v \
+    idf-build-apps build \
         -p ${TEMPLATE_APP_PATH} \
-        -t all \
         ${CONFIG_STR} \
         --work-dir ${BUILD_PATH}/cmake \
         --build-dir ${BUILD_DIR} \
         --build-log ${BUILD_LOG_CMAKE} \
-        --size-file size.json \
-        --keep-going \
-        --collect-size-info size_info.txt \
         --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c5 esp32c6 esp32h2 esp32p4 esp32c61 esp32h21 esp32h4
 }
 
 build_stage1() {
     CONFIG_STR=$(get_config_str sdkconfig.ci2.*=)
-    python -m idf_build_apps build -v \
+    idf-build-apps build \
         -p ${TEMPLATE_APP_PATH} \
-        -t all \
         ${CONFIG_STR} \
         --work-dir ${BUILD_PATH}/cmake \
         --build-dir ${BUILD_DIR} \
         --build-log ${BUILD_LOG_CMAKE} \
-        --size-file size.json \
-        --collect-size-info size_info.txt \
         --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c5 esp32c6 esp32h2 esp32p4 esp32c61 esp32h21 esp32h4
 }
 

tools/ci/ci_build_apps.py

@@ -1,294 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-"""
-This file is used in CI generate binary files for different kinds of apps
-"""
-
-import argparse
-import os
-import sys
-import typing as t
-import unittest
-from pathlib import Path
-
-import yaml
-from dynamic_pipelines.constants import DEFAULT_TEST_PATHS
-from idf_build_apps import build_apps
-from idf_build_apps import setup_logging
-from idf_build_apps.utils import semicolon_separated_str_to_list
-from idf_pytest.constants import DEFAULT_BUILD_TEST_RULES_FILEPATH
-from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR
-from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_COMPONENTS
-from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_FILEPATTERNS
-from idf_pytest.constants import DEFAULT_IGNORE_WARNING_FILEPATH
-from idf_pytest.script import get_all_apps
-
-CI_ENV_VARS = {
-    'EXTRA_CFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable '
-    '-Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes',
-    'EXTRA_CXXFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable '
-    '-Werror=unused-but-set-variable -Werror=unused-function',
-    'LDGEN_CHECK_MAPPING': '1',
-    'IDF_CI_BUILD': '1',
-}
-
-
-def main(args: argparse.Namespace) -> None:
-    extra_default_build_targets: t.List[str] = []
-    if args.default_build_test_rules:
-        with open(args.default_build_test_rules) as fr:
-            configs = yaml.safe_load(fr)
-
-        if configs:
-            extra_default_build_targets = configs.get('extra_default_build_targets') or []
-
-    test_related_apps, non_test_related_apps = get_all_apps(
-        args.paths,
-        args.target,
-        config_rules_str=args.config,
-        marker_expr=args.marker_expr,
-        filter_expr=args.filter_expr,
-        preserve_all=args.preserve_all,
-        extra_default_build_targets=extra_default_build_targets,
-        modified_files=args.modified_files,
-        modified_components=args.modified_components,
-        ignore_app_dependencies_components=args.ignore_app_dependencies_components,
-        ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns,
-    )
-
-    if args.pytest_apps:
-        apps = test_related_apps
-    else:
-        apps = non_test_related_apps
-
-    if args.extra_preserve_dirs:
-        for app in apps:
-            if app.preserve:
-                continue
-            for extra_preserve_dir in args.extra_preserve_dirs:
-                abs_extra_preserve_dir = Path(extra_preserve_dir).resolve()
-                abs_app_dir = Path(app.app_dir).resolve()
-                if abs_extra_preserve_dir == abs_app_dir or abs_extra_preserve_dir in abs_app_dir.parents:
-                    app.preserve = True
-
-    res = build_apps(
-        sorted(apps),
-        parallel_count=args.parallel_count,
-        parallel_index=args.parallel_index,
-        dry_run=False,
-        build_verbose=args.build_verbose,
-        keep_going=True,
-        collect_size_info='size_info.txt',
-        collect_app_info=args.collect_app_info,
-        ignore_warning_strs=args.ignore_warning_str,
-        ignore_warning_file=args.ignore_warning_file,
-        copy_sdkconfig=args.copy_sdkconfig,
-        modified_components=args.modified_components,
-        modified_files=args.modified_files,
-        ignore_app_dependencies_components=args.ignore_app_dependencies_components,
-        ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns,
-        junitxml=args.junitxml,
-    )
-
-    sys.exit(res)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description='Build all the apps for different test types. Will auto remove those non-test apps binaries',
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-    parser.add_argument('paths', nargs='*', help='Paths to the apps to build.')
-    parser.add_argument(
-        '-t',
-        '--target',
-        default='all',
-        help='Build apps for given target',
-    )
-    parser.add_argument(
-        '--config',
-        default=DEFAULT_CONFIG_RULES_STR,
-        nargs='+',
-        help='Adds configurations (sdkconfig file names) to build. This can either be '
-        'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, '
-        'relative to the project directory, to be used. Optional NAME can be specified, '
-        'which can be used as a name of this configuration. FILEPATTERN is the name of '
-        'the sdkconfig file, relative to the project directory, with at most one wildcard. '
-        'The part captured by the wildcard is used as the name of the configuration.',
-    )
-    parser.add_argument(
-        '-v',
-        '--verbose',
-        action='count',
-        help='Increase the LOGGER level of the script. Can be specified multiple times.',
-    )
-    parser.add_argument(
-        '--build-verbose',
-        action='store_true',
-        help='Enable verbose output from build system.',
-    )
-    parser.add_argument(
-        '--preserve-all',
-        action='store_true',
-        help='Preserve the binaries for all apps when specified.',
-    )
-    parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.')
-    parser.add_argument(
-        '--parallel-index',
-        default=1,
-        type=int,
-        help='Index (1-based) of the job, out of the number specified by --parallel-count.',
-    )
-    parser.add_argument(
-        '--collect-app-info',
-        default='list_job_@p.txt',
-        help='If specified, the test case name and app info json will be written to this file',
-    )
-    parser.add_argument(
-        '--ignore-warning-str',
-        nargs='+',
-        help='Ignore the warning string that match the specified regex in the build output. space-separated list',
-    )
-    parser.add_argument(
-        '--ignore-warning-file',
-        default=DEFAULT_IGNORE_WARNING_FILEPATH,
-        type=argparse.FileType('r'),
-        help='Ignore the warning strings in the specified file. Each line should be a regex string.',
-    )
-    parser.add_argument(
-        '--copy-sdkconfig',
-        action='store_true',
-        help='Copy the sdkconfig file to the build directory.',
-    )
-    parser.add_argument(
-        '--extra-preserve-dirs',
-        nargs='+',
-        help='also preserve binaries of the apps under the specified dirs',
-    )
-
-    parser.add_argument(
-        '--pytest-apps',
-        action='store_true',
-        help='Only build apps required by pytest scripts. '
-        'Will build non-test-related apps if this flag is unspecified.',
-    )
-    parser.add_argument(
-        '-m',
-        '--marker-expr',
-        default='not host_test',  # host_test apps would be built and tested under the same job
-        help='only build tests matching given mark expression. For example: -m "host_test and generic". Works only'
-        'for pytest',
-    )
-    parser.add_argument(
-        '-k',
-        '--filter-expr',
-        help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only'
-        'for pytest',
-    )
-    parser.add_argument(
-        '--default-build-test-rules',
-        default=DEFAULT_BUILD_TEST_RULES_FILEPATH,
-        help='default build test rules config file',
-    )
-    parser.add_argument(
-        '--skip-setting-flags',
-        action='store_true',
-        help='by default this script would set the build flags exactly the same as the CI ones. '
-        'Set this flag to use your local build flags.',
-    )
-    parser.add_argument(
-        '--modified-components',
-        type=semicolon_separated_str_to_list,
-        default=os.getenv('MR_MODIFIED_COMPONENTS'),
-        help='semicolon-separated string which specifies the modified components. '
-        'app with `depends_components` set in the corresponding manifest files would only be built '
-        'if depends on any of the specified components. '
-        'If set to "", the value would be considered as None. '
-        'If set to ";", the value would be considered as an empty list',
-    )
-    parser.add_argument(
-        '--modified-files',
-        type=semicolon_separated_str_to_list,
-        default=os.getenv('MR_MODIFIED_FILES'),
-        help='semicolon-separated string which specifies the modified files. '
-        'app with `depends_filepatterns` set in the corresponding manifest files would only be built '
-        'if any of the specified file pattern matches any of the specified modified files. '
-        'If set to "", the value would be considered as None. '
-        'If set to ";", the value would be considered as an empty list',
-    )
-    parser.add_argument(
-        '-ic',
-        '--ignore-app-dependencies-components',
-        type=semicolon_separated_str_to_list,
-        help='semicolon-separated string which specifies the modified components used for '
-        'ignoring checking the app dependencies. '
-        'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored '
-        'when any of the specified components matches any of the modified components. '
-        'Must be used together with --modified-components. '
-        'If set to "", the value would be considered as None. '
-        'If set to ";", the value would be considered as an empty list',
-    )
-    parser.add_argument(
-        '-if',
-        '--ignore-app-dependencies-filepatterns',
-        type=semicolon_separated_str_to_list,
-        help='semicolon-separated string which specifies the file patterns used for '
-        'ignoring checking the app dependencies. '
-        'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored '
-        'when any of the specified file patterns matches any of the modified files. '
-        'Must be used together with --modified-files. '
-        'If set to "", the value would be considered as None. '
-        'If set to ";", the value would be considered as an empty list',
-    )
-    parser.add_argument(
-        '--junitxml',
-        default='build_summary_@p.xml',
-        help='Path to the junitxml file. If specified, the junitxml file will be generated',
-    )
-
-    arguments = parser.parse_args()
-
-    setup_logging(arguments.verbose)
-
-    # set default paths
-    if not arguments.paths:
-        arguments.paths = DEFAULT_TEST_PATHS
-
-    # skip setting flags in CI
-    if not arguments.skip_setting_flags and not os.getenv('CI_JOB_ID'):
-        for _k, _v in CI_ENV_VARS.items():
-            os.environ[_k] = _v  # type: ignore
-            print(f'env var {_k} set to "{_v}"')
-
-    if not os.getenv('CI_MERGE_REQUEST_IID') or os.getenv('IDF_CI_SELECT_ALL_PYTEST_CASES') == '1':
-        print('Build and run all test cases, and compile all cmake apps')
-        arguments.modified_components = None
-        arguments.modified_files = None
-        arguments.ignore_app_dependencies_components = None
-        arguments.ignore_app_dependencies_filepatterns = None
-    else:
-        print(
-            f'Build and run only test cases matching:\n'
-            f'- modified components: {arguments.modified_components}\n'
-            f'- modified files: {arguments.modified_files}'
-        )
-
-        if arguments.modified_components is not None and not arguments.ignore_app_dependencies_components:
-            # setting default values
-            arguments.ignore_app_dependencies_components = DEFAULT_FULL_BUILD_TEST_COMPONENTS
-
-        if arguments.modified_files is not None and not arguments.ignore_app_dependencies_filepatterns:
-            # setting default values
-            arguments.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS
-
-    main(arguments)
-
-
-class TestParsingShellScript(unittest.TestCase):
-    """
-    This test case is run in CI jobs to make sure the CI build flags is the same as the ones recorded in CI_ENV_VARS
-    """
-
-    def test_parse_result(self) -> None:
-        for k, v in CI_ENV_VARS.items():
-            self.assertEqual(os.getenv(k), v)

tools/ci/dynamic_pipelines/report.py

@@ -89,7 +89,7 @@ class ReportGenerator:
         with open(output_filepath, 'w') as file:
             file.write(report_str)
 
-        # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt
+        # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/app_info_84.txt
         # CI_PAGES_URL is {URL}/esp-idf, which missed one `-`
         report_url: str = get_artifacts_url(job_id, output_filepath)
         return report_url

tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py

@@ -1,75 +0,0 @@
-# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-import argparse
-import sys
-
-import __init__  # noqa: F401 # inject the system path
-from idf_build_apps import build_apps
-from idf_build_apps import setup_logging
-from idf_build_apps.utils import semicolon_separated_str_to_list
-from idf_ci_local.app import import_apps_from_txt
-from idf_pytest.constants import DEFAULT_IGNORE_WARNING_FILEPATH
-
-from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Build Apps for Dynamic Pipeline')
-    parser.add_argument('app_list_file', default=TEST_RELATED_APPS_FILENAME, help='List of apps to build')
-    parser.add_argument(
-        '--build-verbose',
-        action='store_true',
-        help='Enable verbose output from build system.',
-    )
-    parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.')
-    parser.add_argument(
-        '--parallel-index',
-        default=1,
-        type=int,
-        help='Index (1-based) of the job, out of the number specified by --parallel-count.',
-    )
-    parser.add_argument(
-        '--ignore-warning-file',
-        default=DEFAULT_IGNORE_WARNING_FILEPATH,
-        type=argparse.FileType('r'),
-        help='Ignore the warning strings in the specified file. Each line should be a regex string.',
-    )
-    parser.add_argument(
-        '--modified-components',
-        type=semicolon_separated_str_to_list,
-        help='semicolon-separated string which specifies the modified components. '
-        'app with `depends_components` set in the corresponding manifest files would only be built '
-        'if depends on any of the specified components. '
-        'If set to "", the value would be considered as None. '
-        'If set to ";", the value would be considered as an empty list',
-    )
-    parser.add_argument(
-        '--collect-app-info',
-        default='list_job_@p.txt',
-        help='If specified, the test case name and app info json will be written to this file',
-    )
-    parser.add_argument(
-        '--junitxml',
-        default='build_summary_@p.xml',
-        help='Path to the junitxml file. If specified, the junitxml file will be generated',
-    )
-
-    args = parser.parse_args()
-
-    setup_logging(verbose=1)
-
-    sys.exit(
-        build_apps(
-            import_apps_from_txt(args.app_list_file),
-            build_verbose=args.build_verbose,
-            keep_going=True,
-            ignore_warning_file=args.ignore_warning_file,
-            modified_components=args.modified_components,
-            check_app_dependencies=True,
-            parallel_count=args.parallel_count,
-            parallel_index=args.parallel_index,
-            collect_size_info='size_info.txt',
-            collect_app_info=args.collect_app_info,
-            junitxml=args.junitxml,
-            copy_sdkconfig=True,
-        )
-    )

tools/ci/dynamic_pipelines/scripts/generate_report.py

@@ -65,7 +65,7 @@ def common_arguments(parser: argparse.ArgumentParser) -> None:
 
 def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
     if report_type_args.report_type == 'build':
-        parser.add_argument('--app-list-filepattern', default='list_job_*.txt', help='Pattern to match app list files')
+        parser.add_argument('--app-list-filepattern', default='app_info_*.txt', help='Pattern to match app list files')
     elif report_type_args.report_type == 'target_test':
         parser.add_argument(
             '--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files'

tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py

@@ -183,7 +183,7 @@ if __name__ == '__main__':
     )
     parser.add_argument(
         '--app-info-filepattern',
-        default='list_job_*.txt',
+        default='app_info_*.txt',
         help='glob pattern to specify the files that include built app info generated by '
         '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary '
         'paths not exist in local file system if not listed recorded in the app info.',

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -26,23 +26,21 @@
     paths:
       # The other artifacts patterns are defined under tools/ci/artifacts_handler.py
       # Now we're uploading/downloading the binary files from our internal storage server
-      #
+
       # keep the log file to help debug
       - "**/build*/build_log.txt"
       # build spec files
       - build_summary_*.xml
       # list of built apps
-      - list_job_*.txt
+      - app_info_*.txt
     when: always
     expire_in: 1 week
   script:
     # CI specific options start from "--parallel-count xxx". could ignore when running locally
-    - run_cmd python tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py $APP_LIST_FILE
+    - run_cmd idf-ci build run
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-files ${MR_MODIFIED_FILES}
-      --modified-components ${MR_MODIFIED_COMPONENTS}
-      --junitxml "build_summary_${CI_JOB_NAME_SLUG}.xml"
     - run_cmd python tools/ci/artifacts_handler.py upload --type size_reports
 
 .dynamic_target_test_template:

tools/ci/idf_ci_local/app.py

@@ -3,20 +3,21 @@
 import os
 import sys
 import typing as t
-from typing import Literal
 
 from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME
 from idf_build_apps import App
 from idf_build_apps import CMakeApp
 from idf_build_apps import json_to_app
 
-from .uploader import AppUploader
 from .uploader import get_app_uploader
 
+if t.TYPE_CHECKING:
+    from .uploader import AppUploader
+
 
 class IdfCMakeApp(CMakeApp):
     uploader: t.ClassVar[t.Optional['AppUploader']] = get_app_uploader()
-    build_system: Literal['idf_cmake'] = 'idf_cmake'
+    build_system: t.Literal['idf_cmake'] = 'idf_cmake'
 
     def _initialize_hook(self, **kwargs: t.Any) -> None:
         # ensure this env var exists

tools/ci/idf_ci_local/uploader.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import abc
 import glob
@@ -15,8 +15,6 @@ from artifacts_handler import getenv
 from idf_build_apps import App
 from idf_build_apps.utils import rmdir
 from idf_ci_utils import IDF_PATH
-from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME
-from idf_pytest.constants import DEFAULT_SIZE_JSON_FILENAME
 
 
 class AppDownloader:
@@ -63,7 +61,7 @@ class AppUploader(AppDownloader):
             'project_description.json',
         ],
         ArtifactType.LOGS: [
-            DEFAULT_BUILD_LOG_FILENAME,
+            'build_log.txt',
         ],
     }
 
@@ -116,7 +114,7 @@ class AppUploader(AppDownloader):
             uploaded |= self._upload_app(app_build_path, upload_type)
 
         if uploaded:
-            rmdir(app_build_path, exclude_file_patterns=[DEFAULT_BUILD_LOG_FILENAME, DEFAULT_SIZE_JSON_FILENAME])
+            rmdir(app_build_path, exclude_file_patterns=['build_log.txt', 'size.json'])
 
     def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None:
         app_path, build_dir = os.path.split(app_build_path)

tools/ci/idf_pytest/constants.py

@@ -156,33 +156,7 @@ TIMEOUT_4H_MARKERS = [
 ]
 
 DEFAULT_CONFIG_RULES_STR = ['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default']
-DEFAULT_IGNORE_WARNING_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt')
 DEFAULT_BUILD_TEST_RULES_FILEPATH = os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml')
-DEFAULT_FULL_BUILD_TEST_COMPONENTS = [
-    'cxx',
-    'esp_common',
-    'esp_hw_support',
-    'esp_rom',
-    'esp_system',
-    'esp_timer',
-    'freertos',
-    'hal',
-    'heap',
-    'log',
-    'newlib',
-    'riscv',
-    'soc',
-    'xtensa',
-]
-DEFAULT_FULL_BUILD_TEST_FILEPATTERNS = [
-    # tools
-    'tools/cmake/**/*',
-    'tools/tools.json',
-    # ci
-    'tools/ci/ignore_build_warnings.txt',
-]
-DEFAULT_BUILD_LOG_FILENAME = 'build_log.txt'
-DEFAULT_SIZE_JSON_FILENAME = 'size.json'
 
 
 class CollectMode(str, Enum):

tools/ci/idf_pytest/script.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import fnmatch
 import io
@@ -22,9 +22,7 @@ from idf_ci_utils import to_list
 from idf_py_actions.constants import PREVIEW_TARGETS as TOOLS_PREVIEW_TARGETS
 from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGETS
 
-from .constants import DEFAULT_BUILD_LOG_FILENAME
 from .constants import DEFAULT_CONFIG_RULES_STR
-from .constants import DEFAULT_SIZE_JSON_FILENAME
 from .constants import CollectMode
 from .constants import PytestCase
 from .plugin import IdfPytestEmbedded
@@ -163,8 +161,8 @@ def get_all_apps(
                 recursive=True,
                 build_dir='build_@t_@w',
                 config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR,
-                build_log_filename=DEFAULT_BUILD_LOG_FILENAME,
+                build_log_filename='build_log.txt',
-                size_json_filename=DEFAULT_SIZE_JSON_FILENAME,
+                size_json_filename='size.json',
                 check_warnings=True,
                 manifest_rootpath=IDF_PATH,
                 compare_manifest_sha_filepath=compare_manifest_sha_filepath,