Merge branch 'ci/improve_job_name_and_concurrency' into 'master'

Ci/improve job name and concurrency

Closes IDF-7197

See merge request espressif/esp-idf!23086

.gitlab-ci.yml

@@ -150,7 +150,7 @@ cache:
   # target test jobs
   if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
     # ttfw jobs
-    if ! echo "${CI_JOB_NAME}" | egrep ".+_pytest_.+"; then
+    if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
       run_cmd bash install.sh --enable-ci --enable-ttfw
     else
       run_cmd bash install.sh --enable-ci --enable-pytest
@@ -160,7 +160,7 @@ cache:
   elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
     run_cmd bash install.sh --enable-ci --enable-pytest
   else
-    if ! echo "${CI_JOB_NAME}" | egrep ".+_pytest_.+"; then
+    if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
       run_cmd bash install.sh --enable-ci
     else
       run_cmd bash install.sh --enable-ci --enable-pytest

.gitlab/ci/assign-test.yml

@@ -44,36 +44,6 @@ assign_example_test:
     TEST_TYPE: example_test
     TEST_DIR: examples
 
-assign_custom_test:
-  extends:
-    - .assign_test_template
-    - .rules:build:custom_test
-  needs:
-    - job: build_test_apps_esp32
-      artifacts: false
-      optional: true
-    - job: build_test_apps_esp32s2
-      artifacts: false
-      optional: true
-    - job: build_test_apps_esp32c2
-      artifacts: false
-      optional: true
-    - job: build_test_apps_esp32c3
-      artifacts: false
-      optional: true
-    - job: build_test_apps_esp32c6
-      artifacts: false
-      optional: true
-    - job: build_test_apps_esp32h2
-      artifacts: false
-      optional: true
-    - job: build_test_apps_esp32s3
-      artifacts: false
-      optional: true
-  variables:
-    TEST_TYPE: custom_test
-    TEST_DIR: tools/test_apps
-
 assign_unit_test:
   extends:
     - .assign_test_template

.gitlab/ci/build.yml

@@ -306,11 +306,12 @@ build_pytest_components_esp32h2:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32h2
+  parallel: 4
   variables:
     IDF_TARGET: esp32h2
     TEST_DIR: components
 
-build_non_test_components_apps:
+build_only_components_apps:
   extends:
     - .build_cmake_template
     - .rules:build:component_ut
@@ -405,6 +406,20 @@ build_pytest_test_apps_esp32h2:
     IDF_TARGET: esp32h2
     TEST_DIR: tools/test_apps
 
+build_only_tools_test_apps:
+  extends:
+    - .build_cmake_template
+    - .rules:build:custom_test
+  parallel: 9
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py tools/test_apps -v
+      -t all
+      --collect-size-info size_info.txt
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+
 .build_template_app_template:
   extends:
     - .build_template
@@ -660,67 +675,6 @@ build_examples_cmake_esp32h2:
     IDF_TARGET: esp32h2
     TEST_DIR: examples
 
-
-build_test_apps_esp32:
-  extends:
-    - .build_cmake_template
-    - .rules:build:custom_test-esp32
-  parallel: 2
-  variables:
-    IDF_TARGET: esp32
-    TEST_DIR: tools/test_apps
-
-build_test_apps_esp32s2:
-  extends:
-    - .build_cmake_template
-    - .rules:build:custom_test-esp32s2
-  parallel: 2
-  variables:
-    IDF_TARGET: esp32s2
-    TEST_DIR: tools/test_apps
-
-build_test_apps_esp32s3:
-  extends:
-    - .build_cmake_template
-    - .rules:build:custom_test-esp32s3
-  parallel: 2
-  variables:
-    IDF_TARGET: esp32s3
-    TEST_DIR: tools/test_apps
-
-build_test_apps_esp32c3:
-  extends:
-    - .build_cmake_template
-    - .rules:build:custom_test-esp32c3
-  parallel: 2
-  variables:
-    IDF_TARGET: esp32c3
-    TEST_DIR: tools/test_apps
-
-build_test_apps_esp32c2:
-  extends:
-    - .build_cmake_template
-    - .rules:build:custom_test-esp32c2
-  variables:
-    IDF_TARGET: esp32c2
-    TEST_DIR: tools/test_apps
-
-build_test_apps_esp32c6:
-  extends:
-    - .build_cmake_template
-    - .rules:build:custom_test-esp32c6
-  variables:
-    IDF_TARGET: esp32c6
-    TEST_DIR: tools/test_apps
-
-build_test_apps_esp32h2:
-  extends:
-    - .build_cmake_template
-    - .rules:build:custom_test-esp32h2
-  variables:
-    IDF_TARGET: esp32h2
-    TEST_DIR: tools/test_apps
-
 build_clang_test_apps_esp32:
   extends:
     - .build_cmake_clang_template

.gitlab/ci/rules.yml

@@ -1835,24 +1835,12 @@
     - <<: *if-protected
     - <<: *if-label-lan8720
 
-.rules:labels:iperf_stress_test:
-  rules:
-    - <<: *if-revert-branch
-      when: never
-    - <<: *if-label-iperf_stress_test
-
 .rules:labels:nvs_coverage:
   rules:
     - <<: *if-revert-branch
       when: never
     - <<: *if-label-nvs_coverage
 
-.rules:labels:weekend_test:
-  rules:
-    - <<: *if-revert-branch
-      when: never
-    - <<: *if-label-weekend_test
-
 .rules:test:component_ut-esp32:
   rules:
     - <<: *if-revert-branch

tools/ci/ci_build_apps.py

@@ -60,18 +60,13 @@ def get_pytest_apps(
     LOGGER.info(f'Found {len(app_dirs)} apps')
     app_dirs.sort()
 
-    default_size_json_path = 'size.json'
-    build_dir = 'build_@t_@w'
-    if target == 'linux':  # no esp_idf_size for linux target
-        default_size_json_path = None  # type: ignore
-
     apps = find_apps(
         app_dirs,
         target=target,
-        build_dir=build_dir,
+        build_dir='build_@t_@w',
         config_rules_str=config_rules_str,
         build_log_path='build_log.txt',
-        size_json_path=default_size_json_path,
+        size_json_path='size.json',
         check_warnings=True,
         manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')],
         default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets,
@@ -83,6 +78,9 @@ def get_pytest_apps(
         if not preserve_all and not is_test_related:
             app.preserve = False
 
+        if app.target == 'linux':
+            app._size_json_path = None  # no esp_idf_size for linux target
+
     return apps  # type: ignore
 
 
@@ -94,6 +92,7 @@ def get_cmake_apps(
     extra_default_build_targets: Optional[List[str]] = None,
 ) -> List[App]:
     ttfw_app_dirs = get_ttfw_app_paths(paths, target)
+
     apps = find_apps(
         paths,
         recursive=True,
@@ -119,6 +118,9 @@ def get_cmake_apps(
             LOGGER.debug('Skipping build app with pytest scripts: %s', app)
             continue
 
+        if app.target == 'linux':
+            app._size_json_path = None  # no esp_idf_size for linux target
+
         apps_for_build.append(app)
 
     return apps_for_build

tools/test_apps/README.md

@@ -12,9 +12,7 @@ Test applications are treated the same way as ESP-IDF examples, so each project
 
 - Build recipe in cmake and the main component with app sources
 - Configuration files, `sdkconfig.ci` and similar (see below)
-- Test executor in `ttfw_idf` format if the project is intended to also run tests (otherwise the example is build only)
+- Python test scripts, `pytest_....py` (optional)
-  - test file in the project dir must end with `_test.py`, by should be named `app_test.py`
-  - test cases shall be decorated with `@ttfw_idf.idf_custom_test(env_tag="...")`
 
 ## Test Apps layout
 
@@ -39,52 +37,6 @@ The CI system expects to see at least a "default" config, so add `sdkconfig.ci`
 - By default, every CI configurations is built for every target SoC (an `m * n` configuration matrix). However if any `sdkconfig.ci.*` file contains a line of the form `CONFIG_IDF_TARGET="targetname"` then that CI config is only built for that one target. This only works in `sdkconfig.ci.CONFIG`, not in the default `sdkconfig.ci`.
 - Each configuration is also built with the contents of any `sdkconfig.defaults` file or a file named `sdkconfig.defaults.<TARGET>` appended. (Same as a normal ESP-IDF project build.)
 
-### Test Execution
+## Test Apps local execution
-
-If an example test or test app test supports more targets than just `ESP32`, then the `app_test.py` file needs to specify the list of supported targets in the test decorator. For example:
-
-```python
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2'])
-def test_app_xyz(env, extra_data):
-```
-
-If the app test supports multiple targets but you only want some of these targets to be run automatically in CI, the list can be further filtered down by adding the `ci_target` list:
-
-```python
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2'], ci_target=['esp32'])
-def test_app_xyz(env, extra_data):
-```
-
-(If no ci_target list is specified, all supported targets will be tested in CI.)
-
-## Test Apps local execution (ttfw)
-
-All the following instructions are general. Part of them may be complemented by more particular instructions in the corresponding app's README.
-
-### Requirements
-
-Install Python dependencies and export the Python path where the IDF CI Python modules are found with the following commands:
-
-```bash
-bash install.sh --enable-ttfw
-source export.sh
-export PYTHONPATH=$IDF_PATH/tools/ci/python_packages:$PYTHONPATH
-```
-You should also set the port via the environment variable `ESPPORT` to prevent the tools from looking and iterating over all serial ports. The latter causes much trouble, currently:
-
-```
-export ESPPORT=/dev/ttyUSB<X>
-```
-
-### Execution
-
-- Create an sdkconfig file from the relevant `sdkconfig.ci.<CONFIG>` and `sdkconfig.defaults`: `cat sdkconfig.defaults sdkconfig.ci.<CONFIG> > sdkconfig`
-- Run `idf.py menuconfig` to configure local project attributes
-- Run `idf.py build` to build the test app
-- Run `python app_test.py` to run the test locally
-
-## Test Apps local execution (pytest)
 
 Some of the examples have `pytest_....py` scripts that are using the `pytest` as the test framework. For detailed information, please refer to the "Run the Tests Locally" Section under [ESP-IDF tests in Pytest documentation](../../docs/en/contribute/esp-idf-tests-with-pytest.rst)
-
-Using `pytest` is the recommended way to write new tests. We will migrate all the test apps scripts to this new framework soon.

tools/test_apps/build_system/custom_partition_subtypes/CMakeLists.txt

@@ -3,6 +3,7 @@
 cmake_minimum_required(VERSION 3.16)
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+set(COMPONENTS main partition_table custom)
 
 project(custom_partition_subtypes)
 
@@ -20,6 +21,10 @@ add_custom_command(OUTPUT ${blank_file}
     ${partition_size} ${blank_file})
 
 add_custom_target(blank_bin ALL DEPENDS ${blank_file})
-add_dependencies(flash blank_bin)
 
-esptool_py_flash_to_partition(flash "${partition}" "${blank_file}")
+idf_build_get_property(target IDF_TARGET)
+
+if(NOT ${target} STREQUAL "linux")
+    add_dependencies(flash blank_bin)
+    esptool_py_flash_to_partition(flash "${partition}" "${blank_file}")
+endif()

tools/test_apps/build_system/custom_partition_subtypes/main/CMakeLists.txt

@@ -1,2 +1,3 @@
-idf_component_register(SRCS test_main.c)
+idf_component_register(SRCS test_main.c
+                       REQUIRES esp_partition)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml

@@ -1,2 +0,0 @@
-CaseConfig:
-- name: test_app_protocol_mqtt_publish_connect

tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml

@@ -1,6 +0,0 @@
-CaseConfig:
-- name: test_app_protocol_mqtt_publish_connect
-  overwrite:
-    dut:
-      class: ESP32QEMUDUT
-      package: ttfw_idf

tools/test_apps/security/secure_boot/README.md

@@ -95,21 +95,22 @@ Under `Security features`
 
 - Set UART ROM download mode to ENABLED (Required for the script to read the EFUSE)
 
-- Install and export TTFW requirements
+- Install pytest requirements
-```
-python -m pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt
 
-export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages"
+    ```
-```
+    bash $IDF_PATH/install.sh --enable-pytest
+    ```
 
 ### Build and test
 
 - Build the example
-```
+
-idf.py build
+    ```
-```
+    idf.py build
+    ```
 
 - Run the example test
-```
+
-python example_test.py
+    ```
-```
+    pytest --target esp32c3
+    ```