Merge branch 'ci/fixes' into 'master'

ci: a few fixes

Closes IDFCI-3018

See merge request espressif/esp-idf!40610

.gitlab/CODEOWNERS

@@ -51,6 +51,8 @@
 /.github/workflows/                   @esp-idf-codeowners/ci
 /.gitlab-ci.yml                       @esp-idf-codeowners/ci
 /.gitlab/ci/                          @esp-idf-codeowners/ci
+/.idf_build_apps.toml                 @esp-idf-codeowners/ci
+/.idf_ci.toml                         @esp-idf-codeowners/ci
 /.pre-commit-config.yaml              @esp-idf-codeowners/ci
 /.readthedocs.yml                     @esp-idf-codeowners/docs
 /.vale.ini                            @esp-idf-codeowners/docs

.gitlab/ci/host-test.yml

@@ -300,13 +300,11 @@ test_pytest_qemu:
     - run_cmd idf-ci build run
       --build-system cmake
       --target $IDF_TARGET
-      --only-test-related
       -m qemu
       --modified-files ${MR_MODIFIED_FILES}
     - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - run_cmd pytest
       --target $IDF_TARGET
-      --log-cli-level DEBUG
       -m qemu
       --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml

.idf_build_apps.toml

@@ -21,13 +21,12 @@ ignore_warning_files = [
 
 build_dir = "build_@t_@w"
 build_log_filename = "build_log.txt"
-size_json_filename = "size.json"
+size_json_filename = "size_${CI_JOB_ID}.json"
 
 verbose = 1  # INFO
 
 # collect
 collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt"
-collect_size_info_filename = "size_info_${CI_JOB_NAME_SLUG}.txt"  # TODO remove this file when ci-dashboard is ready
 junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml"
 
 # manifest

.idf_ci.toml

@@ -74,7 +74,7 @@ patterns = [
 bucket = "idf-artifacts"
 patterns = [
     '**/build*/build_log.txt',
-    '**/build*/size.json',
+    '**/build*/size*.json',
 ]
 
 [gitlab.artifacts.s3.junit]

components/esp_driver_usb_serial_jtag/test_apps/usb_serial_jtag_vfs/pytest_usb_serial_jtag_vfs.py

@@ -13,9 +13,10 @@ from pytest_embedded_idf.utils import idf_parametrize
     ],
     indirect=True,
 )
-@pytest.mark.parametrize('test_message', ['test123456789!@#%^&*'])
 @idf_parametrize('target', ['esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'], indirect=['target'])
-def test_usj_vfs_select(dut: Dut, test_message: list) -> None:
+def test_usj_vfs_select(dut: Dut) -> None:
+    test_message = 'test123456789!@#%^&*'
+
     dut.expect_exact('Press ENTER to see the list of tests')
     dut.write('"test select read, write and timeout"')
     dut.expect_exact('select timed out', timeout=2)
@@ -32,9 +33,10 @@ def test_usj_vfs_select(dut: Dut, test_message: list) -> None:
     ],
     indirect=True,
 )
-@pytest.mark.parametrize('test_message', ['!(@*#&(!*@&#((SDasdkjhad\nce'])
 @idf_parametrize('target', ['esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'], indirect=['target'])
-def test_usj_vfs_read_return(dut: Dut, test_message: list) -> None:
+def test_usj_vfs_read_return(dut: Dut) -> None:
+    test_message = '!(@*#&(!*@&#((SDasdkjhad\nce'
+
     dut.expect_exact('Press ENTER to see the list of tests')
     dut.write('"read does not return on new line character"')
     dut.expect_exact('ready to receive', timeout=2)
@@ -50,9 +52,10 @@ def test_usj_vfs_read_return(dut: Dut, test_message: list) -> None:
     ],
     indirect=True,
 )
-@pytest.mark.parametrize('test_message', ['testdata'])
 @idf_parametrize('target', ['esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'], indirect=['target'])
-def test_usj_vfs_read_blocking(dut: Dut, test_message: list) -> None:
+def test_usj_vfs_read_blocking(dut: Dut) -> None:
+    test_message = 'testdata'
+
     dut.expect_exact('Press ENTER to see the list of tests')
     dut.write('"blocking read returns with available data"')
     dut.expect_exact('ready to receive', timeout=2)

components/esp_vfs_console/test_apps/usb_cdc_vfs/pytest_usb_cdc_vfs.py

@@ -13,9 +13,10 @@ from pytest_embedded_idf.utils import idf_parametrize
     ],
     indirect=True,
 )
-@pytest.mark.parametrize('test_message', ['test123456789!@#%^&*'])
 @idf_parametrize('target', ['esp32s3'], indirect=['target'])
-def test_usb_cdc_vfs_default(dut: Dut, test_message: str) -> None:
+def test_usb_cdc_vfs_default(dut: Dut) -> None:
+    test_message = 'test123456789!@#%^&*'
+
     # test run: test_usb_cdc_select
     dut.expect_exact('test_usb_cdc_select', timeout=2)
     dut.expect_exact('select timed out', timeout=2)

conftest.py

@@ -26,7 +26,6 @@ import signal
 import time
 import typing as t
 from copy import deepcopy
-from telnetlib import Telnet
 from urllib.parse import quote
 
 import common_test_methods  # noqa: F401
@@ -48,6 +47,7 @@ from pytest_embedded.utils import to_bytes
 from pytest_embedded.utils import to_str
 from pytest_embedded_idf.dut import IdfDut
 from pytest_embedded_idf.unity_tester import CaseTester
+from pytest_embedded_jtag._telnetlib.telnetlib import Telnet  # python 3.13 removed telnetlib, use this instead
 
 
 ############
@@ -300,13 +300,19 @@ def build_dir(
     """
     # download from minio on CI
     case: PytestCase = request.node.stash[IDF_CI_PYTEST_CASE_KEY]
-    if app_downloader:
+    if 'skip_app_downloader' in case.all_markers:
+        logging.debug('skip_app_downloader marker found, skip downloading app')
+        downloader = None
+    else:
+        downloader = app_downloader
+
+    if downloader:
         # somehow hardcoded...
         app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}')
         if requires_elf_or_map(case):
-            app_downloader.download_app(app_build_path)
+            downloader.download_app(app_build_path)
         else:
-            app_downloader.download_app(app_build_path, 'flash')
+            downloader.download_app(app_build_path, 'flash')
         check_dirs = [f'build_{target}_{config}']
     else:
         check_dirs = []

examples/openthread/pytest_otbr.py

@@ -7,10 +7,12 @@ import random
 import re
 import secrets
 import subprocess
+import sys
 import threading
 import time
 from typing import Tuple
 
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 import ot_ci_function as ocf
 import pexpect
 import pytest

examples/protocols/esp_http_client/pytest_esp_http_client.py

@@ -109,9 +109,6 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None:
 
 
 @pytest.mark.host_test
-# Currently we are just testing the build for esp_http_client on Linux target. So skipping the test run.
-# Later we will enable the test run for Linux target as well.
-@pytest.mark.skipif('config.getvalue("target") == "linux"', reason='Do not run on Linux')
 @pytest.mark.parametrize(
     'config',
     [
@@ -121,5 +118,11 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None:
     indirect=True,
 )
 @idf_parametrize('target', ['linux'], indirect=['target'])
-def test_examples_protocol_esp_http_client_linux(dut: Dut) -> None:
+def test_examples_protocol_esp_http_client_linux(target: str, dut: Dut) -> None:
+    if target == 'linux':
+        pytest.skip(
+            'Currently we are just testing the build for esp_http_client on Linux target. '
+            'So skipping the test run. Later we will enable the test run for Linux target as well.'
+        )
+
     dut.expect('Finish http example', timeout=60)

pytest.ini

@@ -39,6 +39,7 @@ junit_log_passing_tests = False
 markers =
     temp_skip_ci: mark test to be skipped in CI
     temp_skip: mark test to be skipped in CI and locally
+    skip_app_downloader: mark test required apps built locally, not downloaded from CI
     require_elf: mark test to be skipped if no elf file is found
 
 env_markers =

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -82,7 +82,7 @@
     # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally
     - run_cmd pytest $nodes
       --pipeline-id $PARENT_PIPELINE_ID
-      --junitxml=XUNIT_RESULT_${CI_JOB_NAME_SLUG}.xml
+      --junitxml=XUNIT_RESULT_${CI_JOB_ID}.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}

tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml

@@ -47,7 +47,11 @@ generate_pytest_child_pipeline:
     - build
     - shiny
   needs:
-    - build_test_related_apps  # won't work if the parallel count exceeds 100, now it's around 50
+    - job: build_test_related_apps
+      optional: true
+    - job: build_non_test_related_apps  # make sure all build jobs are passed
+      optional: true
+      artifacts: false
     - pipeline: $PARENT_PIPELINE_ID
       job: pipeline_variables
   artifacts:

tools/ci/idf_pytest/plugin.py

@@ -150,6 +150,9 @@ class IdfLocalPlugin:
             if 'esp32c2' in case.targets and 'xtal_26mhz' not in case.all_markers:
                 item.add_marker('xtal_40mhz')
 
+            if 'host_test' in case.all_markers:
+                item.add_marker('skip_app_downloader')  # host_test jobs will build the apps itself
+
     def pytest_custom_test_case_name(self, item: Function) -> str:
         return item.funcargs.get('test_case_name', item.nodeid)  # type: ignore
 

tools/test_apps/system/panic/pytest_panic.py

@@ -936,11 +936,10 @@ def test_rtc_fast_reg1_execute_violation(dut: PanicTestDut, test_func_name: str)
 
 
 @pytest.mark.generic
-@pytest.mark.skipif(
+@pytest.mark.temp_skip(
-    'config.getvalue("target") in ["esp32c5", "esp32c6", "esp32h2", "esp32p4", "esp32h21"]',
+    targets=['esp32c5', 'esp32c6', 'esp32h2', 'esp32p4', 'esp32h21'],
-    reason='Not a violation condition, no PMS peripheral case',
+    reason='Not a violation condition, no PMS peripheral cases',
 )
-@pytest.mark.temp_skip_ci(targets=['esp32h21'], reason='lack of runners')
 @idf_parametrize('config, target', CONFIGS_MEMPROT_RTC_FAST_MEM, indirect=['config', 'target'])
 def test_rtc_fast_reg2_execute_violation(dut: PanicTestDut, test_func_name: str) -> None:
     dut.run_test_func(test_func_name)