From 288fc561b7c4bee5f81d813627e028d95ebf8249 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Jul 2025 08:40:39 +0200 Subject: [PATCH 1/8] tests: fix skipif markers. use `temp_skip` instead --- .../esp_http_client/pytest_esp_http_client.py | 11 +++++++---- tools/test_apps/system/panic/pytest_panic.py | 7 +++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/protocols/esp_http_client/pytest_esp_http_client.py b/examples/protocols/esp_http_client/pytest_esp_http_client.py index c329a16503..6c44b8b3f7 100644 --- a/examples/protocols/esp_http_client/pytest_esp_http_client.py +++ b/examples/protocols/esp_http_client/pytest_esp_http_client.py @@ -109,9 +109,6 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None: @pytest.mark.host_test -# Currently we are just testing the build for esp_http_client on Linux target. So skipping the test run. -# Later we will enable the test run for Linux target as well. -@pytest.mark.skipif('config.getvalue("target") == "linux"', reason='Do not run on Linux') @pytest.mark.parametrize( 'config', [ @@ -121,5 +118,11 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None: indirect=True, ) @idf_parametrize('target', ['linux'], indirect=['target']) -def test_examples_protocol_esp_http_client_linux(dut: Dut) -> None: +def test_examples_protocol_esp_http_client_linux(target: str, dut: Dut) -> None: + if target == 'linux': + pytest.skip( + 'Currently we are just testing the build for esp_http_client on Linux target. ' + 'So skipping the test run. Later we will enable the test run for Linux target as well.' + ) + dut.expect('Finish http example', timeout=60) diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py index 76c77c076c..23f062f063 100644 --- a/tools/test_apps/system/panic/pytest_panic.py +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -936,11 +936,10 @@ def test_rtc_fast_reg1_execute_violation(dut: PanicTestDut, test_func_name: str) @pytest.mark.generic -@pytest.mark.skipif( - 'config.getvalue("target") in ["esp32c5", "esp32c6", "esp32h2", "esp32p4", "esp32h21"]', - reason='Not a violation condition, no PMS peripheral case', +@pytest.mark.temp_skip( + targets=['esp32c5', 'esp32c6', 'esp32h2', 'esp32p4', 'esp32h21'], + reason='Not a violation condition, no PMS peripheral cases', ) -@pytest.mark.temp_skip_ci(targets=['esp32h21'], reason='lack of runners') @idf_parametrize('config, target', CONFIGS_MEMPROT_RTC_FAST_MEM, indirect=['config', 'target']) def test_rtc_fast_reg2_execute_violation(dut: PanicTestDut, test_func_name: str) -> None: dut.run_test_func(test_func_name) From d1a860bda5c1079d54a6fc3a6f83ec07e2abb52e Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Jul 2025 08:41:17 +0200 Subject: [PATCH 2/8] ci: use fixed telnetlib since python 3.13 removed this from stdlib --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index a37ec4d3f0..5bc43efe1b 100644 --- a/conftest.py +++ b/conftest.py @@ -26,7 +26,6 @@ import signal import time import typing as t from copy import deepcopy -from telnetlib import Telnet from urllib.parse import quote import common_test_methods # noqa: F401 @@ -48,6 +47,7 @@ from pytest_embedded.utils import to_bytes from pytest_embedded.utils import to_str from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.unity_tester import CaseTester +from pytest_embedded_jtag._telnetlib.telnetlib import Telnet # python 3.13 removed telnetlib, use this instead ############ From d280d364058f1d41be6b059e358a23023a5484d9 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Jul 2025 08:42:58 +0200 Subject: [PATCH 3/8] tests: fix special characters in parametrize with variables --- .../pytest_usb_serial_jtag_vfs.py | 15 +++++++++------ .../test_apps/usb_cdc_vfs/pytest_usb_cdc_vfs.py | 5 +++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/components/esp_driver_usb_serial_jtag/test_apps/usb_serial_jtag_vfs/pytest_usb_serial_jtag_vfs.py b/components/esp_driver_usb_serial_jtag/test_apps/usb_serial_jtag_vfs/pytest_usb_serial_jtag_vfs.py index f64dc7298f..2a4d28565e 100644 --- a/components/esp_driver_usb_serial_jtag/test_apps/usb_serial_jtag_vfs/pytest_usb_serial_jtag_vfs.py +++ b/components/esp_driver_usb_serial_jtag/test_apps/usb_serial_jtag_vfs/pytest_usb_serial_jtag_vfs.py @@ -13,9 +13,10 @@ from pytest_embedded_idf.utils import idf_parametrize ], indirect=True, ) -@pytest.mark.parametrize('test_message', ['test123456789!@#%^&*']) @idf_parametrize('target', ['esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'], indirect=['target']) -def test_usj_vfs_select(dut: Dut, test_message: list) -> None: +def test_usj_vfs_select(dut: Dut) -> None: + test_message = 'test123456789!@#%^&*' + dut.expect_exact('Press ENTER to see the list of tests') dut.write('"test select read, write and timeout"') dut.expect_exact('select timed out', timeout=2) @@ -32,9 +33,10 @@ def test_usj_vfs_select(dut: Dut, test_message: list) -> None: ], indirect=True, ) -@pytest.mark.parametrize('test_message', ['!(@*#&(!*@&#((SDasdkjhad\nce']) @idf_parametrize('target', ['esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'], indirect=['target']) -def test_usj_vfs_read_return(dut: Dut, test_message: list) -> None: +def test_usj_vfs_read_return(dut: Dut) -> None: + test_message = '!(@*#&(!*@&#((SDasdkjhad\nce' + dut.expect_exact('Press ENTER to see the list of tests') dut.write('"read does not return on new line character"') dut.expect_exact('ready to receive', timeout=2) @@ -50,9 +52,10 @@ def test_usj_vfs_read_return(dut: Dut, test_message: list) -> None: ], indirect=True, ) -@pytest.mark.parametrize('test_message', ['testdata']) @idf_parametrize('target', ['esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'], indirect=['target']) -def test_usj_vfs_read_blocking(dut: Dut, test_message: list) -> None: +def test_usj_vfs_read_blocking(dut: Dut) -> None: + test_message = 'testdata' + dut.expect_exact('Press ENTER to see the list of tests') dut.write('"blocking read returns with available data"') dut.expect_exact('ready to receive', timeout=2) diff --git a/components/esp_vfs_console/test_apps/usb_cdc_vfs/pytest_usb_cdc_vfs.py b/components/esp_vfs_console/test_apps/usb_cdc_vfs/pytest_usb_cdc_vfs.py index e1408e95a3..8693a8b0b9 100644 --- a/components/esp_vfs_console/test_apps/usb_cdc_vfs/pytest_usb_cdc_vfs.py +++ b/components/esp_vfs_console/test_apps/usb_cdc_vfs/pytest_usb_cdc_vfs.py @@ -13,9 +13,10 @@ from pytest_embedded_idf.utils import idf_parametrize ], indirect=True, ) -@pytest.mark.parametrize('test_message', ['test123456789!@#%^&*']) @idf_parametrize('target', ['esp32s3'], indirect=['target']) -def test_usb_cdc_vfs_default(dut: Dut, test_message: str) -> None: +def test_usb_cdc_vfs_default(dut: Dut) -> None: + test_message = 'test123456789!@#%^&*' + # test run: test_usb_cdc_select dut.expect_exact('test_usb_cdc_select', timeout=2) dut.expect_exact('select timed out', timeout=2) From 95fbd6b81aa4ce4e8da7bd3a966b1495a5d290cc Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Jul 2025 09:01:53 +0200 Subject: [PATCH 4/8] ci: make sure test child pipeline only triggered when all build jobs succeeded --- .../ci/dynamic_pipelines/templates/test_child_pipeline.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index 0828a81fef..08154e01e9 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -47,7 +47,11 @@ generate_pytest_child_pipeline: - build - shiny needs: - - build_test_related_apps # won't work if the parallel count exceeds 100, now it's around 50 + - job: build_test_related_apps + optional: true + - job: build_non_test_related_apps # make sure all build jobs are passed + optional: true + artifacts: false - pipeline: $PARENT_PIPELINE_ID job: pipeline_variables artifacts: From 158eb78555b3971723fa7966c7c6ca0c7ad76908 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Jul 2025 13:53:44 +0200 Subject: [PATCH 5/8] ci: add `CI_JOB_ID` as suffix to help ci-dashboard parse info easier --- .idf_build_apps.toml | 3 +-- .idf_ci.toml | 2 +- tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.idf_build_apps.toml b/.idf_build_apps.toml index bbc0025290..470ad263ab 100644 --- a/.idf_build_apps.toml +++ b/.idf_build_apps.toml @@ -21,13 +21,12 @@ ignore_warning_files = [ build_dir = "build_@t_@w" build_log_filename = "build_log.txt" -size_json_filename = "size.json" +size_json_filename = "size_${CI_JOB_ID}.json" verbose = 1 # INFO # collect collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt" -collect_size_info_filename = "size_info_${CI_JOB_NAME_SLUG}.txt" # TODO remove this file when ci-dashboard is ready junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml" # manifest diff --git a/.idf_ci.toml b/.idf_ci.toml index 84fdd0d9ad..cc97333f84 100644 --- a/.idf_ci.toml +++ b/.idf_ci.toml @@ -74,7 +74,7 @@ patterns = [ bucket = "idf-artifacts" patterns = [ '**/build*/build_log.txt', - '**/build*/size.json', + '**/build*/size*.json', ] [gitlab.artifacts.s3.junit] diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 14596eebbf..f42e7c8bbb 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -82,7 +82,7 @@ # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally - run_cmd pytest $nodes --pipeline-id $PARENT_PIPELINE_ID - --junitxml=XUNIT_RESULT_${CI_JOB_NAME_SLUG}.xml + --junitxml=XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} From 76181e5fced3cda8e6e749e39e549f91ef29d6c8 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Jul 2025 10:35:21 +0200 Subject: [PATCH 6/8] ci: skip app downloader for host test cases --- .gitlab/ci/host-test.yml | 2 -- conftest.py | 12 +++++++++--- pytest.ini | 1 + tools/ci/idf_pytest/plugin.py | 3 +++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 79bfdc19ca..b5843f2763 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -300,13 +300,11 @@ test_pytest_qemu: - run_cmd idf-ci build run --build-system cmake --target $IDF_TARGET - --only-test-related -m qemu --modified-files ${MR_MODIFIED_FILES} - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - run_cmd pytest --target $IDF_TARGET - --log-cli-level DEBUG -m qemu --embedded-services idf,qemu --junitxml=XUNIT_RESULT.xml diff --git a/conftest.py b/conftest.py index 5bc43efe1b..aa19094c2a 100644 --- a/conftest.py +++ b/conftest.py @@ -300,13 +300,19 @@ def build_dir( """ # download from minio on CI case: PytestCase = request.node.stash[IDF_CI_PYTEST_CASE_KEY] - if app_downloader: + if 'skip_app_downloader' in case.all_markers: + logging.debug('skip_app_downloader marker found, skip downloading app') + downloader = None + else: + downloader = app_downloader + + if downloader: # somehow hardcoded... app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}') if requires_elf_or_map(case): - app_downloader.download_app(app_build_path) + downloader.download_app(app_build_path) else: - app_downloader.download_app(app_build_path, 'flash') + downloader.download_app(app_build_path, 'flash') check_dirs = [f'build_{target}_{config}'] else: check_dirs = [] diff --git a/pytest.ini b/pytest.ini index cb9d30af80..5d570974f9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -39,6 +39,7 @@ junit_log_passing_tests = False markers = temp_skip_ci: mark test to be skipped in CI temp_skip: mark test to be skipped in CI and locally + skip_app_downloader: mark test required apps built locally, not downloaded from CI require_elf: mark test to be skipped if no elf file is found env_markers = diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index 81980a5d89..86c1005607 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -150,6 +150,9 @@ class IdfLocalPlugin: if 'esp32c2' in case.targets and 'xtal_26mhz' not in case.all_markers: item.add_marker('xtal_40mhz') + if 'host_test' in case.all_markers: + item.add_marker('skip_app_downloader') # host_test jobs will build the apps itself + def pytest_custom_test_case_name(self, item: Function) -> str: return item.funcargs.get('test_case_name', item.nodeid) # type: ignore From c5a562e6e00938f813ac36ad714092996e662a7d Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Jul 2025 12:45:54 +0200 Subject: [PATCH 7/8] ci: add examples/openthread to PYTHONPATH --- examples/openthread/pytest_otbr.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/openthread/pytest_otbr.py b/examples/openthread/pytest_otbr.py index e3a968eec7..3663068858 100644 --- a/examples/openthread/pytest_otbr.py +++ b/examples/openthread/pytest_otbr.py @@ -7,10 +7,12 @@ import random import re import secrets import subprocess +import sys import threading import time from typing import Tuple +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import ot_ci_function as ocf import pexpect import pytest From d4cc559709cf26860e4f906304dc62e14c66a863 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Jul 2025 14:16:19 +0200 Subject: [PATCH 8/8] chore: add ci config files to CODEOWNER --- .gitlab/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index fbae991b36..815576c30f 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -51,6 +51,8 @@ /.github/workflows/ @esp-idf-codeowners/ci /.gitlab-ci.yml @esp-idf-codeowners/ci /.gitlab/ci/ @esp-idf-codeowners/ci +/.idf_build_apps.toml @esp-idf-codeowners/ci +/.idf_ci.toml @esp-idf-codeowners/ci /.pre-commit-config.yaml @esp-idf-codeowners/ci /.readthedocs.yml @esp-idf-codeowners/docs /.vale.ini @esp-idf-codeowners/docs