From 13547d44e0f7d7e0169d5c885108ecb051bba8ac Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 11 Feb 2022 15:53:46 +0800 Subject: [PATCH 1/5] ci: set build target test apps rules --- .gitlab/ci/dependencies/dependencies.yml | 1 + .gitlab/ci/rules.yml | 72 ++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 52fd8c9d9a..8a9fca3662 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -63,6 +63,7 @@ patterns: - build_components - build_system + - build_target_test included_in: - "build:{0}" - build:target_test diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 70321d9736..b39b28dadd 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -26,12 +26,6 @@ - "tools/ci/python_packages/tiny_test_fw/**/*" - "tools/ci/python_packages/ttfw_idf/**/*" - - "tools/ci/find_apps_build_apps.sh" - - "tools/ci/build_pytest_apps.py" - - "tools/build_apps.py" - - "tools/find_apps.py" - - "tools/find_build_apps/**/*" - - "tools/esp_prov/**/*" - "examples/**/*" @@ -43,6 +37,14 @@ - "components/**/*" - "examples/cxx/experimental/experimental_cpp_component/*" +.patterns-build_target_test: &patterns-build_target_test + - "tools/ci/find_apps_build_apps.sh" + - "tools/build_apps.py" + - "tools/find_apps.py" + - "tools/find_build_apps/**/*" + + - "tools/ci/build_pytest_apps.py" + .patterns-build_system: &patterns-build_system - "tools/cmake/**/*" - "tools/kconfig_new/**/*" @@ -437,6 +439,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-component_ut @@ -455,6 +459,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-component_ut @@ -473,6 +479,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-component_ut @@ -491,6 +499,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-component_ut @@ -509,6 +519,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-component_ut @@ -527,6 +539,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-component_ut @@ -545,6 +559,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-component_ut @@ -567,6 +583,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -584,6 +602,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -600,6 +620,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -616,6 +638,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -632,6 +656,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -648,6 +674,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -664,6 +692,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-custom_test @@ -698,6 +728,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test @@ -717,6 +749,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test @@ -735,6 +769,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test @@ -753,6 +789,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test @@ -771,6 +809,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test @@ -789,6 +829,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test @@ -807,6 +849,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test @@ -880,6 +924,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-component_ut - <<: *if-dev-push @@ -909,6 +955,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-unit_test @@ -925,6 +973,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-unit_test @@ -941,6 +991,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-unit_test @@ -957,6 +1009,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-unit_test @@ -973,6 +1027,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-unit_test @@ -989,6 +1045,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-unit_test @@ -1005,6 +1063,8 @@ changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system + - <<: *if-dev-push + changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-unit_test From 4f0393b0d19f6e889f89fd09dfaf4b563886ee32 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 18 Feb 2022 15:37:39 +0800 Subject: [PATCH 2/5] ci(pytest): add --sdkconfig to trigger only the sdkconfig tests --- conftest.py | 51 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/conftest.py b/conftest.py index fe107e7ffa..c43ada5d0d 100644 --- a/conftest.py +++ b/conftest.py @@ -29,6 +29,7 @@ from pytest_embedded.utils import find_by_suffix SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3'] PREVIEW_TARGETS = ['linux', 'esp32h2', 'esp32c2'] +DEFAULT_SDKCONFIG = 'default' ################## @@ -57,7 +58,7 @@ def item_marker_names(item: Item) -> List[str]: ############ @pytest.fixture def config(request: FixtureRequest) -> str: - return getattr(request, 'param', None) or request.config.getoption('config', 'default') # type: ignore + return getattr(request, 'param', None) or DEFAULT_SDKCONFIG @pytest.fixture @@ -67,7 +68,9 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str: @pytest.fixture @parse_configuration -def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], config: Optional[str]) -> str: +def build_dir( + request: FixtureRequest, app_path: str, target: Optional[str], config: Optional[str] +) -> str: """ Check local build dir with the following priority: @@ -85,8 +88,10 @@ def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], con Returns: valid build directory """ - param_or_cli: str = getattr(request, 'param', None) or request.config.option.__dict__.get('build_dir') - if param_or_cli is not None: # respect the parametrize and the cli + param_or_cli: str = getattr( + request, 'param', None + ) or request.config.option.__dict__.get('build_dir') + if param_or_cli is not None: # respect the param and the cli return param_or_cli check_dirs = [] @@ -104,16 +109,21 @@ def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], con logging.info(f'find valid binary path: {binary_path}') return check_dir - logging.warning(f'checking binary path: {binary_path}... missing... try another place') + logging.warning( + f'checking binary path: {binary_path}... missing... try another place' + ) recommend_place = check_dirs[0] logging.error( - f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again') + f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again' + ) sys.exit(1) @pytest.fixture(autouse=True) -def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, object], None]) -> None: +def junit_properties( + test_case_name: str, record_xml_attribute: Callable[[str, object], None] +) -> None: """ This fixture is autoused and will modify the junit report test case name to .. """ @@ -123,12 +133,25 @@ def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, o ################## # Hook functions # ################## +def pytest_addoption(parser: pytest.Parser) -> None: + base_group = parser.getgroup('idf') + base_group.addoption( + '--sdkconfig', + help='sdkconfig postfix, like sdkconfig.ci.. (Default: None, which would build all found apps)', + ) + + @pytest.hookimpl(tryfirst=True) -def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None: +def pytest_collection_modifyitems(config: Config, items: List[Function]) -> None: target = config.getoption('target', None) # use the `build` dir if not target: return + def _get_param_config(_item: Function) -> str: + if hasattr(_item, 'callspec'): + return _item.callspec.params.get('config', DEFAULT_SDKCONFIG) # type: ignore + return DEFAULT_SDKCONFIG + # add markers for special markers for item in items: if 'supported_targets' in item_marker_names(item): @@ -144,6 +167,14 @@ def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None: # filter all the test cases with "--target" items[:] = [item for item in items if target in item_marker_names(item)] + # filter all the test cases with cli option "config" + if config.getoption('sdkconfig'): + items[:] = [ + item + for item in items + if _get_param_config(item) == config.getoption('sdkconfig') + ] + @pytest.hookimpl(trylast=True) def pytest_runtest_teardown(item: Function) -> None: @@ -166,5 +197,7 @@ def pytest_runtest_teardown(item: Function) -> None: for case in testcases: case.attrib['name'] = format_case_id(target, config, case.attrib['name']) if 'file' in case.attrib: - case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework + case.attrib['file'] = case.attrib['file'].replace( + '/IDF/', '' + ) # our unity test framework xml.write(junit) From 1b095db5c9f3f10b579371f4d48fd7b3078b1c5e Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 11 Feb 2022 14:58:50 +0800 Subject: [PATCH 3/5] ci(pytest): refactor panic test to pytest --- conftest.py | 7 +- tools/ci/check_copyright_config.yaml | 1 + tools/test_apps/system/panic/app_test.py | 345 ------------------ tools/test_apps/system/panic/conftest.py | 279 ++++++++++++++ tools/test_apps/system/panic/panic_tests.py | 196 ---------- tools/test_apps/system/panic/pytest_panic.py | 287 +++++++++++++++ .../system/panic/test_panic_util/__init__.py | 0 .../panic/test_panic_util/test_panic_util.py | 319 ---------------- 8 files changed, 573 insertions(+), 861 deletions(-) delete mode 100644 tools/test_apps/system/panic/app_test.py create mode 100644 tools/test_apps/system/panic/conftest.py delete mode 100644 tools/test_apps/system/panic/panic_tests.py create mode 100644 tools/test_apps/system/panic/pytest_panic.py delete mode 100644 tools/test_apps/system/panic/test_panic_util/__init__.py delete mode 100644 tools/test_apps/system/panic/test_panic_util/test_panic_util.py diff --git a/conftest.py b/conftest.py index c43ada5d0d..12a88a88a8 100644 --- a/conftest.py +++ b/conftest.py @@ -61,6 +61,11 @@ def config(request: FixtureRequest) -> str: return getattr(request, 'param', None) or DEFAULT_SDKCONFIG +@pytest.fixture +def test_func_name(request: FixtureRequest) -> str: + return request.node.function.__name__ # type: ignore + + @pytest.fixture def test_case_name(request: FixtureRequest, target: str, config: str) -> str: return format_case_id(target, config, request.node.originalname) @@ -110,7 +115,7 @@ def build_dir( return check_dir logging.warning( - f'checking binary path: {binary_path}... missing... try another place' + 'checking binary path: %s... missing... try another place', binary_path ) recommend_place = check_dirs[0] diff --git a/tools/ci/check_copyright_config.yaml b/tools/ci/check_copyright_config.yaml index f494074933..250b9a6035 100644 --- a/tools/ci/check_copyright_config.yaml +++ b/tools/ci/check_copyright_config.yaml @@ -52,6 +52,7 @@ examples_and_unit_tests: - 'examples/' - 'components/**/test/**' - 'components/**/test_apps/**' + - 'tools/test_apps/**' allowed_licenses: - Apache-2.0 - Unlicense diff --git a/tools/test_apps/system/panic/app_test.py b/tools/test_apps/system/panic/app_test.py deleted file mode 100644 index 5807231447..0000000000 --- a/tools/test_apps/system/panic/app_test.py +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env python -import sys - -import panic_tests as test -from test_panic_util.test_panic_util import panic_test, run_all - - -# test_task_wdt -@panic_test(target=['ESP32', 'ESP32S2']) -def test_panic_task_wdt(env, _extra_data): - test.task_wdt_inner(env, 'panic') - - -@panic_test() -def test_coredump_task_wdt_uart_elf_crc(env, _extra_data): - test.task_wdt_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_task_wdt_uart_bin_crc(env, _extra_data): - test.task_wdt_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_task_wdt_flash_elf_sha(env, _extra_data): - test.task_wdt_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_task_wdt_flash_bin_crc(env, _extra_data): - test.task_wdt_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_task_wdt(env, _extra_data): - test.task_wdt_inner(env, 'gdbstub') - - -# test_int_wdt - -@panic_test() -def test_panic_int_wdt(env, _extra_data): - test.int_wdt_inner(env, 'panic') - - -@panic_test() -def test_coredump_int_wdt_uart_elf_crc(env, _extra_data): - test.int_wdt_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_int_wdt_uart_bin_crc(env, _extra_data): - test.int_wdt_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_int_wdt_flash_elf_sha(env, _extra_data): - test.int_wdt_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_int_wdt_flash_bin_crc(env, _extra_data): - test.int_wdt_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_int_wdt(env, _extra_data): - test.int_wdt_inner(env, 'gdbstub') - - -# test_int_wdt_cache_disabled - -@panic_test() -def test_panic_int_wdt_cache_disabled(env, _extra_data): - test.int_wdt_cache_disabled_inner(env, 'panic') - - -@panic_test() -def test_coredump_int_wdt_cache_disabled_uart_elf_crc(env, _extra_data): - test.int_wdt_cache_disabled_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_int_wdt_cache_disabled_uart_bin_crc(env, _extra_data): - test.int_wdt_cache_disabled_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_int_wdt_cache_disabled_flash_elf_sha(env, _extra_data): - test.int_wdt_cache_disabled_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_int_wdt_cache_disabled_flash_bin_crc(env, _extra_data): - test.int_wdt_cache_disabled_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_int_wdt_cache_disabled(env, _extra_data): - test.int_wdt_cache_disabled_inner(env, 'gdbstub') - - -# test_cache_error - -@panic_test() -def test_panic_cache_error(env, _extra_data): - test.cache_error_inner(env, 'panic') - - -@panic_test() -def test_coredump_cache_error_uart_elf_crc(env, _extra_data): - test.cache_error_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_cache_error_uart_bin_crc(env, _extra_data): - test.cache_error_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_cache_error_flash_elf_sha(env, _extra_data): - test.cache_error_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_cache_error_flash_bin_crc(env, _extra_data): - test.cache_error_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_cache_error(env, _extra_data): - test.cache_error_inner(env, 'gdbstub') - - -# test_stack_overflow - -@panic_test(target=['ESP32', 'ESP32S2']) -def test_panic_stack_overflow(env, _extra_data): - test.stack_overflow_inner(env, 'panic') - - -@panic_test() -def test_coredump_stack_overflow_uart_elf_crc(env, _extra_data): - test.stack_overflow_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_stack_overflow_uart_bin_crc(env, _extra_data): - test.stack_overflow_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_stack_overflow_flash_elf_sha(env, _extra_data): - test.stack_overflow_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_stack_overflow_flash_bin_crc(env, _extra_data): - test.stack_overflow_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_stack_overflow(env, _extra_data): - test.stack_overflow_inner(env, 'gdbstub') - - -# test_instr_fetch_prohibited - -@panic_test(target=['ESP32', 'ESP32S2']) -def test_panic_instr_fetch_prohibited(env, _extra_data): - test.instr_fetch_prohibited_inner(env, 'panic') - - -@panic_test() -def test_coredump_instr_fetch_prohibited_uart_elf_crc(env, _extra_data): - test.instr_fetch_prohibited_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_instr_fetch_prohibited_uart_bin_crc(env, _extra_data): - test.instr_fetch_prohibited_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_instr_fetch_prohibited_flash_elf_sha(env, _extra_data): - test.instr_fetch_prohibited_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_instr_fetch_prohibited_flash_bin_crc(env, _extra_data): - test.instr_fetch_prohibited_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_instr_fetch_prohibited(env, _extra_data): - test.instr_fetch_prohibited_inner(env, 'gdbstub') - - -# test_illegal_instruction - -@panic_test(target=['ESP32', 'ESP32S2']) -def test_panic_illegal_instruction(env, _extra_data): - test.illegal_instruction_inner(env, 'panic') - - -@panic_test() -def test_coredump_illegal_instruction_uart_elf_crc(env, _extra_data): - test.illegal_instruction_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_illegal_instruction_uart_bin_crc(env, _extra_data): - test.illegal_instruction_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_illegal_instruction_flash_elf_sha(env, _extra_data): - test.illegal_instruction_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_illegal_instruction_flash_bin_crc(env, _extra_data): - test.illegal_instruction_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_illegal_instruction(env, _extra_data): - test.illegal_instruction_inner(env, 'gdbstub') - - -# test_storeprohibited - -@panic_test(target=['ESP32', 'ESP32S2']) -def test_panic_storeprohibited(env, _extra_data): - test.storeprohibited_inner(env, 'panic') - - -@panic_test() -def test_coredump_storeprohibited_uart_elf_crc(env, _extra_data): - test.storeprohibited_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_storeprohibited_uart_bin_crc(env, _extra_data): - test.storeprohibited_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_storeprohibited_flash_elf_sha(env, _extra_data): - test.storeprohibited_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_storeprohibited_flash_bin_crc(env, _extra_data): - test.storeprohibited_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_storeprohibited(env, _extra_data): - test.storeprohibited_inner(env, 'gdbstub') - - -# test_abort - -@panic_test(target=['ESP32', 'ESP32S2']) -def test_panic_abort(env, _extra_data): - test.abort_inner(env, 'panic') - - -@panic_test(target=['ESP32']) -def test_panic_abort_cache_disabled(env, _extra_data): - test.abort_cached_disabled_inner(env, 'panic') - - -@panic_test() -def test_coredump_abort_uart_elf_crc(env, _extra_data): - test.abort_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_abort_uart_bin_crc(env, _extra_data): - test.abort_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_abort_flash_elf_sha(env, _extra_data): - test.abort_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_abort_flash_bin_crc(env, _extra_data): - test.abort_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_abort(env, _extra_data): - test.abort_inner(env, 'gdbstub') - - -# test_assert - -@panic_test(target=['ESP32', 'ESP32S2']) -def test_panic_assert(env, _extra_data): - test.assert_inner(env, 'panic') - - -@panic_test(target=['ESP32']) -def test_panic_assert_cache_disabled(env, _extra_data): - test.assert_cached_disabled_inner(env, 'panic') - - -# test_ub - -@panic_test() -def test_panic_ub(env, _extra_data): - test.ub_inner(env, 'panic') - - -@panic_test() -def test_coredump_ub_uart_elf_crc(env, _extra_data): - test.ub_inner(env, 'coredump_uart_elf_crc') - - -@panic_test() -def test_coredump_ub_uart_bin_crc(env, _extra_data): - test.ub_inner(env, 'coredump_uart_bin_crc') - - -@panic_test() -def test_coredump_ub_flash_elf_sha(env, _extra_data): - test.ub_inner(env, 'coredump_flash_elf_sha') - - -@panic_test() -def test_coredump_ub_flash_bin_crc(env, _extra_data): - test.ub_inner(env, 'coredump_flash_bin_crc') - - -@panic_test() -def test_gdbstub_ub(env, _extra_data): - test.ub_inner(env, 'gdbstub') - - -if __name__ == '__main__': - run_all(__file__, sys.argv[1:]) diff --git a/tools/test_apps/system/panic/conftest.py b/tools/test_apps/system/panic/conftest.py new file mode 100644 index 0000000000..f1c0c6f223 --- /dev/null +++ b/tools/test_apps/system/panic/conftest.py @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=W0621 # redefined-outer-name + +import hashlib +import logging +import os +import subprocess +import sys +from typing import Any, Dict, List, TextIO + +import pexpect +import pytest +from _pytest.fixtures import FixtureRequest +from _pytest.monkeypatch import MonkeyPatch +from pygdbmi.gdbcontroller import GdbController, GdbTimeoutError, NoGdbProcessError +from pytest_embedded_idf.app import IdfApp +from pytest_embedded_idf.dut import IdfDut +from pytest_embedded_idf.serial import IdfSerial + + +def sha256(file: str) -> str: + res = hashlib.sha256() + with open(file, 'rb') as fr: + res.update(fr.read()) + return res.hexdigest() + + +class PanicTestDut(IdfDut): + BOOT_CMD_ADDR = 0x9000 + BOOT_CMD_SIZE = 0x1000 + DEFAULT_EXPECT_TIMEOUT = 10 + COREDUMP_UART_START = '================= CORE DUMP START =================' + COREDUMP_UART_END = '================= CORE DUMP END =================' + + app: IdfApp + serial: IdfSerial + + def __init__(self, *args, **kwargs) -> None: # type: ignore + super().__init__(*args, **kwargs) + + self.gdb: GdbController = None # type: ignore + # record this since pygdbmi is using logging.debug to generate some single character mess + self.log_level = logging.getLogger().level + # pygdbmi is using logging.debug to generate some single character mess + if self.log_level <= logging.DEBUG: + logging.getLogger().setLevel(logging.INFO) + + self.coredump_output: TextIO = None # type: ignore + + def close(self) -> None: + if self.gdb: + self.gdb.exit() + + super().close() + + def revert_log_level(self) -> None: + logging.getLogger().setLevel(self.log_level) + + def expect_test_func_name(self, test_func_name: str) -> None: + self.expect_exact('Enter test name:') + self.write(test_func_name) + self.expect_exact('Got test name: ' + test_func_name) + + def expect_none(self, pattern, **kwargs) -> None: # type: ignore + """like dut.expect_all, but with an inverse logic""" + if 'timeout' not in kwargs: + kwargs['timeout'] = 1 + + try: + res = self.expect(pattern, **kwargs) + raise AssertionError(f'Unexpected: {res.group().decode("utf8")}') + except pexpect.TIMEOUT: + pass + + def expect_backtrace(self) -> None: + self.expect_exact('Backtrace:') + self.expect_none('CORRUPTED') + + def expect_gme(self, reason: str) -> None: + """Expect method for Guru Meditation Errors""" + self.expect_exact(f"Guru Meditation Error: Core 0 panic'ed ({reason})") + + def expect_reg_dump(self, core: int = 0) -> None: + """Expect method for the register dump""" + self.expect(r'Core\s+%d register dump:' % core) + + def expect_elf_sha256(self) -> None: + """Expect method for ELF SHA256 line""" + elf_sha256 = sha256(self.app.elf_file) + elf_sha256_len = int( + self.app.sdkconfig.get('CONFIG_APP_RETRIEVE_LEN_ELF_SHA', '16') + ) + self.expect_exact('ELF file SHA256: ' + elf_sha256[0:elf_sha256_len]) + + def _call_espcoredump( + self, extra_args: List[str], coredump_file_name: str, output_file_name: str + ) -> None: + # no "with" here, since we need the file to be open for later inspection by the test case + if not self.coredump_output: + self.coredump_output = open(output_file_name, 'w') + + espcoredump_script = os.path.join( + os.environ['IDF_PATH'], 'components', 'espcoredump', 'espcoredump.py' + ) + espcoredump_args = [ + sys.executable, + espcoredump_script, + 'info_corefile', + '--core', + coredump_file_name, + ] + espcoredump_args += extra_args + espcoredump_args.append(self.app.elf_file) + logging.info('Running %s', ' '.join(espcoredump_args)) + logging.info('espcoredump output is written to %s', self.coredump_output.name) + + subprocess.check_call(espcoredump_args, stdout=self.coredump_output) + self.coredump_output.flush() + self.coredump_output.seek(0) + + def process_coredump_uart(self) -> None: + """Extract the core dump from UART output of the test, run espcoredump on it""" + self.expect(self.COREDUMP_UART_START) + res = self.expect('(.+)' + self.COREDUMP_UART_END) + coredump_base64 = res.group(1).decode('utf8') + with open(os.path.join(self.logdir, 'coredump_data.b64'), 'w') as coredump_file: + logging.info('Writing UART base64 core dump to %s', coredump_file.name) + coredump_file.write(coredump_base64) + + output_file_name = os.path.join(self.logdir, 'coredump_uart_result.txt') + self._call_espcoredump( + ['--core-format', 'b64'], coredump_file.name, output_file_name + ) + + def process_coredump_flash(self) -> None: + """Extract the core dump from flash, run espcoredump on it""" + coredump_file_name = os.path.join(self.logdir, 'coredump_data.bin') + logging.info('Writing flash binary core dump to %s', coredump_file_name) + self.serial.dump_flash(coredump_file_name, partition='coredump') + + output_file_name = os.path.join(self.logdir, 'coredump_flash_result.txt') + self._call_espcoredump( + ['--core-format', 'raw'], coredump_file_name, output_file_name + ) + + def gdb_write(self, command: str) -> Any: + """ + Wrapper to write to gdb with a longer timeout, as test runner + host can be slow sometimes + """ + return self.gdb.write(command, timeout_sec=10) + + def start_gdb(self) -> None: + """ + Runs GDB and connects it to the "serial" port of the DUT. + After this, the DUT expect methods can no longer be used to capture output. + """ + self.gdb = GdbController(gdb_path=self.toolchain_prefix + 'gdb') + + # pygdbmi logs to console by default, make it log to a file instead + pygdbmi_log_file_name = os.path.join(self.logdir, 'pygdbmi_log.txt') + pygdbmi_logger = self.gdb.logger + pygdbmi_logger.setLevel(logging.DEBUG) + while pygdbmi_logger.hasHandlers(): + pygdbmi_logger.removeHandler(pygdbmi_logger.handlers[0]) + log_handler = logging.FileHandler(pygdbmi_log_file_name) + log_handler.setFormatter( + logging.Formatter('%(asctime)s %(levelname)s: %(message)s') + ) + pygdbmi_logger.addHandler(log_handler) + + logging.info('Running command: %s', self.gdb.get_subprocess_cmd()) + for _ in range(10): + try: + # GdbController creates a process with subprocess.Popen(). Is it really running? It is probable that + # an RPI under high load will get non-responsive during creating a lot of processes. + resp = self.gdb.get_gdb_response( + timeout_sec=10 + ) # calls verify_valid_gdb_subprocess() internally + # it will be interesting to look up this response if the next GDB command fails (times out) + logging.info('GDB response: %s', resp) + break # success + except GdbTimeoutError: + logging.warning( + 'GDB internal error: cannot get response from the subprocess' + ) + except NoGdbProcessError: + logging.error('GDB internal error: process is not running') + break # failure - TODO: create another GdbController + except ValueError: + logging.error( + 'GDB internal error: select() returned an unexpected file number' + ) + + # Set up logging for GDB remote protocol + gdb_remotelog_file_name = os.path.join(self.logdir, 'gdb_remote_log.txt') + self.gdb_write('-gdb-set remotelogfile ' + gdb_remotelog_file_name) + + # Load the ELF file + self.gdb_write('-file-exec-and-symbols {}'.format(self.app.elf_file)) + + # Connect GDB to UART + self.serial.proc.close() + logging.info('Connecting to GDB Stub...') + self.gdb_write('-gdb-set serial baud 115200') + responses = self.gdb_write('-target-select remote ' + self.serial.port) + + # Make sure we get the 'stopped' notification + stop_response = self.find_gdb_response('stopped', 'notify', responses) + if not stop_response: + responses = self.gdb_write('-exec-interrupt') + stop_response = self.find_gdb_response('stopped', 'notify', responses) + assert stop_response + frame = stop_response['payload']['frame'] + if 'file' not in frame: + frame['file'] = '?' + if 'line' not in frame: + frame['line'] = '?' + logging.info('Stopped in {func} at {addr} ({file}:{line})'.format(**frame)) + + # Drain remaining responses + self.gdb.get_gdb_response(raise_error_on_timeout=False) + + def gdb_backtrace(self) -> Any: + """ + Returns the list of stack frames for the current thread. + Each frame is a dictionary, refer to pygdbmi docs for the format. + """ + assert self.gdb + + responses = self.gdb_write('-stack-list-frames') + return self.find_gdb_response('done', 'result', responses)['payload']['stack'] + + @staticmethod + def match_backtrace( + gdb_backtrace: List[Any], expected_functions_list: List[Any] + ) -> bool: + """ + Returns True if the function names listed in expected_functions_list match the backtrace + given by gdb_backtrace argument. The latter is in the same format as returned by gdb_backtrace() + function. + """ + return all( + [ + frame['func'] == expected_functions_list[i] + for i, frame in enumerate(gdb_backtrace) + ] + ) + + @staticmethod + def find_gdb_response( + message: str, response_type: str, responses: List[Any] + ) -> Any: + """ + Helper function which extracts one response from an array of GDB responses, filtering + by message and type. Returned message is a dictionary, refer to pygdbmi docs for the format. + """ + + def match_response(response: Dict[str, Any]) -> bool: + return response['message'] == message and response['type'] == response_type # type: ignore + + filtered_responses = [r for r in responses if match_response(r)] + if not filtered_responses: + return None + return filtered_responses[0] + + +@pytest.fixture(scope='module') +def monkeypatch_module(request: FixtureRequest) -> MonkeyPatch: + mp = MonkeyPatch() + request.addfinalizer(mp.undo) + return mp + + +@pytest.fixture(scope='module', autouse=True) +def replace_dut_class(monkeypatch_module: MonkeyPatch) -> None: + monkeypatch_module.setattr('pytest_embedded_idf.dut.IdfDut', PanicTestDut) diff --git a/tools/test_apps/system/panic/panic_tests.py b/tools/test_apps/system/panic/panic_tests.py deleted file mode 100644 index 10bd5fbac5..0000000000 --- a/tools/test_apps/system/panic/panic_tests.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python -import re -from pprint import pformat - -from test_panic_util.test_panic_util import get_dut - - -def get_default_backtrace(test_name): - return [ - test_name, - 'app_main', - 'main_task', - 'vPortTaskWrapper' - ] - - -def test_common(dut, test_name, expected_backtrace=None): - if expected_backtrace is None: - expected_backtrace = get_default_backtrace(dut.test_name) - - if 'gdbstub' in test_name: - dut.expect('Entering gdb stub now.') - dut.start_gdb() - frames = dut.gdb_backtrace() - if not dut.match_backtrace(frames, expected_backtrace): - raise AssertionError('Unexpected backtrace in test {}:\n{}'.format(test_name, pformat(frames))) - return - - if 'uart' in test_name: - dut.expect(dut.COREDUMP_UART_END) - - dut.expect('Rebooting...') - - if 'uart' in test_name: - dut.process_coredump_uart() - # TODO: check backtrace - elif 'flash' in test_name: - dut.process_coredump_flash() - # TODO: check backtrace - elif 'panic' in test_name: - # TODO: check backtrace - pass - - -def task_wdt_inner(env, test_name): - with get_dut(env, test_name, 'test_task_wdt', qemu_wdt_enable=True) as dut: - dut.expect('Task watchdog got triggered. The following tasks did not reset the watchdog in time:') - dut.expect('CPU 0: main') - dut.expect(re.compile(r'abort\(\) was called at PC [0-9xa-f]+ on core 0')) - dut.expect_none('register dump:') - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') - if ('gdbstub' in test_name): - test_common(dut, test_name, expected_backtrace=[ - # Backtrace interrupted when abort is called, IDF-842 - 'panic_abort', 'esp_system_abort' - ]) - else: - test_common(dut, test_name) - - -def int_wdt_inner(env, test_name): - with get_dut(env, test_name, 'test_int_wdt', qemu_wdt_enable=True) as dut: - dut.expect_gme('Interrupt wdt timeout on CPU0') - dut.expect_reg_dump(0) - dut.expect_backtrace() - dut.expect_none('Guru Meditation') - dut.expect_reg_dump(1) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') - test_common(dut, test_name) - - -def int_wdt_cache_disabled_inner(env, test_name): - with get_dut(env, test_name, 'test_int_wdt_cache_disabled', qemu_wdt_enable=True) as dut: - dut.expect_gme('Interrupt wdt timeout on CPU0') - dut.expect_reg_dump(0) - dut.expect('Backtrace:') - dut.expect_none('Guru Meditation') - dut.expect_reg_dump(1) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') - test_common(dut, test_name) - - -def cache_error_inner(env, test_name): - with get_dut(env, test_name, 'test_cache_error') as dut: - dut.expect_gme('Cache disabled but cached memory region accessed') - dut.expect_reg_dump(0) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') - test_common(dut, test_name, - expected_backtrace=['die'] + get_default_backtrace(dut.test_name)) - - -def abort_inner(env, test_name): - with get_dut(env, test_name, 'test_abort') as dut: - dut.expect(re.compile(r'abort\(\) was called at PC [0-9xa-f]+ on core 0')) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation', 'Re-entered core dump') - if ('gdbstub' in test_name): - test_common(dut, test_name, expected_backtrace=[ - # Backtrace interrupted when abort is called, IDF-842 - 'panic_abort', 'esp_system_abort' - ]) - else: - test_common(dut, test_name) - - -def abort_cached_disabled_inner(env, test_name): - with get_dut(env, test_name, 'test_abort_cache_disabled') as dut: - dut.expect(re.compile(r'abort\(\) was called at PC [0-9xa-f]+ on core 0')) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation', 'Re-entered core dump') - test_common(dut, test_name) - - -def assert_inner(env, test_name): - with get_dut(env, test_name, 'test_assert') as dut: - dut.expect(re.compile(r'(assert failed:[\s\w\(\)]*?\s[\.\w\/]*\.(?:c|cpp|h|hpp):\d*.*)')) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation', 'Re-entered core dump') - test_common(dut, test_name) - - -def assert_cached_disabled_inner(env, test_name): - with get_dut(env, test_name, 'test_assert_cache_disabled') as dut: - dut.expect(re.compile(r'(assert failed: [0-9xa-fA-F]+.*)')) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation', 'Re-entered core dump') - test_common(dut, test_name) - - -def storeprohibited_inner(env, test_name): - with get_dut(env, test_name, 'test_storeprohibited') as dut: - dut.expect_gme('StoreProhibited') - dut.expect_reg_dump(0) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') - test_common(dut, test_name) - - -def stack_overflow_inner(env, test_name): - with get_dut(env, test_name, 'test_stack_overflow') as dut: - dut.expect_gme('Unhandled debug exception') - dut.expect('Stack canary watchpoint triggered (main)') - dut.expect_reg_dump(0) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') - test_common(dut, test_name) - - -def illegal_instruction_inner(env, test_name): - with get_dut(env, test_name, 'test_illegal_instruction') as dut: - dut.expect_gme('IllegalInstruction') - dut.expect_reg_dump(0) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') - test_common(dut, test_name) - - -def instr_fetch_prohibited_inner(env, test_name): - with get_dut(env, test_name, 'test_instr_fetch_prohibited') as dut: - dut.expect_gme('InstrFetchProhibited') - dut.expect_reg_dump(0) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') - test_common(dut, test_name, - expected_backtrace=['_init'] + get_default_backtrace(dut.test_name)) - - -def ub_inner(env, test_name): - with get_dut(env, test_name, 'test_ub') as dut: - dut.expect(re.compile(r'Undefined behavior of type out_of_bounds')) - dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation', 'Re-entered core dump') - if ('gdbstub' in test_name): - test_common(dut, test_name, expected_backtrace=[ - # Backtrace interrupted when abort is called, IDF-842 - 'panic_abort', 'esp_system_abort' - ]) - else: - test_common(dut, test_name) diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py new file mode 100644 index 0000000000..a1813abdda --- /dev/null +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -0,0 +1,287 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import re +from pprint import pformat +from typing import List, Optional + +import pytest + +from conftest import PanicTestDut + +CONFIGS = [ + pytest.param('coredump_flash_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), + pytest.param('coredump_flash_elf_sha', marks=[pytest.mark.esp32]), # sha256 only supported on esp32 + pytest.param('coredump_uart_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), + pytest.param('coredump_uart_elf_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), + pytest.param('gdbstub', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), + pytest.param('panic', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), +] + + +def get_default_backtrace(config: str) -> List[str]: + return [config, 'app_main', 'main_task', 'vPortTaskWrapper'] + + +def common_test(dut: PanicTestDut, config: str, expected_backtrace: Optional[List[str]] = None) -> None: + if 'gdbstub' in config: + dut.expect_exact('Entering gdb stub now.') + dut.start_gdb() + frames = dut.gdb_backtrace() + if not dut.match_backtrace(frames, expected_backtrace): + raise AssertionError( + 'Unexpected backtrace in test {}:\n{}'.format(config, pformat(frames)) + ) + dut.revert_log_level() + return + + if 'uart' in config: + dut.process_coredump_uart() + elif 'flash' in config: + dut.process_coredump_flash() + elif 'panic' in config: + pass + + dut.expect('Rebooting...') + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_task_wdt(dut: PanicTestDut, config: str, test_func_name: str) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect_exact( + 'Task watchdog got triggered. The following tasks did not reset the watchdog in time:' + ) + dut.expect_exact('CPU 0: main') + dut.expect(r'abort\(\) was called at PC [0-9xa-f]+ on core 0') + dut.expect_none('register dump:') + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + + if config == 'gdbstub': + common_test( + dut, + config, + expected_backtrace=[ + # Backtrace interrupted when abort is called, IDF-842 + 'panic_abort', + 'esp_system_abort', + ], + ) + else: + common_test(dut, config) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_int_wdt( + dut: PanicTestDut, target: str, config: str, test_func_name: str +) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect_gme('Interrupt wdt timeout on CPU0') + dut.expect_reg_dump(0) + dut.expect_backtrace() + if target == 'esp32s2': + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + + if target != 'esp32s2': # esp32s2 is single-core + dut.expect_reg_dump(1) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + + common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_int_wdt_cache_disabled( + dut: PanicTestDut, target: str, config: str, test_func_name: str +) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect_gme('Interrupt wdt timeout on CPU0') + dut.expect_reg_dump(0) + dut.expect_backtrace() + if target == 'esp32s2': + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + + if target != 'esp32s2': # esp32s2 is single-core + dut.expect_reg_dump(1) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + + common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') +@pytest.mark.generic +def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect_gme('Cache disabled but cached memory region accessed') + dut.expect_reg_dump(0) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + common_test( + dut, config, expected_backtrace=['die'] + get_default_backtrace(test_func_name) + ) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_stack_overflow(dut: PanicTestDut, config: str, test_func_name: str) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect_gme('Unhandled debug exception') + dut.expect_exact('Stack canary watchpoint triggered (main)') + dut.expect_reg_dump(0) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_instr_fetch_prohibited( + dut: PanicTestDut, config: str, test_func_name: str +) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect_gme('InstrFetchProhibited') + dut.expect_reg_dump(0) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + common_test( + dut, + config, + expected_backtrace=['_init'] + get_default_backtrace(test_func_name), + ) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_illegal_instruction( + dut: PanicTestDut, config: str, test_func_name: str +) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect_gme('IllegalInstruction') + dut.expect_reg_dump(0) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_storeprohibited(dut: PanicTestDut, config: str, test_func_name: str) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect_gme('StoreProhibited') + dut.expect_reg_dump(0) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_abort(dut: PanicTestDut, config: str, test_func_name: str) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect(r'abort\(\) was called at PC [0-9xa-f]+ on core 0') + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none(['Guru Meditation', 'Re-entered core dump']) + + if config == 'gdbstub': + common_test( + dut, + config, + expected_backtrace=[ + # Backtrace interrupted when abort is called, IDF-842 + 'panic_abort', + 'esp_system_abort', + ], + ) + else: + common_test(dut, config) + + +@pytest.mark.parametrize('config', CONFIGS, indirect=True) +@pytest.mark.generic +def test_ub(dut: PanicTestDut, config: str, test_func_name: str) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect('Undefined behavior of type out_of_bounds') + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none(['Guru Meditation', 'Re-entered core dump']) + + if config == 'gdbstub': + common_test( + dut, + config, + expected_backtrace=[ + # Backtrace interrupted when abort is called, IDF-842 + 'panic_abort', + 'esp_system_abort', + ], + ) + else: + common_test(dut, config) + + +######################### +# for config panic only # +######################### +@pytest.mark.esp32 +@pytest.mark.esp32s2 +@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') +@pytest.mark.parametrize('config', ['panic'], indirect=True) +@pytest.mark.generic +def test_abort_cache_disabled( + dut: PanicTestDut, config: str, test_func_name: str +) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect(r'abort\(\) was called at PC [0-9xa-f]+ on core 0') + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none(['Guru Meditation', 'Re-entered core dump']) + common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + + +@pytest.mark.esp32 +@pytest.mark.esp32s2 +@pytest.mark.parametrize('config', ['panic'], indirect=True) +@pytest.mark.generic +def test_assert(dut: PanicTestDut, config: str, test_func_name: str) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect( + re.compile( + rb'assert failed:[\s\w()]*?\s[.\w/]*\.(?:c|cpp|h|hpp):\d.*$', re.MULTILINE + ) + ) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none(['Guru Meditation', 'Re-entered core dump']) + common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + + +@pytest.mark.esp32 +@pytest.mark.esp32s2 +@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') +@pytest.mark.parametrize('config', ['panic'], indirect=True) +@pytest.mark.generic +def test_assert_cache_disabled( + dut: PanicTestDut, config: str, test_func_name: str +) -> None: + dut.expect_test_func_name(test_func_name) + dut.expect(re.compile(rb'assert failed: [0-9xa-fA-F]+.*$', re.MULTILINE)) + dut.expect_backtrace() + dut.expect_elf_sha256() + dut.expect_none(['Guru Meditation', 'Re-entered core dump']) + common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) diff --git a/tools/test_apps/system/panic/test_panic_util/__init__.py b/tools/test_apps/system/panic/test_panic_util/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py deleted file mode 100644 index 4be900c520..0000000000 --- a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py +++ /dev/null @@ -1,319 +0,0 @@ -import logging -import os -import re -import subprocess -import sys - -import ttfw_idf -from pygdbmi.gdbcontroller import GdbController, GdbTimeoutError, NoGdbProcessError -from tiny_test_fw import DUT, TinyFW, Utility -from tiny_test_fw.Utility import CaseConfig, SearchCases - -# hard-coded to the path one level above - only intended to be used from the panic test app -TEST_PATH = os.path.relpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'), os.getenv('IDF_PATH')) -TEST_SUITE = 'Panic' - - -def ok(data): - """ Helper function used with dut.expect_any """ - pass - - -def unexpected(data): - """ Helper function used with dut.expect_any """ - raise AssertionError('Unexpected: {}'.format(data)) - - -class PanicTestApp(ttfw_idf.TestApp): - pass - - -class PanicTestMixin(object): - """ Provides custom functionality for the panic test DUT """ - BOOT_CMD_ADDR = 0x9000 - BOOT_CMD_SIZE = 0x1000 - DEFAULT_EXPECT_TIMEOUT = 10 - COREDUMP_UART_START = '================= CORE DUMP START =================' - COREDUMP_UART_END = '================= CORE DUMP END =================' - - def start_test(self, test_name): - """ Starts the app and sends it the test name """ - self.test_name = test_name - # Start the app and verify that it has started up correctly - self.start_capture_raw_data() - self.start_app() - self.expect('Enter test name: ') - Utility.console_log('Setting boot command: ' + test_name) - self.write(test_name) - self.expect('Got test name: ' + test_name) - - def expect_none(self, *patterns, **timeout_args): - """ like dut.expect_all, but with an inverse logic """ - found_data = [] - if 'timeout' not in timeout_args: - timeout_args['timeout'] = 1 - - def found(data): - raise AssertionError('Unexpected: {}'.format(data)) - found_data.append(data) - try: - expect_items = [(pattern, found) for pattern in patterns] - self.expect_any(*expect_items, **timeout_args) - raise AssertionError('Unexpected: {}'.format(found_data)) - except DUT.ExpectTimeout: - return True - - def expect_gme(self, reason): - """ Expect method for Guru Meditation Errors """ - self.expect(r"Guru Meditation Error: Core 0 panic'ed (%s)" % reason) - - def expect_reg_dump(self, core=0): - """ Expect method for the register dump """ - self.expect(re.compile(r'Core\s+%d register dump:' % core)) - - def expect_elf_sha256(self): - """ Expect method for ELF SHA256 line """ - elf_sha256 = self.app.get_elf_sha256() - sdkconfig = self.app.get_sdkconfig() - elf_sha256_len = int(sdkconfig.get('CONFIG_APP_RETRIEVE_LEN_ELF_SHA', '16')) - self.expect('ELF file SHA256: ' + elf_sha256[0:elf_sha256_len]) - - def expect_backtrace(self): - self.expect('Backtrace:') - self.expect_none('CORRUPTED') - - def __enter__(self): - self._raw_data = None - self.gdb = None - return self - - def __exit__(self, type, value, traceback): - log_folder = self.app.get_log_folder(TEST_SUITE) - with open(os.path.join(log_folder, 'log_' + self.test_name + '.txt'), 'w') as log_file: - Utility.console_log('Writing output of {} to {}'.format(self.test_name, log_file.name)) - log_file.write(self.get_raw_data()) - if self.gdb: - self.gdb.exit() - self.close() - - def get_raw_data(self): - if not self._raw_data: - self._raw_data = self.stop_capture_raw_data() - return self._raw_data - - def _call_espcoredump(self, extra_args, coredump_file_name, output_file_name): - # no "with" here, since we need the file to be open for later inspection by the test case - self.coredump_output = open(output_file_name, 'w') - espcoredump_script = os.path.join(os.environ['IDF_PATH'], 'components', 'espcoredump', 'espcoredump.py') - espcoredump_args = [ - sys.executable, - espcoredump_script, - 'info_corefile', - '--core', coredump_file_name, - ] - espcoredump_args += extra_args - espcoredump_args.append(self.app.elf_file) - Utility.console_log('Running ' + ' '.join(espcoredump_args)) - Utility.console_log('espcoredump output is written to ' + self.coredump_output.name) - - subprocess.check_call(espcoredump_args, stdout=self.coredump_output) - self.coredump_output.flush() - self.coredump_output.seek(0) - - def process_coredump_uart(self): - """ Extract the core dump from UART output of the test, run espcoredump on it """ - log_folder = self.app.get_log_folder(TEST_SUITE) - data = self.get_raw_data() - coredump_start = data.find(self.COREDUMP_UART_START) - coredump_end = data.find(self.COREDUMP_UART_END) - coredump_base64 = data[coredump_start + len(self.COREDUMP_UART_START):coredump_end] - with open(os.path.join(log_folder, 'coredump_data_' + self.test_name + '.b64'), 'w') as coredump_file: - Utility.console_log('Writing UART base64 core dump to ' + coredump_file.name) - coredump_file.write(coredump_base64) - - output_file_name = os.path.join(log_folder, 'coredump_uart_result_' + self.test_name + '.txt') - self._call_espcoredump(['--core-format', 'b64'], coredump_file.name, output_file_name) - - def process_coredump_flash(self): - """ Extract the core dump from flash, run espcoredump on it """ - log_folder = self.app.get_log_folder(TEST_SUITE) - coredump_file_name = os.path.join(log_folder, 'coredump_data_' + self.test_name + '.bin') - Utility.console_log('Writing flash binary core dump to ' + coredump_file_name) - self.dump_flash(coredump_file_name, partition='coredump') - - output_file_name = os.path.join(log_folder, 'coredump_flash_result_' + self.test_name + '.txt') - self._call_espcoredump(['--core-format', 'raw'], coredump_file_name, output_file_name) - - def _gdb_write(self, command): - """ - Wrapper to write to gdb with a longer timeout, as test runner - host can be slow sometimes - """ - return self.gdb.write(command, timeout_sec=10) - - def start_gdb(self): - """ - Runs GDB and connects it to the "serial" port of the DUT. - After this, the DUT expect methods can no longer be used to capture output. - """ - self.stop_receive() - self._port_close() - - Utility.console_log('Starting GDB...', 'orange') - self.gdb = GdbController(gdb_path=self.TOOLCHAIN_PREFIX + 'gdb') - Utility.console_log('Running command: {}'.format(self.gdb.get_subprocess_cmd()), 'orange') - - for _ in range(10): - try: - # GdbController creates a process with subprocess.Popen(). Is it really running? It is probable that - # an RPI under high load will get non-responsive during creating a lot of processes. - resp = self.gdb.get_gdb_response(timeout_sec=10) # calls verify_valid_gdb_subprocess() internally - # it will be interesting to look up this response if the next GDB command fails (times out) - Utility.console_log('GDB response: {}'.format(resp), 'orange') - break # success - except GdbTimeoutError: - Utility.console_log('GDB internal error: cannot get response from the subprocess', 'orange') - except NoGdbProcessError: - Utility.console_log('GDB internal error: process is not running', 'red') - break # failure - TODO: create another GdbController - except ValueError: - Utility.console_log('GDB internal error: select() returned an unexpected file number', 'red') - - # pygdbmi logs to console by default, make it log to a file instead - log_folder = self.app.get_log_folder(TEST_SUITE) - pygdbmi_log_file_name = os.path.join(log_folder, 'pygdbmi_log_' + self.test_name + '.txt') - pygdbmi_logger = self.gdb.logger - pygdbmi_logger.setLevel(logging.DEBUG) - while pygdbmi_logger.hasHandlers(): - pygdbmi_logger.removeHandler(pygdbmi_logger.handlers[0]) - log_handler = logging.FileHandler(pygdbmi_log_file_name) - log_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) - pygdbmi_logger.addHandler(log_handler) - - # Set up logging for GDB remote protocol - gdb_remotelog_file_name = os.path.join(log_folder, 'gdb_remote_log_' + self.test_name + '.txt') - self._gdb_write('-gdb-set remotelogfile ' + gdb_remotelog_file_name) - - # Load the ELF file - self._gdb_write('-file-exec-and-symbols {}'.format(self.app.elf_file)) - - # Connect GDB to UART - Utility.console_log('Connecting to GDB Stub...', 'orange') - self._gdb_write('-gdb-set serial baud 115200') - responses = self._gdb_write('-target-select remote ' + self.get_gdb_remote()) - - # Make sure we get the 'stopped' notification - stop_response = self.find_gdb_response('stopped', 'notify', responses) - if not stop_response: - responses = self._gdb_write('-exec-interrupt') - stop_response = self.find_gdb_response('stopped', 'notify', responses) - assert stop_response - frame = stop_response['payload']['frame'] - if 'file' not in frame: - frame['file'] = '?' - if 'line' not in frame: - frame['line'] = '?' - Utility.console_log('Stopped in {func} at {addr} ({file}:{line})'.format(**frame), 'orange') - - # Drain remaining responses - self.gdb.get_gdb_response(raise_error_on_timeout=False) - - def gdb_backtrace(self): - """ - Returns the list of stack frames for the current thread. - Each frame is a dictionary, refer to pygdbmi docs for the format. - """ - assert self.gdb - - responses = self._gdb_write('-stack-list-frames') - return self.find_gdb_response('done', 'result', responses)['payload']['stack'] - - @staticmethod - def match_backtrace(gdb_backtrace, expected_functions_list): - """ - Returns True if the function names listed in expected_functions_list match the backtrace - given by gdb_backtrace argument. The latter is in the same format as returned by gdb_backtrace() - function. - """ - return all([frame['func'] == expected_functions_list[i] for i, frame in enumerate(gdb_backtrace)]) - - @staticmethod - def find_gdb_response(message, response_type, responses): - """ - Helper function which extracts one response from an array of GDB responses, filtering - by message and type. Returned message is a dictionary, refer to pygdbmi docs for the format. - """ - def match_response(response): - return (response['message'] == message and - response['type'] == response_type) - - filtered_responses = [r for r in responses if match_response(r)] - if not filtered_responses: - return None - return filtered_responses[0] - - -class ESP32PanicTestDUT(ttfw_idf.ESP32DUT, PanicTestMixin): - def get_gdb_remote(self): - return self.port - - -class ESP32S2PanicTestDUT(ttfw_idf.ESP32S2DUT, PanicTestMixin): - def get_gdb_remote(self): - return self.port - - -PANIC_TEST_DUT_DICT = { - 'ESP32': ESP32PanicTestDUT, - 'ESP32S2': ESP32S2PanicTestDUT -} - - -def panic_test(**kwargs): - """ Decorator for the panic tests, sets correct App and DUT classes """ - if 'target' not in kwargs: - kwargs['target'] = ['ESP32'] - - if 'additional_duts' not in kwargs: - kwargs['additional_duts'] = PANIC_TEST_DUT_DICT - return ttfw_idf.idf_custom_test(app=PanicTestApp, env_tag='Example_GENERIC', **kwargs) - - -def get_dut(env, app_config_name, test_name, qemu_wdt_enable=False): - dut = env.get_dut('panic', TEST_PATH, app_config_name=app_config_name, allow_dut_exception=True) - dut.qemu_wdt_enable = qemu_wdt_enable - """ Wrapper for getting the DUT and starting the test """ - dut.start_test(test_name) - return dut - - -def run_all(filename, case_filter=[]): - """ Helper function to run test cases defined in a file; to be called from __main__. - case_filter is an optional list of case names to run. - If not specified, all test cases are run. - """ - TinyFW.set_default_config(env_config_file=None, test_suite_name=TEST_SUITE) - test_methods = SearchCases.Search.search_test_cases(filename) - test_methods = filter(lambda m: not m.case_info['ignore'], test_methods) - test_cases = CaseConfig.Parser.apply_config(test_methods, None) - tests_failed = [] - for case in test_cases: - test_name = case.test_method.__name__ - if case_filter: - if case_filter[0].endswith('*'): - if not test_name.startswith(case_filter[0][:-1]): - continue - else: - if test_name not in case_filter: - continue - result = case.run() - if not result: - tests_failed.append(case) - - if tests_failed: - print('The following tests have failed:') - for case in tests_failed: - print(' - ' + case.test_method.__name__) - raise SystemExit(1) - - print('Tests pass') From 31fafaea932992e52cf10cb5cce06b67ec6d61f8 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 18 Feb 2022 15:38:54 +0800 Subject: [PATCH 4/5] ci(pytest): reorder test sequence to run tests faster --- conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conftest.py b/conftest.py index 12a88a88a8..1bf5b55643 100644 --- a/conftest.py +++ b/conftest.py @@ -152,11 +152,16 @@ def pytest_collection_modifyitems(config: Config, items: List[Function]) -> None if not target: return + # sort by file path and callspec.config + # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting + # https://github.com/pytest-dev/pytest/discussions/9689 def _get_param_config(_item: Function) -> str: if hasattr(_item, 'callspec'): return _item.callspec.params.get('config', DEFAULT_SDKCONFIG) # type: ignore return DEFAULT_SDKCONFIG + items.sort(key=lambda x: (os.path.dirname(x.path), _get_param_config(x))) + # add markers for special markers for item in items: if 'supported_targets' in item_marker_names(item): From 4cbaf6fbb154c3e9a7a2c5c28c84ca3f15083b7f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 11 Feb 2022 15:47:32 +0800 Subject: [PATCH 5/5] ci: enable pytest panic test --- .gitlab-ci.yml | 2 +- .gitlab/ci/build.yml | 14 ++++++++++++++ .gitlab/ci/target-test.yml | 36 ++++++++++++++++++++++++++++++++--- tools/ci/build_pytest_apps.py | 30 +++++++++++++++++++++-------- 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5364dc6077..bb627dba95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,7 +73,7 @@ variables: TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git" CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git" CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1" - PYTEST_EMBEDDED_TAG: "v0.5.1" + PYTEST_EMBEDDED_TAG: "v0.6.0rc0" # cache python dependencies PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index e8337e1bcd..952ad484d1 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -93,6 +93,20 @@ build_pytest_components_esp32c3: script: - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv +build_pytest_test_apps_esp32: + extends: + - .build_pytest_template + - .rules:build:custom_test-esp32 + script: + - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32 --size-info $SIZE_INFO_LOCATION -vv + +build_pytest_test_apps_esp32s2: + extends: + - .build_pytest_template + - .rules:build:custom_test-esp32s2 + script: + - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv + build_non_test_components_apps: extends: - .build_template diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 277df150b5..db1622a7db 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -167,6 +167,39 @@ component_ut_pytest_esp32c3_generic: - ESP32C3 - COMPONENT_UT_GENERIC +.pytest_test_apps_dir_template: + extends: .pytest_template + variables: + TEST_DIR: tools/test_apps + +test_app_test_pytest_esp32_generic: + extends: + - .pytest_test_apps_dir_template + - .rules:test:custom_test-esp32 + needs: + - build_pytest_test_apps_esp32 + variables: + TARGET: esp32 + ENV_MARKER: generic + SETUP_TOOLS: "1" # need gdb + tags: + - ESP32 + - Example_GENERIC + +test_app_test_pytest_esp32s2_generic: + extends: + - .pytest_test_apps_dir_template + - .rules:test:custom_test-esp32s2 + needs: + - build_pytest_test_apps_esp32s2 + variables: + TARGET: esp32s2 + ENV_MARKER: generic + SETUP_TOOLS: "1" # need gdb + tags: + - ESP32S2 + - Example_GENERIC + # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2") # we need to convert to pattern "job_name_index.yml" .define_config_file_name: &define_config_file_name | @@ -538,12 +571,9 @@ test_app_test_005: test_app_test_esp32_generic: extends: .test_app_esp32_template - parallel: 5 tags: - ESP32 - Example_GENERIC - variables: - SETUP_TOOLS: "1" test_app_test_flash_psram_f4r4: extends: .test_app_esp32s3_template diff --git a/tools/ci/build_pytest_apps.py b/tools/ci/build_pytest_apps.py index e5683d17fa..8de9964339 100644 --- a/tools/ci/build_pytest_apps.py +++ b/tools/ci/build_pytest_apps.py @@ -6,6 +6,7 @@ This file is used to generate binary files for the given path. """ import argparse +import copy import logging import os import sys @@ -58,17 +59,25 @@ def main(args: argparse.Namespace) -> None: build_system='cmake', config_rules=config_rules, ) - logging.info(f'Found {len(build_items)} builds') - build_items.sort(key=lambda x: x.build_path) # type: ignore + modified_build_items = [] # auto clean up the binaries if no flag --preserve-all - if args.preserve_all is False: - for item in build_items: - if item.config_name not in app_configs[item.app_dir]: - item.preserve = False + for item in build_items: + is_test_related = item.config_name in app_configs[item.app_dir] + if args.test_only and not is_test_related: + logging.info(f'Skipping non-test app: {item}') + continue + + copied_item = copy.deepcopy(item) + if not args.preserve_all and not is_test_related: + copied_item.preserve = False + modified_build_items.append(copied_item) + + logging.info(f'Found {len(modified_build_items)} builds') + modified_build_items.sort(key=lambda x: x.build_path) # type: ignore build_apps( - build_items=build_items, + build_items=modified_build_items, parallel_count=args.parallel_count, parallel_index=args.parallel_index, dry_run=False, @@ -128,7 +137,12 @@ if __name__ == '__main__': parser.add_argument( '--preserve-all', action='store_true', - help='add this flag to preserve the binaries for all apps', + help='Preserve the binaries for all apps when specified.', + ) + parser.add_argument( + '--test-only', + action='store_true', + help='Build only test related app when specified.', ) arguments = parser.parse_args() setup_logging(arguments)