diff --git a/tools/test_apps/.build-test-rules.yml b/tools/test_apps/.build-test-rules.yml index be1ddf8ed1..8aa351709a 100644 --- a/tools/test_apps/.build-test-rules.yml +++ b/tools/test_apps/.build-test-rules.yml @@ -165,9 +165,9 @@ tools/test_apps/system/panic: enable: - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "esp32h4" disable_test: - - if: IDF_TARGET not in ["esp32", "esp32s2"] + - if: IDF_TARGET not in ["esp32", "esp32s2", "esp32c3", "esp32s3", "esp32c2"] temporary: true - reason: lack of runners + reason: test app not ported to this target yet tools/test_apps/system/startup: enable: diff --git a/tools/test_apps/system/panic/README.md b/tools/test_apps/system/panic/README.md index 17ce10199e..a5a7b613e5 100644 --- a/tools/test_apps/system/panic/README.md +++ b/tools/test_apps/system/panic/README.md @@ -1,27 +1,64 @@ | Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H4 | ESP32-S2 | ESP32-S3 | | ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | +# Introduction + +The panic test app checks the behavior of ESP-IDF Panic Handler. + +This test app is relatively complex because it has to check many possible combinations of: +- Failure scenario: abort, assertion, interrupt watchdog, illegal instruction, ... +- Chip target: esp32, esp32c3, ... +- Configuration: default, GDB Stub, Core Dump to UART, ... + +Failure scenarios are implemented in [test_panic_main.c](main/test_panic_main.c). The test application receives the name of the scenario from console (e.g. `test_illegal_instruction` ). The failure scenario is executed and the app panics. Once the panic output is printed, the pytest-based test case parses the output and verifies that the behavior of the panic handler was correct. + +In [pytest_panic.py](pytest_panic.py), there typically is one test function for each failure scenario. Each test function is then parametrized by `config` parameter. This creates "copies" of the test case for each of the configurations (default, GDB Stub, etc.) Tests are also parametrized with target-specific markers. Most tests can run on every target, but there are a few exceptions, such as failure scenarios specific to the dual-core chips. + +The test cases use a customized DUT class `PanicTestDut`, defined in [panic_dut.py](test_panic_util/panic_dut.py). This class is derived from [`IdfDut`](https://docs.espressif.com/projects/pytest-embedded/en/latest/references/pytest_embedded_idf/#pytest_embedded_idf.dut.IdfDut). It defines several helper functions to make the test cases easier to read. + # Building Several configurations are provided as `sdkconfig.ci.XXX` and serve as a template. -## Example with configuration "panic" for target ESP32 -``` -idf.py set-target esp32 +For example, to build the test app with configuration `panic` for ESP32-C3, run: +```bash +idf.py set-target esp32c3 cat sdkconfig.defaults sdkconfig.ci.panic > sdkconfig idf.py build ``` -# Running -All the setup needs to be done as described in the [test apps README](../../README.md), except that the test cases need to be specified when running the app: +# Building multiple configurations side by side -``` -python app_test.py test_panic_illegal_instruction +If you need to work with multiple configurations at the same time it can be useful to keep each build in a separate directory. For example, to build the `panic` configuration for ESP32-C3 in a separate directory, run: +```bash +idf.py -DIDF_TARGET=esp32c3 -DSDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.ci.panic" -DSDKCONFIG=build_esp32c3_panic/sdkconfig -B build_esp32c3_panic build ``` -Multiple test cases are passed as additional arguments: +This way, all the build products and the sdkconfig file are kept in the directory `build_esp32c3_gdbstub`. pytest-embedded will search for binaries in this directory if you run tests as shown in the section below. +This approach allows switching between different build configurations and targets without deleting the build directories. + +# Running the app manually + +```bash +idf.py flash monitor ``` -python app_test.py test_panic_illegal_instruction test_panic_int_wdt test_panic_storeprohibited +(don't forget the -B argument if you have built the app in a directory other than `build`) + +Once the app is running, input the name of the test (e.g. `test_abort`) and press Enter. + +# Running tests + +Suppose you have built the app for a specific target and with a certain `sdkconfig.ci.CONFIG` config. You need to run the tests just for this config and the target: +```bash +pytest --target TARGET -k '[CONFIG]' ``` -*Note that you need to pick the correct test cases at run time according to the configuration you built before. The above examples are for configuration "panic"* +For example, if you have built the `panic` config for ESP32-C3, run: +```bash +pytest --target esp32c3 -k '[panic]' +``` + +Or, to run a single test for the given config, e.g. `test_abort`: +```bash +pytest --target esp32c3 -k 'test_abort[panic]' +``` diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py index ff7a64e76c..bcc8343658 100644 --- a/tools/test_apps/system/panic/pytest_panic.py +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -1,39 +1,47 @@ -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import re -from pprint import pformat from typing import List, Optional import pexpect import pytest from test_panic_util import PanicTestDut +# Markers for all the targets this test currently runs on +TARGETS_TESTED = [pytest.mark.esp32, pytest.mark.esp32s2, pytest.mark.esp32c3, pytest.mark.esp32s3, pytest.mark.esp32c2] + +# Most tests run on all targets and with all configs. +# This list is passed to @pytest.mark.parametrize for each of the test cases. +# It creates an outer product of the sets: [configs] x [targets], +# with some exceptions. CONFIGS = [ - pytest.param('coredump_flash_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('coredump_flash_elf_sha', marks=[pytest.mark.esp32]), # sha256 only supported on esp32 - pytest.param('coredump_uart_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('coredump_uart_elf_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('gdbstub', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('panic', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), + pytest.param('coredump_flash_bin_crc', marks=TARGETS_TESTED), + pytest.param('coredump_flash_elf_sha', marks=[pytest.mark.esp32]), # sha256 only supported on esp32, IDF-1820 + pytest.param('coredump_uart_bin_crc', marks=TARGETS_TESTED), + pytest.param('coredump_uart_elf_crc', marks=TARGETS_TESTED), + pytest.param('gdbstub', marks=TARGETS_TESTED), + pytest.param('panic', marks=TARGETS_TESTED), ] -# An ESP32-only config, used for tests requiring two cores -CONFIGS_ESP32 = [ - pytest.param('coredump_flash_bin_crc', marks=[pytest.mark.esp32]), - pytest.param('coredump_flash_elf_sha', marks=[pytest.mark.esp32]), - pytest.param('coredump_uart_bin_crc', marks=[pytest.mark.esp32]), - pytest.param('coredump_uart_elf_crc', marks=[pytest.mark.esp32]), - pytest.param('gdbstub', marks=[pytest.mark.esp32]), - pytest.param('panic', marks=[pytest.mark.esp32]), +# Some tests only run on dual-core targets, they use the config below. +TARGETS_DUAL_CORE = [pytest.mark.esp32, pytest.mark.esp32s3] +CONFIGS_DUAL_CORE = [ + pytest.param('coredump_flash_bin_crc', marks=TARGETS_DUAL_CORE), + pytest.param('coredump_flash_elf_sha', marks=[pytest.mark.esp32]), # sha256 only supported on esp32, IDF-1820 + pytest.param('coredump_uart_bin_crc', marks=TARGETS_DUAL_CORE), + pytest.param('coredump_uart_elf_crc', marks=TARGETS_DUAL_CORE), + pytest.param('gdbstub', marks=TARGETS_DUAL_CORE), + pytest.param('panic', marks=TARGETS_DUAL_CORE), ] -# IDF-5692: Uncomment the marks related to ESP32-S3 and quad_psram once ESP32-S3 runners are available -CONFIG_EXTRAM_STACK = [ - pytest.param('coredump_extram_stack', - marks=[pytest.mark.esp32, pytest.mark.esp32s2, pytest.mark.psram, - # pytest.mark.esp32s3, pytest.mark.quad_psram - ]) +# Some tests run on all targets but need to behave differently on the dual-core ones. +# This list is used to check if the target is a dual-core one. +TARGETS_DUAL_CORE_NAMES = [x.mark.name for x in TARGETS_DUAL_CORE] + +# The tests which panic on external stack require PSRAM capable runners +CONFIGS_EXTRAM_STACK = [ + pytest.param('coredump_extram_stack', marks=[pytest.mark.esp32, pytest.mark.esp32s2, pytest.mark.psram, pytest.mark.esp32s3, pytest.mark.quad_psram]) ] @@ -46,17 +54,10 @@ def common_test(dut: PanicTestDut, config: str, expected_backtrace: Optional[Lis dut.expect_exact('Entering gdb stub now.') dut.start_gdb() frames = dut.gdb_backtrace() - # Make sure frames and the expected_backtrace have the same size, else, an exception will occur if expected_backtrace is not None: - size = min(len(frames), len(expected_backtrace)) - frames = frames[0:size] - expected_backtrace = expected_backtrace[0:size] - if not dut.match_backtrace(frames, expected_backtrace): - raise AssertionError( - 'Unexpected backtrace in test {}:\n{}'.format(config, pformat(frames)) - ) - dut.revert_log_level() - return + dut.verify_gdb_backtrace(frames, expected_backtrace) + dut.revert_log_level() + return # don't expect "Rebooting" output below if 'uart' in config: dut.process_coredump_uart() @@ -71,86 +72,91 @@ def common_test(dut: PanicTestDut, config: str, expected_backtrace: Optional[Lis @pytest.mark.parametrize('config', CONFIGS, indirect=True) @pytest.mark.generic def test_task_wdt_cpu0(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) + dut.run_test_func(test_func_name) dut.expect_exact( 'Task watchdog got triggered. The following tasks/users did not reset the watchdog in time:' ) dut.expect_exact('CPU 0: main') - dut.expect_none('register dump:') - dut.expect_exact('Print CPU 0 (current core) backtrace') - dut.expect_backtrace() + if dut.is_xtensa: + # on Xtensa, dumping registers on abort is not necessary, we only need to dump the backtrace + dut.expect_none('register dump:') + dut.expect_exact('Print CPU 0 (current core) backtrace') + dut.expect_backtrace() + else: + # on RISC-V, need to dump both registers and stack memory to reconstruct the backtrace + dut.expect_reg_dump(core=0) + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none('Guru Meditation') - if config == 'gdbstub': - common_test( - dut, - config, - expected_backtrace=[ - 'test_task_wdt_cpu0', - 'app_main' - ], - ) - else: - common_test(dut, config) + common_test( + dut, + config, + expected_backtrace=get_default_backtrace(test_func_name), + ) -@pytest.mark.parametrize('config', CONFIGS_ESP32, indirect=True) +@pytest.mark.parametrize('config', CONFIGS_DUAL_CORE, indirect=True) @pytest.mark.generic def test_task_wdt_cpu1(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) + dut.run_test_func(test_func_name) dut.expect_exact( 'Task watchdog got triggered. The following tasks/users did not reset the watchdog in time:' ) dut.expect_exact('CPU 1: Infinite loop') - dut.expect_none('register dump:') - dut.expect_exact('Print CPU 1 backtrace') - dut.expect_backtrace() + if dut.is_xtensa: + # see comment in test_task_wdt_cpu0 + dut.expect_none('register dump:') + dut.expect_exact('Print CPU 1 backtrace') + dut.expect_backtrace() + # On Xtensa, we get incorrect backtrace from GDB in this test + expected_backtrace = ['infinite_loop', 'vPortTaskWrapper'] + else: + assert False, 'No dual-core RISC-V chips yet, check this test case later' + dut.expect_elf_sha256() dut.expect_none('Guru Meditation') - if config == 'gdbstub': - common_test( - dut, - config, - expected_backtrace=[ - 'infinite_loop' - ], - ) - else: - common_test(dut, config) + common_test( + dut, + config, + expected_backtrace=expected_backtrace, + ) -@pytest.mark.parametrize('config', CONFIGS_ESP32, indirect=True) +@pytest.mark.parametrize('config', CONFIGS_DUAL_CORE, indirect=True) @pytest.mark.generic def test_task_wdt_both_cpus(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) + if dut.target == 'esp32s3': + pytest.xfail(reason='Only prints "Print CPU 1 backtrace", IDF-6560') + dut.run_test_func(test_func_name) dut.expect_exact( 'Task watchdog got triggered. The following tasks/users did not reset the watchdog in time:' ) dut.expect_exact('CPU 0: Infinite loop') dut.expect_exact('CPU 1: Infinite loop') - dut.expect_none('register dump:') - dut.expect_exact('Print CPU 0 (current core) backtrace') - dut.expect_backtrace() - dut.expect_exact('Print CPU 1 backtrace') - dut.expect_backtrace() + if dut.is_xtensa: + # see comment in test_task_wdt_cpu0 + dut.expect_none('register dump:') + dut.expect_exact('Print CPU 0 (current core) backtrace') + dut.expect_backtrace() + dut.expect_exact('Print CPU 1 backtrace') + dut.expect_backtrace() + # On Xtensa, we get incorrect backtrace from GDB in this test + expected_backtrace = ['infinite_loop', 'vPortTaskWrapper'] + else: + assert False, 'No dual-core RISC-V chips yet, check this test case later' dut.expect_elf_sha256() dut.expect_none('Guru Meditation') - if config == 'gdbstub': - common_test( - dut, - config, - expected_backtrace=[ - 'infinite_loop' - ], - ) - else: - common_test(dut, config) + common_test( + dut, + config, + expected_backtrace=expected_backtrace, + ) -@pytest.mark.parametrize('config', CONFIG_EXTRAM_STACK, indirect=True) +@pytest.mark.parametrize('config', CONFIGS_EXTRAM_STACK, indirect=True) def test_panic_extram_stack(dut: PanicTestDut, config: str, test_func_name: str) -> None: dut.expect_test_func_name(test_func_name) dut.expect_none('Allocated stack is not in external RAM') @@ -172,19 +178,21 @@ def test_panic_extram_stack(dut: PanicTestDut, config: str, test_func_name: str) def test_int_wdt( dut: PanicTestDut, target: str, config: str, test_func_name: str ) -> None: - dut.expect_test_func_name(test_func_name) + dut.run_test_func(test_func_name) dut.expect_gme('Interrupt wdt timeout on CPU0') dut.expect_reg_dump(0) - dut.expect_backtrace() - if target == 'esp32s2': - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() - if target != 'esp32s2': # esp32s2 is single-core + if target in TARGETS_DUAL_CORE_NAMES: + assert dut.is_xtensa, 'No dual-core RISC-V chips yet, check the test case' dut.expect_reg_dump(1) dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') + + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) @@ -194,46 +202,68 @@ def test_int_wdt( def test_int_wdt_cache_disabled( dut: PanicTestDut, target: str, config: str, test_func_name: str ) -> None: - dut.expect_test_func_name(test_func_name) + dut.run_test_func(test_func_name) dut.expect_gme('Interrupt wdt timeout on CPU0') dut.expect_reg_dump(0) - dut.expect_backtrace() - if target == 'esp32s2': - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() - if target != 'esp32s2': # esp32s2 is single-core + if target in TARGETS_DUAL_CORE_NAMES: + assert dut.is_xtensa, 'No dual-core RISC-V chips yet, check the test case' dut.expect_reg_dump(1) dut.expect_backtrace() - dut.expect_elf_sha256() - dut.expect_none('Guru Meditation') + + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) @pytest.mark.parametrize('config', CONFIGS, indirect=True) -@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') @pytest.mark.generic def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) - dut.expect_gme('Cache disabled but cached memory region accessed') + dut.run_test_func(test_func_name) + if dut.target in ['esp32c3', 'esp32c2']: + # Cache error interrupt is not raised, IDF-6398 + dut.expect_gme('Illegal instruction') + elif dut.target in ['esp32s2']: + # Cache error interrupt is not enabled, IDF-1558 + dut.expect_gme('IllegalInstruction') + else: + dut.expect_gme('Cache disabled but cached memory region accessed') dut.expect_reg_dump(0) - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none('Guru Meditation') + expected_backtrace = ['die'] + get_default_backtrace(test_func_name) + if dut.target in ['esp32s2', 'esp32s3']: + # 'test_cache_error' missing from GDB backtrace on ESP32-S2 and ESP-S3, IDF-6561 + expected_backtrace = ['die', 'app_main', 'main_task', 'vPortTaskWrapper'] common_test( - dut, config, expected_backtrace=['die'] + get_default_backtrace(test_func_name) + dut, config, expected_backtrace=expected_backtrace ) @pytest.mark.parametrize('config', CONFIGS, indirect=True) @pytest.mark.generic def test_stack_overflow(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) - dut.expect_gme('Unhandled debug exception') - dut.expect_exact('Stack canary watchpoint triggered (main)') + dut.run_test_func(test_func_name) + if dut.is_xtensa: + dut.expect_gme('Unhandled debug exception') + dut.expect_exact('Stack canary watchpoint triggered (main)') + else: + # Stack watchpoint handling missing on RISC-V, IDF-6397 + dut.expect_gme('Breakpoint') dut.expect_reg_dump(0) - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none('Guru Meditation') common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) @@ -244,16 +274,26 @@ def test_stack_overflow(dut: PanicTestDut, config: str, test_func_name: str) -> def test_instr_fetch_prohibited( dut: PanicTestDut, config: str, test_func_name: str ) -> None: - dut.expect_test_func_name(test_func_name) - dut.expect_gme('InstrFetchProhibited') - dut.expect_reg_dump(0) - dut.expect_backtrace() + dut.run_test_func(test_func_name) + if dut.is_xtensa: + dut.expect_gme('InstrFetchProhibited') + dut.expect_reg_dump(0) + dut.expect_backtrace() + expected_backtrace = ['_init'] + get_default_backtrace(test_func_name) + else: + dut.expect_gme('Instruction access fault') + dut.expect_reg_dump(0) + dut.expect_stack_dump() + # On RISC-V, GDB is not able to determine the correct backtrace after + # a jump to an invalid address. + expected_backtrace = ['??'] + dut.expect_elf_sha256() dut.expect_none('Guru Meditation') common_test( dut, config, - expected_backtrace=['_init'] + get_default_backtrace(test_func_name), + expected_backtrace=expected_backtrace, ) @@ -262,10 +302,16 @@ def test_instr_fetch_prohibited( def test_illegal_instruction( dut: PanicTestDut, config: str, test_func_name: str ) -> None: - dut.expect_test_func_name(test_func_name) - dut.expect_gme('IllegalInstruction') + dut.run_test_func(test_func_name) + if dut.is_xtensa: + dut.expect_gme('IllegalInstruction') + else: + dut.expect_gme('Illegal instruction') dut.expect_reg_dump(0) - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none('Guru Meditation') common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) @@ -274,10 +320,16 @@ def test_illegal_instruction( @pytest.mark.parametrize('config', CONFIGS, indirect=True) @pytest.mark.generic def test_storeprohibited(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) - dut.expect_gme('StoreProhibited') + dut.run_test_func(test_func_name) + if dut.is_xtensa: + dut.expect_gme('StoreProhibited') + else: + dut.expect_gme('Store access fault') dut.expect_reg_dump(0) - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none('Guru Meditation') common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) @@ -286,107 +338,130 @@ def test_storeprohibited(dut: PanicTestDut, config: str, test_func_name: str) -> @pytest.mark.parametrize('config', CONFIGS, indirect=True) @pytest.mark.generic def test_abort(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) + dut.run_test_func(test_func_name) dut.expect(r'abort\(\) was called at PC [0-9xa-f]+ on core 0') - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none(['Guru Meditation', 'Re-entered core dump']) - if config == 'gdbstub': - common_test( - dut, - config, - expected_backtrace=[ - 'panic_abort', - 'esp_system_abort', - 'abort' - ] + get_default_backtrace(test_func_name), - ) - else: - common_test(dut, config) + common_test( + dut, + config, + expected_backtrace=[ + 'panic_abort', + 'esp_system_abort', + 'abort' + ] + get_default_backtrace(test_func_name), + ) @pytest.mark.parametrize('config', CONFIGS, indirect=True) @pytest.mark.generic def test_ub(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) + dut.run_test_func(test_func_name) dut.expect('Undefined behavior of type out_of_bounds') - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none(['Guru Meditation', 'Re-entered core dump']) - if config == 'gdbstub': - common_test( - dut, - config, - expected_backtrace=[ - 'panic_abort', - 'esp_system_abort', - '__ubsan_default_handler', - '__ubsan_handle_out_of_bounds' - ] + get_default_backtrace(test_func_name), - ) - else: - common_test(dut, config) + common_test( + dut, + config, + expected_backtrace=[ + 'panic_abort', + 'esp_system_abort', + '__ubsan_default_handler', + '__ubsan_handle_out_of_bounds' + ] + get_default_backtrace(test_func_name), + ) -######################### -# for config panic only # -######################### -@pytest.mark.esp32 -@pytest.mark.esp32s2 -@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') -@pytest.mark.parametrize('config', ['panic'], indirect=True) +@pytest.mark.parametrize('config', CONFIGS, indirect=True) @pytest.mark.generic def test_abort_cache_disabled( dut: PanicTestDut, config: str, test_func_name: str ) -> None: - dut.expect_test_func_name(test_func_name) + if dut.target == 'esp32s2': + pytest.xfail(reason='Crashes in itoa which is not in ROM, IDF-3572') + dut.run_test_func(test_func_name) dut.expect(r'abort\(\) was called at PC [0-9xa-f]+ on core 0') - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none(['Guru Meditation', 'Re-entered core dump']) - common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + common_test( + dut, + config, + expected_backtrace=[ + 'panic_abort', + 'esp_system_abort', + 'abort' + ] + get_default_backtrace(test_func_name), + ) -@pytest.mark.esp32 -@pytest.mark.esp32s2 -@pytest.mark.parametrize('config', ['panic'], indirect=True) +@pytest.mark.parametrize('config', CONFIGS, indirect=True) @pytest.mark.generic def test_assert(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.expect_test_func_name(test_func_name) + dut.run_test_func(test_func_name) dut.expect( re.compile( rb'assert failed:[\s\w()]*?\s[.\w/]*\.(?:c|cpp|h|hpp):\d.*$', re.MULTILINE ) ) - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none(['Guru Meditation', 'Re-entered core dump']) - common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + common_test( + dut, + config, + expected_backtrace=[ + 'panic_abort', + 'esp_system_abort', + '__assert_func' + ] + get_default_backtrace(test_func_name)) -@pytest.mark.esp32 -@pytest.mark.esp32s2 -@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') -@pytest.mark.parametrize('config', ['panic'], indirect=True) +@pytest.mark.parametrize('config', CONFIGS, indirect=True) @pytest.mark.generic def test_assert_cache_disabled( dut: PanicTestDut, config: str, test_func_name: str ) -> None: - dut.expect_test_func_name(test_func_name) + if dut.target == 'esp32s2': + pytest.xfail(reason='Crashes in itoa which is not in ROM, IDF-3572') + dut.run_test_func(test_func_name) dut.expect(re.compile(rb'assert failed: [0-9xa-fA-F]+.*$', re.MULTILINE)) - dut.expect_backtrace() + if dut.is_xtensa: + dut.expect_backtrace() + else: + dut.expect_stack_dump() dut.expect_elf_sha256() dut.expect_none(['Guru Meditation', 'Re-entered core dump']) - common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name)) + common_test( + dut, + config, + expected_backtrace=[ + 'panic_abort', + 'esp_system_abort', + '__assert_func' + ] + get_default_backtrace(test_func_name)) @pytest.mark.esp32 @pytest.mark.parametrize('config', ['panic_delay'], indirect=True) -@pytest.mark.generic def test_panic_delay(dut: PanicTestDut) -> None: - dut.expect_test_func_name('test_storeprohibited') + dut.run_test_func('test_storeprohibited') try: dut.expect_exact('Rebooting...', timeout=4) except pexpect.TIMEOUT: diff --git a/tools/test_apps/system/panic/test_panic_util/panic_dut.py b/tools/test_apps/system/panic/test_panic_util/panic_dut.py index e956e0ec5f..da63a5ec91 100644 --- a/tools/test_apps/system/panic/test_panic_util/panic_dut.py +++ b/tools/test_apps/system/panic/test_panic_util/panic_dut.py @@ -1,10 +1,10 @@ -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 import logging import os import subprocess import sys -from typing import Any, Dict, List, TextIO +from typing import Any, Dict, List, Optional, TextIO import pexpect from panic_utils import NoGdbProcessError, attach_logger, quote_string, sha256, verify_valid_gdb_subprocess @@ -24,28 +24,33 @@ class PanicTestDut(IdfDut): app: IdfApp serial: IdfSerial - def __init__(self, *args, **kwargs) -> None: # type: ignore + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.gdb: GdbController = None # type: ignore + self.gdbmi: Optional[GdbController] = None # record this since pygdbmi is using logging.debug to generate some single character mess self.log_level = logging.getLogger().level # pygdbmi is using logging.debug to generate some single character mess if self.log_level <= logging.DEBUG: logging.getLogger().setLevel(logging.INFO) - self.coredump_output: TextIO = None # type: ignore + self.coredump_output: Optional[TextIO] = None def close(self) -> None: - if self.gdb: - self.gdb.exit() + if self.gdbmi: + logging.info('Waiting for GDB to exit') + self.gdbmi.exit() super().close() def revert_log_level(self) -> None: logging.getLogger().setLevel(self.log_level) - def expect_test_func_name(self, test_func_name: str) -> None: + @property + def is_xtensa(self) -> bool: + return self.target in self.XTENSA_TARGETS + + def run_test_func(self, test_func_name: str) -> None: self.expect_exact('Enter test name:') self.write(test_func_name) self.expect_exact('Got test name: ' + test_func_name) @@ -62,8 +67,13 @@ class PanicTestDut(IdfDut): pass def expect_backtrace(self) -> None: - self.expect_exact('Backtrace:') - self.expect_none('CORRUPTED') + assert self.is_xtensa, 'Backtrace can be printed only on Xtensa' + match = self.expect(r'Backtrace:( 0x[0-9a-fA-F]{8}:0x[0-9a-fA-F]{8})+(?P \|<-CORRUPTED)?') + assert not match.group('corrupted') + + def expect_stack_dump(self) -> None: + assert not self.is_xtensa, 'Stack memory dump is only printed on RISC-V' + self.expect_exact('Stack memory:') def expect_gme(self, reason: str) -> None: """Expect method for Guru Meditation Errors""" @@ -137,25 +147,29 @@ class PanicTestDut(IdfDut): Wrapper to write to gdb with a longer timeout, as test runner host can be slow sometimes """ - return self.gdb.write(command, timeout_sec=10) + assert self.gdbmi, 'This function should be called only after start_gdb' + return self.gdbmi.write(command, timeout_sec=10) def start_gdb(self) -> None: """ Runs GDB and connects it to the "serial" port of the DUT. After this, the DUT expect methods can no longer be used to capture output. """ - gdb_path = self.toolchain_prefix + 'gdb' + if self.is_xtensa: + gdb_path = f'xtensa-{self.target}-elf-gdb' + else: + gdb_path = 'riscv32-esp-elf-gdb' try: from pygdbmi.constants import GdbTimeoutError default_gdb_args = ['--nx', '--quiet', '--interpreter=mi2'] gdb_command = [gdb_path] + default_gdb_args - self.gdb = GdbController(command=gdb_command) + self.gdbmi = GdbController(command=gdb_command) pygdbmi_logger = attach_logger() except ImportError: # fallback for pygdbmi<0.10.0.0. from pygdbmi.gdbcontroller import GdbTimeoutError - self.gdb = GdbController(gdb_path=gdb_path) - pygdbmi_logger = self.gdb.logger + self.gdbmi = GdbController(gdb_path=gdb_path) + pygdbmi_logger = self.gdbmi.logger # pygdbmi logs to console by default, make it log to a file instead pygdbmi_log_file_name = os.path.join(self.logdir, 'pygdbmi_log.txt') @@ -166,22 +180,23 @@ class PanicTestDut(IdfDut): log_handler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s: %(message)s') ) + logging.info(f'Saving pygdbmi logs to {pygdbmi_log_file_name}') pygdbmi_logger.addHandler(log_handler) try: - gdb_command = self.gdb.command + gdb_command = self.gdbmi.command except AttributeError: # fallback for pygdbmi < 0.10 - gdb_command = self.gdb.cmd + gdb_command = self.gdbmi.cmd logging.info(f'Running command: "{" ".join(quote_string(c) for c in gdb_command)}"') for _ in range(10): try: # GdbController creates a process with subprocess.Popen(). Is it really running? It is probable that # an RPI under high load will get non-responsive during creating a lot of processes. - if not hasattr(self.gdb, 'verify_valid_gdb_subprocess'): + if not hasattr(self.gdbmi, 'verify_valid_gdb_subprocess'): # for pygdbmi >= 0.10.0.0 - verify_valid_gdb_subprocess(self.gdb.gdb_process) - resp = self.gdb.get_gdb_response( + verify_valid_gdb_subprocess(self.gdbmi.gdb_process) + resp = self.gdbmi.get_gdb_response( timeout_sec=10 ) # calls verify_valid_gdb_subprocess() internally for pygdbmi < 0.10.0.0 # it will be interesting to look up this response if the next GDB command fails (times out) @@ -207,17 +222,25 @@ class PanicTestDut(IdfDut): self.gdb_write('-file-exec-and-symbols {}'.format(self.app.elf_file)) # Connect GDB to UART - self.serial.proc.close() + self.serial.close() logging.info('Connecting to GDB Stub...') self.gdb_write('-gdb-set serial baud 115200') - responses = self.gdb_write('-target-select remote ' + self.serial.port) + + if sys.platform == 'darwin': + assert '/dev/tty.' not in self.serial.port, \ + '/dev/tty.* ports can\'t be used with GDB on macOS. Use with /dev/cu.* instead.' # Make sure we get the 'stopped' notification + responses = self.gdb_write('-target-select remote ' + self.serial.port) stop_response = self.find_gdb_response('stopped', 'notify', responses) - if not stop_response: + + retries = 3 + while not stop_response and retries > 0: + logging.info('Sending -exec-interrupt') responses = self.gdb_write('-exec-interrupt') stop_response = self.find_gdb_response('stopped', 'notify', responses) - assert stop_response + retries -= 1 + frame = stop_response['payload']['frame'] if 'file' not in frame: frame['file'] = '?' @@ -226,33 +249,32 @@ class PanicTestDut(IdfDut): logging.info('Stopped in {func} at {addr} ({file}:{line})'.format(**frame)) # Drain remaining responses - self.gdb.get_gdb_response(raise_error_on_timeout=False) + self.gdbmi.get_gdb_response(raise_error_on_timeout=False) def gdb_backtrace(self) -> Any: """ Returns the list of stack frames for the current thread. Each frame is a dictionary, refer to pygdbmi docs for the format. """ - assert self.gdb + assert self.gdbmi responses = self.gdb_write('-stack-list-frames') return self.find_gdb_response('done', 'result', responses)['payload']['stack'] @staticmethod - def match_backtrace( + def verify_gdb_backtrace( gdb_backtrace: List[Any], expected_functions_list: List[Any] - ) -> bool: + ) -> None: """ - Returns True if the function names listed in expected_functions_list match the backtrace + Raises an assert if the function names listed in expected_functions_list do not match the backtrace given by gdb_backtrace argument. The latter is in the same format as returned by gdb_backtrace() function. """ - return all( - [ - frame['func'] == expected_functions_list[i] - for i, frame in enumerate(gdb_backtrace) - ] - ) + actual_functions_list = [frame['func'] for frame in gdb_backtrace] + if actual_functions_list != expected_functions_list: + logging.error(f'Expected backtrace: {expected_functions_list}') + logging.error(f'Actual backtrace: {actual_functions_list}') + assert False, 'Got unexpected backtrace' @staticmethod def find_gdb_response(