From c753271de1148e1a5f228e3abee10a1a04f88828 Mon Sep 17 00:00:00 2001 From: Erhan Kurubas Date: Tue, 27 Feb 2024 23:51:28 +0100 Subject: [PATCH 1/5] ci(panic): add coredump tcb corrupted test --- .../system/panic/main/include/test_panic.h | 2 + .../system/panic/main/test_app_main.c | 1 + .../test_apps/system/panic/main/test_panic.c | 9 +++++ tools/test_apps/system/panic/pytest_panic.py | 40 ++++++++++++++++++- 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/tools/test_apps/system/panic/main/include/test_panic.h b/tools/test_apps/system/panic/main/include/test_panic.h index 3b52370a34..1ed1735475 100644 --- a/tools/test_apps/system/panic/main/include/test_panic.h +++ b/tools/test_apps/system/panic/main/include/test_panic.h @@ -67,6 +67,8 @@ void test_illegal_access(void); void test_capture_dram(void); +void test_tcb_corrupted(void); + #if CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH && CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF void test_setup_coredump_summary(void); void test_coredump_summary(void); diff --git a/tools/test_apps/system/panic/main/test_app_main.c b/tools/test_apps/system/panic/main/test_app_main.c index 6e3931b97b..3869bc7e50 100644 --- a/tools/test_apps/system/panic/main/test_app_main.c +++ b/tools/test_apps/system/panic/main/test_app_main.c @@ -116,6 +116,7 @@ void app_main(void) HANDLE_TEST(test_name, test_assert_cache_disabled); HANDLE_TEST(test_name, test_assert_cache_write_back_error_can_print_backtrace); HANDLE_TEST(test_name, test_assert_cache_write_back_error_can_print_backtrace2); + HANDLE_TEST(test_name, test_tcb_corrupted); #if CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH && CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF HANDLE_TEST(test_name, test_setup_coredump_summary); HANDLE_TEST(test_name, test_coredump_summary); diff --git a/tools/test_apps/system/panic/main/test_panic.c b/tools/test_apps/system/panic/main/test_panic.c index 5603279d48..a9d41b0227 100644 --- a/tools/test_apps/system/panic/main/test_panic.c +++ b/tools/test_apps/system/panic/main/test_panic.c @@ -301,6 +301,15 @@ void test_coredump_summary(void) } #endif +void test_tcb_corrupted(void) +{ + uint32_t *tcb_ptr = (uint32_t *)xTaskGetIdleTaskHandleForCore(0); + for (size_t i = 0; i < sizeof(StaticTask_t) / sizeof(uint32_t); ++i) { + tcb_ptr[i] = 0xDEADBEE0; + } + vTaskDelay(2); +} + /* NOTE: The following test verifies the behaviour for the * Xtensa-specific MPU instructions (Refer WDTLB, DSYNC, WDTIB, ISYNC) * used for memory protection. diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py index 34335987b4..249639ba7e 100644 --- a/tools/test_apps/system/panic/pytest_panic.py +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -1,8 +1,11 @@ # SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import re +from typing import Any from typing import List from typing import Optional +from typing import Pattern +from typing import Sequence from typing import Union import pexpect @@ -108,7 +111,7 @@ def get_default_backtrace(config: str) -> List[str]: def common_test(dut: PanicTestDut, config: str, expected_backtrace: Optional[List[str]] = None, check_cpu_reset: Optional[bool] = True, - expected_coredump: Optional[List[Union[str, re.Pattern]]] = None) -> None: + expected_coredump: Optional[Sequence[Union[str, Pattern[Any]]]] = None) -> None: if 'gdbstub' in config: dut.expect_exact('Entering gdb stub now.') dut.start_gdb_for_gdbstub() @@ -1042,3 +1045,38 @@ def test_coredump_summary(dut: PanicTestDut) -> None: @pytest.mark.parametrize('config', CONFIG_COREDUMP_SUMMARY_FLASH_ENCRYPTED, indirect=True) def test_coredump_summary_flash_encrypted(dut: PanicTestDut, config: str) -> None: _test_coredump_summary(dut, True, config == 'coredump_flash_encrypted') + + +@pytest.mark.parametrize('config', [pytest.param('coredump_flash_elf_sha', marks=TARGETS_ALL)], indirect=True) +@pytest.mark.generic +def test_tcb_corrupted(dut: PanicTestDut, target: str, config: str, test_func_name: str) -> None: + dut.run_test_func(test_func_name) + if dut.is_xtensa: + dut.expect_gme('LoadProhibited') + dut.expect_reg_dump(0) + dut.expect_corrupted_backtrace() + else: + dut.expect_gme('Load access fault') + dut.expect_reg_dump(0) + dut.expect_stack_dump() + + dut.expect_elf_sha256() + dut.expect_none('Guru Meditation') + + # TCB NAME + # ---------- ---------------- + if dut.is_multi_core: + regex_patterns = [rb'[0-9xa-fA-F] main', + rb'[0-9xa-fA-F] ipc0', + rb'[0-9xa-fA-F] ipc1'] + else: + regex_patterns = [rb'[0-9xa-fA-F] main'] + + coredump_pattern = [re.compile(pattern.decode('utf-8')) for pattern in regex_patterns] + + common_test( + dut, + config, + expected_backtrace=None, + expected_coredump=coredump_pattern + ) From 1848079dec7c77f35ed029bc79144df5db782768 Mon Sep 17 00:00:00 2001 From: Erhan Kurubas Date: Mon, 29 Apr 2024 22:53:59 +0200 Subject: [PATCH 2/5] fix(coredump): properly save task's stack when stack is located in PSRAM --- components/espcoredump/src/core_dump_elf.c | 99 ++++++++++++---------- 1 file changed, 55 insertions(+), 44 deletions(-) diff --git a/components/espcoredump/src/core_dump_elf.c b/components/espcoredump/src/core_dump_elf.c index db05493ca1..f374bc93ab 100644 --- a/components/espcoredump/src/core_dump_elf.c +++ b/components/espcoredump/src/core_dump_elf.c @@ -18,6 +18,7 @@ #ifdef CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF #include // for the MIN macro #include "esp_app_desc.h" +#include "esp_memory_utils.h" #endif #define ELF_CLASS ELFCLASS32 @@ -300,22 +301,6 @@ static int elf_add_regs(core_dump_elf_t *self, core_dump_task_header_t *task) len); } -static int elf_add_stack(core_dump_elf_t *self, core_dump_task_header_t *task) -{ - uint32_t stack_vaddr, stack_len = 0, stack_paddr = 0; - - ELF_CHECK_ERR((task), ELF_PROC_ERR_OTHER, "Invalid task pointer."); - - stack_len = esp_core_dump_get_stack(task, &stack_vaddr, &stack_paddr); - ESP_COREDUMP_LOG_PROCESS("Add stack for task 0x%x: addr 0x%x, sz %u", - task->tcb_addr, stack_vaddr, stack_len); - int ret = elf_add_segment(self, PT_LOAD, - (uint32_t)stack_vaddr, - (void*)stack_paddr, - (uint32_t) stack_len); - return ret; -} - static int elf_add_tcb(core_dump_elf_t *self, core_dump_task_header_t *task) { ELF_CHECK_ERR((task), ELF_PROC_ERR_OTHER, "Invalid task pointer."); @@ -348,17 +333,34 @@ static int elf_process_task_tcb(core_dump_elf_t *self, core_dump_task_header_t * static int elf_process_task_stack(core_dump_elf_t *self, core_dump_task_header_t *task) { - int ret = ELF_PROC_ERR_OTHER; + int ret = 0; + uint32_t stack_vaddr, stack_len = 0, stack_paddr = 0; ELF_CHECK_ERR((task), ELF_PROC_ERR_OTHER, "Invalid input data."); - ret = elf_add_stack(self, task); - if (ret <= 0) { - ESP_COREDUMP_LOGE("Task (TCB:%x), (Stack:%x), stack processing failure = %d.", - task->tcb_addr, - task->stack_start, - ret); + stack_len = esp_core_dump_get_stack(task, &stack_vaddr, &stack_paddr); + ESP_COREDUMP_LOG_PROCESS("Add stack for task 0x%x: addr 0x%x, sz %u", + task->tcb_addr, stack_vaddr, stack_len); + +#if CONFIG_ESP_COREDUMP_CAPTURE_DRAM + /* If the task data located in the PSRAM, we will save it here. + Otherwise it will be saved in esp_core_dump_store_section() + */ + if (esp_ptr_external_ram((void *)stack_vaddr)) +#endif + { + ret = elf_add_segment(self, PT_LOAD, + (uint32_t)stack_vaddr, + (void*)stack_paddr, + (uint32_t) stack_len); + if (ret <= 0) { + ESP_COREDUMP_LOGE("Task (TCB:%x), (Stack:%x), stack processing failure = %d.", + task->tcb_addr, + task->stack_start, + ret); + } } + return ret; } @@ -453,20 +455,13 @@ static int elf_save_task(core_dump_elf_t *self, core_dump_task_header_t *task) return elf_len; } -static int elf_write_tasks_data(core_dump_elf_t *self) +static int elf_process_task_data(core_dump_elf_t *self) { - int elf_len = 0; + int elf_len = 0, ret; core_dump_task_header_t task_hdr = { 0 }; core_dump_mem_seg_header_t interrupted_stack = { 0 }; TaskIterator_t task_iter; - int ret = ELF_PROC_ERR_OTHER; - uint16_t tasks_num = 0; - uint16_t bad_tasks_num = 0; - - ESP_COREDUMP_LOG_PROCESS("================ Processing task registers ================"); - ret = elf_process_tasks_regs(self); - ELF_CHECK_ERR((ret > 0), ret, "Tasks regs addition failed, return (%d).", ret); - elf_len += ret; + uint16_t tasks_num = 0, bad_tasks_num = 0; ESP_COREDUMP_LOG_PROCESS("================ Processing task data ================"); // processes all task's stack data and writes segment data into partition @@ -479,17 +474,16 @@ static int elf_write_tasks_data(core_dump_elf_t *self) bad_tasks_num++; continue; } - -#if CONFIG_ESP_COREDUMP_CAPTURE_DRAM - /* Only crashed task data will be saved here. The other task's data will be automatically saved within the sections */ - if (esp_core_dump_get_current_task_handle() == task_iter.pxTaskHandle) -#endif - { - ret = elf_save_task(self, &task_hdr); - ELF_CHECK_ERR((ret > 0), ret, - "Task %x, TCB write failed, return (%d).", task_iter.pxTaskHandle, ret); - elf_len += ret; - } + ret = elf_save_task(self, &task_hdr); + ELF_CHECK_ERR((ret > 0), ret, + "Task %x, TCB write failed, return (%d).", task_iter.pxTaskHandle, ret); + elf_len += ret; + /* interrupt stacks: + - 'port_IntStack' is in the data section for xtensa + - 'xIsrStack' is in the bss section for risc-v + When DRAM capture is enabled, interrupt stack saving can be done during the full section store + */ +#if !CONFIG_ESP_COREDUMP_CAPTURE_DRAM if (interrupted_stack.size > 0) { ESP_COREDUMP_LOG_PROCESS("Add interrupted task stack %lu bytes @ %x", interrupted_stack.size, interrupted_stack.start); @@ -500,11 +494,28 @@ static int elf_write_tasks_data(core_dump_elf_t *self) ELF_CHECK_ERR((ret > 0), ret, "Interrupted task stack write failed, return (%d).", ret); elf_len += ret; } +#endif } ESP_COREDUMP_LOG_PROCESS("Found %d bad task out of %d", bad_tasks_num, tasks_num); + return elf_len; } +static int elf_write_tasks_data(core_dump_elf_t *self) +{ + ESP_COREDUMP_LOG_PROCESS("================ Processing task registers ================"); + int ret = elf_process_tasks_regs(self); + ELF_CHECK_ERR((ret > 0), ret, "Tasks regs addition failed, return (%d).", ret); + int elf_len = ret; + + ret = elf_process_task_data(self); + if (ret <= 0) { + return ret; + } + + return elf_len + ret; +} + #if CONFIG_ESP_COREDUMP_CAPTURE_DRAM /* Coredump stack will also be used by the checksum functions while saving sections. From 5f3d3f5e92ab33da0b0ac5deecf3581394424368 Mon Sep 17 00:00:00 2001 From: Erhan Kurubas Date: Fri, 3 May 2024 00:00:03 +0200 Subject: [PATCH 3/5] ci(panic): extend extram_stack tests --- components/espcoredump/src/core_dump_elf.c | 18 ++++++---- tools/test_apps/system/panic/CMakeLists.txt | 2 +- .../system/panic/main/CMakeLists.txt | 4 +-- .../system/panic/main/include/test_panic.h | 5 ++- .../system/panic/main/test_app_main.c | 5 ++- .../test_apps/system/panic/main/test_panic.c | 11 +++++- tools/test_apps/system/panic/pytest_panic.py | 34 +++++++++++++++---- ...g.ci.coredump_flash_extram_stack_bss_esp32 | 13 +++++++ ...ci.coredump_flash_extram_stack_bss_esp32s2 | 13 +++++++ ...ci.coredump_flash_extram_stack_bss_esp32s3 | 13 +++++++ ...ci.coredump_flash_extram_stack_heap_esp32} | 2 -- ...i.coredump_flash_extram_stack_heap_esp32s2 | 9 +++++ ...i.coredump_flash_extram_stack_heap_esp32s3 | 9 +++++ 13 files changed, 117 insertions(+), 21 deletions(-) create mode 100644 tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32 create mode 100644 tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32s2 create mode 100644 tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32s3 rename tools/test_apps/system/panic/{sdkconfig.ci.coredump_extram_stack => sdkconfig.ci.coredump_flash_extram_stack_heap_esp32} (79%) create mode 100644 tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32s2 create mode 100644 tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32s3 diff --git a/components/espcoredump/src/core_dump_elf.c b/components/espcoredump/src/core_dump_elf.c index f374bc93ab..57ea48e115 100644 --- a/components/espcoredump/src/core_dump_elf.c +++ b/components/espcoredump/src/core_dump_elf.c @@ -334,7 +334,9 @@ static int elf_process_task_tcb(core_dump_elf_t *self, core_dump_task_header_t * static int elf_process_task_stack(core_dump_elf_t *self, core_dump_task_header_t *task) { int ret = 0; - uint32_t stack_vaddr, stack_len = 0, stack_paddr = 0; + uint32_t stack_vaddr; + uint32_t stack_len = 0; + uint32_t stack_paddr = 0; ELF_CHECK_ERR((task), ELF_PROC_ERR_OTHER, "Invalid input data."); @@ -343,8 +345,11 @@ static int elf_process_task_stack(core_dump_elf_t *self, core_dump_task_header_t task->tcb_addr, stack_vaddr, stack_len); #if CONFIG_ESP_COREDUMP_CAPTURE_DRAM - /* If the task data located in the PSRAM, we will save it here. - Otherwise it will be saved in esp_core_dump_store_section() + /* + When saving all data sections (enabled by `CONFIG_ESP_COREDUMP_CAPTURE_DRAM`), + the task stack located in DRAM will be saved in `esp_core_dump_store_section()`. + Therefore, we filter them out here. + PSRAM data do not fall into any ELF section, so we always save such stacks here. */ if (esp_ptr_external_ram((void *)stack_vaddr)) #endif @@ -457,11 +462,12 @@ static int elf_save_task(core_dump_elf_t *self, core_dump_task_header_t *task) static int elf_process_task_data(core_dump_elf_t *self) { - int elf_len = 0, ret; + int elf_len = 0; core_dump_task_header_t task_hdr = { 0 }; core_dump_mem_seg_header_t interrupted_stack = { 0 }; TaskIterator_t task_iter; - uint16_t tasks_num = 0, bad_tasks_num = 0; + uint16_t tasks_num = 0; + uint16_t bad_tasks_num = 0; ESP_COREDUMP_LOG_PROCESS("================ Processing task data ================"); // processes all task's stack data and writes segment data into partition @@ -474,7 +480,7 @@ static int elf_process_task_data(core_dump_elf_t *self) bad_tasks_num++; continue; } - ret = elf_save_task(self, &task_hdr); + int ret = elf_save_task(self, &task_hdr); ELF_CHECK_ERR((ret > 0), ret, "Task %x, TCB write failed, return (%d).", task_iter.pxTaskHandle, ret); elf_len += ret; diff --git a/tools/test_apps/system/panic/CMakeLists.txt b/tools/test_apps/system/panic/CMakeLists.txt index e9db4327f5..3305b7b4f6 100644 --- a/tools/test_apps/system/panic/CMakeLists.txt +++ b/tools/test_apps/system/panic/CMakeLists.txt @@ -17,7 +17,7 @@ if(CONFIG_TEST_MEMPROT) endif() endif() -if(NOT CONFIG_TEST_MEMPROT AND NOT CONFIG_ESP_COREDUMP_CAPTURE_DRAM) +if(NOT CONFIG_TEST_MEMPROT AND NOT CONFIG_ESP_COREDUMP_CAPTURE_DRAM AND NOT CONFIG_SPIRAM) # Enable UBSAN checks # # shift-base sanitizer is disabled due to the following pattern found in register header files: diff --git a/tools/test_apps/system/panic/main/CMakeLists.txt b/tools/test_apps/system/panic/main/CMakeLists.txt index 90e5bcf973..e3f9f4a0d2 100644 --- a/tools/test_apps/system/panic/main/CMakeLists.txt +++ b/tools/test_apps/system/panic/main/CMakeLists.txt @@ -13,8 +13,8 @@ endif() idf_component_register(SRCS "${srcs}" INCLUDE_DIRS "include" - REQUIRES spi_flash esp_psram esp_system esp_partition espcoredump - PRIV_REQUIRES esp_gdbstub) + REQUIRES spi_flash esp_psram esp_system esp_partition + PRIV_REQUIRES esp_gdbstub espcoredump) target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-unused-variable" "-Wno-infinite-recursion" diff --git a/tools/test_apps/system/panic/main/include/test_panic.h b/tools/test_apps/system/panic/main/include/test_panic.h index 1ed1735475..71733add77 100644 --- a/tools/test_apps/system/panic/main/include/test_panic.h +++ b/tools/test_apps/system/panic/main/include/test_panic.h @@ -32,7 +32,10 @@ void test_hw_stack_guard_cpu1(void); #endif // CONFIG_ESP_SYSTEM_HW_STACK_GUARD #if CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH && CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY -void test_panic_extram_stack(void); +void test_panic_extram_stack_heap(void); +#if CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY +void test_panic_extram_stack_bss(void); +#endif #endif #if !CONFIG_FREERTOS_UNICORE diff --git a/tools/test_apps/system/panic/main/test_app_main.c b/tools/test_apps/system/panic/main/test_app_main.c index 3869bc7e50..b55f7de765 100644 --- a/tools/test_apps/system/panic/main/test_app_main.c +++ b/tools/test_apps/system/panic/main/test_app_main.c @@ -96,7 +96,10 @@ void app_main(void) #endif // CONFIG_FREERTOS_UNICORE #endif // CONFIG_ESP_SYSTEM_HW_STACK_GUARD #if CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH && CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY - HANDLE_TEST(test_name, test_panic_extram_stack); + HANDLE_TEST(test_name, test_panic_extram_stack_heap); +#if CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY + HANDLE_TEST(test_name, test_panic_extram_stack_bss); +#endif #endif #if CONFIG_ESP_COREDUMP_CAPTURE_DRAM HANDLE_TEST(test_name, test_capture_dram); diff --git a/tools/test_apps/system/panic/main/test_panic.c b/tools/test_apps/system/panic/main/test_panic.c index a9d41b0227..612615ffdf 100644 --- a/tools/test_apps/system/panic/main/test_panic.c +++ b/tools/test_apps/system/panic/main/test_panic.c @@ -104,7 +104,7 @@ static void stack_in_extram(void* arg) { abort(); } -void test_panic_extram_stack(void) { +void test_panic_extram_stack_heap(void) { /* Start by initializing a Task which has a stack in external RAM */ StaticTask_t handle; const uint32_t stack_size = 8192; @@ -119,8 +119,17 @@ void test_panic_extram_stack(void) { vTaskDelay(1000); } +#if CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY +static EXT_RAM_BSS_ATTR StackType_t stack[8192]; +void test_panic_extram_stack_bss(void) +{ + StaticTask_t handle; + xTaskCreateStatic(stack_in_extram, "Task_stack_extram", sizeof(stack), NULL, 4, stack, &handle); + vTaskDelay(1000); +} +#endif #endif // ESP_COREDUMP_ENABLE_TO_FLASH && SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py index 249639ba7e..eff37f4f9e 100644 --- a/tools/test_apps/system/panic/pytest_panic.py +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -70,11 +70,13 @@ CONFIGS_DUAL_CORE = [ # This list is used to check if the target is a dual-core one. TARGETS_DUAL_CORE_NAMES = [x.mark.name for x in TARGETS_DUAL_CORE] -# The tests which panic on external stack require PSRAM capable runners CONFIGS_EXTRAM_STACK = [ - pytest.param('coredump_extram_stack', marks=[pytest.mark.esp32, pytest.mark.psram]), - pytest.param('coredump_extram_stack', marks=[pytest.mark.esp32s2, pytest.mark.generic]), - pytest.param('coredump_extram_stack', marks=[pytest.mark.esp32s3, pytest.mark.quad_psram]), + pytest.param('coredump_flash_extram_stack_heap_esp32', marks=[pytest.mark.esp32, pytest.mark.psram]), + pytest.param('coredump_flash_extram_stack_heap_esp32s2', marks=[pytest.mark.esp32s2, pytest.mark.generic]), + pytest.param('coredump_flash_extram_stack_heap_esp32s3', marks=[pytest.mark.esp32s3, pytest.mark.quad_psram]), + pytest.param('coredump_flash_extram_stack_bss_esp32', marks=[pytest.mark.esp32, pytest.mark.psram]), + pytest.param('coredump_flash_extram_stack_bss_esp32s2', marks=[pytest.mark.esp32s2, pytest.mark.generic]), + pytest.param('coredump_flash_extram_stack_bss_esp32s3', marks=[pytest.mark.esp32s3, pytest.mark.quad_psram]), ] CONFIGS_HW_STACK_GUARD = [ @@ -208,8 +210,11 @@ def test_task_wdt_cpu1(dut: PanicTestDut, config: str, test_func_name: str) -> N @pytest.mark.parametrize('config', CONFIGS_EXTRAM_STACK, indirect=True) -def test_panic_extram_stack(dut: PanicTestDut, config: str, test_func_name: str) -> None: - dut.run_test_func(test_func_name) +def test_panic_extram_stack(dut: PanicTestDut, config: str) -> None: + if 'heap' in config: + dut.run_test_func('test_panic_extram_stack_heap') + else: + dut.run_test_func('test_panic_extram_stack_bss') dut.expect_none('Allocated stack is not in external RAM') dut.expect_none('Guru Meditation') dut.expect_backtrace() @@ -222,7 +227,22 @@ def test_panic_extram_stack(dut: PanicTestDut, config: str, test_func_name: str) # The caller must be accessible after restoring the stack dut.expect_exact('Core dump has been saved to flash.') - common_test(dut, config) + if dut.target == 'esp32': + # ESP32 External data memory range [0x3f800000-0x3fc00000) + coredump_pattern = re.compile('.coredump.tasks.data (0x3[fF][8-9a-bA-B][0-9a-fA-F]{5}) (0x[a-fA-F0-9]+) RW') + elif dut.target == 'esp32s2': + # ESP32-S2 External data memory range [0x3f500000-0x3ff80000) + coredump_pattern = re.compile('.coredump.tasks.data (0x3[fF][5-9a-fA-F][0-7][0-9a-fA-F]{4}) (0x[a-fA-F0-9]+) RW') + else: + # ESP32-S3 External data memory range [0x3c000000-0x3e000000) + coredump_pattern = re.compile('.coredump.tasks.data (0x3[c-dC-D][0-9a-fA-F]{6}) (0x[a-fA-F0-9]+) RW') + + common_test( + dut, + config, + expected_backtrace=None, + expected_coredump=[coredump_pattern] + ) @pytest.mark.parametrize('config', CONFIGS, indirect=True) diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32 b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32 new file mode 100644 index 0000000000..26769c3c59 --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32 @@ -0,0 +1,13 @@ +CONFIG_IDF_TARGET="esp32" +CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=y +CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=y +CONFIG_ESP_COREDUMP_CHECKSUM_SHA256=y +# We need to have the coredump info log +CONFIG_LOG_DEFAULT_LEVEL_INFO=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY=y +CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y +CONFIG_ESP_COREDUMP_USE_STACK_SIZE=y +CONFIG_ESP_COREDUMP_CAPTURE_DRAM=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_capture_dram.csv" diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32s2 b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32s2 new file mode 100644 index 0000000000..ad1a91fa81 --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32s2 @@ -0,0 +1,13 @@ +CONFIG_IDF_TARGET="esp32s2" +CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=y +CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=y +CONFIG_ESP_COREDUMP_CHECKSUM_SHA256=y +# We need to have the coredump info log +CONFIG_LOG_DEFAULT_LEVEL_INFO=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY=y +CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y +CONFIG_ESP_COREDUMP_USE_STACK_SIZE=y +CONFIG_ESP_COREDUMP_CAPTURE_DRAM=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_capture_dram.csv" diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32s3 b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32s3 new file mode 100644 index 0000000000..46903a0223 --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_bss_esp32s3 @@ -0,0 +1,13 @@ +CONFIG_IDF_TARGET="esp32s3" +CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=y +CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=y +CONFIG_ESP_COREDUMP_CHECKSUM_SHA256=y +# We need to have the coredump info log +CONFIG_LOG_DEFAULT_LEVEL_INFO=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY=y +CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y +CONFIG_ESP_COREDUMP_USE_STACK_SIZE=y +CONFIG_ESP_COREDUMP_CAPTURE_DRAM=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_capture_dram.csv" diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_extram_stack b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32 similarity index 79% rename from tools/test_apps/system/panic/sdkconfig.ci.coredump_extram_stack rename to tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32 index 06ac9fcc8e..18391318a3 100644 --- a/tools/test_apps/system/panic/sdkconfig.ci.coredump_extram_stack +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32 @@ -2,8 +2,6 @@ CONFIG_IDF_TARGET="esp32" CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=y CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=y CONFIG_ESP_COREDUMP_CHECKSUM_SHA256=y -# CONFIG_ESP_SYSTEM_PANIC_GDBSTUB is not set -CONFIG_ESP_SYSTEM_PANIC_PRINT_HALT=y # We need to have the coredump info log CONFIG_LOG_DEFAULT_LEVEL_INFO=y CONFIG_SPIRAM=y diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32s2 b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32s2 new file mode 100644 index 0000000000..981803c0be --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32s2 @@ -0,0 +1,9 @@ +CONFIG_IDF_TARGET="esp32s2" +CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=y +CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=y +CONFIG_ESP_COREDUMP_CHECKSUM_SHA256=y +# We need to have the coredump info log +CONFIG_LOG_DEFAULT_LEVEL_INFO=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY=y +CONFIG_ESP_COREDUMP_USE_STACK_SIZE=y diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32s3 b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32s3 new file mode 100644 index 0000000000..ff952817bd --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_extram_stack_heap_esp32s3 @@ -0,0 +1,9 @@ +CONFIG_IDF_TARGET="esp32s3" +CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=y +CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=y +CONFIG_ESP_COREDUMP_CHECKSUM_SHA256=y +# We need to have the coredump info log +CONFIG_LOG_DEFAULT_LEVEL_INFO=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY=y +CONFIG_ESP_COREDUMP_USE_STACK_SIZE=y From ef14d7ab64887ecce13bfa59b039867f86a3e17e Mon Sep 17 00:00:00 2001 From: Erhan Kurubas Date: Mon, 12 Aug 2024 14:54:29 +0200 Subject: [PATCH 4/5] test(coredump): fix failed core dump tests --- tools/test_apps/system/panic/pytest_panic.py | 43 ++++++++----------- .../panic/sdkconfig.ci.coredump_flash_bin_crc | 1 + .../panic/sdkconfig.ci.coredump_flash_elf_sha | 1 + .../system/panic/test_panic_util/panic_dut.py | 23 +++++----- 4 files changed, 33 insertions(+), 35 deletions(-) diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py index eff37f4f9e..28f541cfa1 100644 --- a/tools/test_apps/system/panic/pytest_panic.py +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -112,9 +112,20 @@ def get_default_backtrace(config: str) -> List[str]: return [config, 'app_main', 'main_task', 'vPortTaskWrapper'] +def expect_coredump_flash_write_logs(dut: PanicTestDut, config: str) -> None: + dut.expect_exact('Save core dump to flash...') + if 'extram_stack' in config: + dut.expect_exact('Backing up stack @') + dut.expect_exact('Restoring stack') + dut.expect_exact('Core dump has been saved to flash.') + dut.expect('Rebooting...') + + def common_test(dut: PanicTestDut, config: str, expected_backtrace: Optional[List[str]] = None, check_cpu_reset: Optional[bool] = True, expected_coredump: Optional[Sequence[Union[str, Pattern[Any]]]] = None) -> None: if 'gdbstub' in config: + if 'coredump' in config: + dut.process_coredump_uart(expected_coredump, False) dut.expect_exact('Entering gdb stub now.') dut.start_gdb_for_gdbstub() frames = dut.gdb_backtrace() @@ -130,11 +141,10 @@ def common_test(dut: PanicTestDut, config: str, expected_backtrace: Optional[Lis if 'uart' in config: dut.process_coredump_uart(expected_coredump) elif 'flash' in config: + expect_coredump_flash_write_logs(dut, config) dut.process_coredump_flash(expected_coredump) elif 'panic' in config: - pass - - dut.expect('Rebooting...') + dut.expect('Rebooting...') if check_cpu_reset: dut.expect_cpu_reset() @@ -219,13 +229,6 @@ def test_panic_extram_stack(dut: PanicTestDut, config: str) -> None: dut.expect_none('Guru Meditation') dut.expect_backtrace() dut.expect_elf_sha256() - # Check that coredump is getting written to flash - dut.expect_exact('Save core dump to flash...') - # And that the stack is replaced and restored - dut.expect_exact('Backing up stack @') - dut.expect_exact('Restoring stack') - # The caller must be accessible after restoring the stack - dut.expect_exact('Core dump has been saved to flash.') if dut.target == 'esp32': # ESP32 External data memory range [0x3f800000-0x3fc00000) @@ -882,7 +885,7 @@ def test_rtc_slow_reg1_execute_violation(dut: PanicTestDut, test_func_name: str) dut.expect_gme('Memory protection fault') dut.expect(r'Read operation at address [0-9xa-f]+ not permitted \((\S+)\)') dut.expect_reg_dump(0) - dut.expect_corrupted_backtrace() + dut.expect_backtrace(corrupted=True) dut.expect_cpu_reset() @@ -893,7 +896,7 @@ def test_rtc_slow_reg2_execute_violation(dut: PanicTestDut, test_func_name: str) dut.expect_gme('Memory protection fault') dut.expect(r'Read operation at address [0-9xa-f]+ not permitted \((\S+)\)') dut.expect_reg_dump(0) - dut.expect_corrupted_backtrace() + dut.expect_backtrace(corrupted=True) dut.expect_cpu_reset() @@ -948,15 +951,7 @@ def test_invalid_memory_region_execute_violation(dut: PanicTestDut, test_func_na def test_gdbstub_coredump(dut: PanicTestDut) -> None: test_func_name = 'test_storeprohibited' dut.run_test_func(test_func_name) - - dut.process_coredump_uart() - - dut.expect_exact('Entering gdb stub now.') - dut.start_gdb_for_gdbstub() - frames = dut.gdb_backtrace() - dut.verify_gdb_backtrace(frames, get_default_backtrace(test_func_name)) - dut.revert_log_level() - return # don't expect "Rebooting" output below + common_test(dut, 'gdbstub_coredump', get_default_backtrace(test_func_name)) def test_hw_stack_guard_cpu(dut: PanicTestDut, cpu: int) -> None: @@ -1019,9 +1014,7 @@ def test_capture_dram(dut: PanicTestDut, config: str, test_func_name: str) -> No dut.expect_elf_sha256() dut.expect_none(['Guru Meditation', 'Re-entered core dump']) - dut.expect_exact('Save core dump to flash...') - dut.expect_exact('Core dump has been saved to flash.') - dut.expect('Rebooting...') + expect_coredump_flash_write_logs(dut, config) core_elf_file = dut.process_coredump_flash() dut.start_gdb_for_coredump(core_elf_file) @@ -1074,7 +1067,7 @@ def test_tcb_corrupted(dut: PanicTestDut, target: str, config: str, test_func_na if dut.is_xtensa: dut.expect_gme('LoadProhibited') dut.expect_reg_dump(0) - dut.expect_corrupted_backtrace() + dut.expect_backtrace() else: dut.expect_gme('Load access fault') dut.expect_reg_dump(0) diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_bin_crc b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_bin_crc index 8dd420e18f..948fec4789 100644 --- a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_bin_crc +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_bin_crc @@ -1,6 +1,7 @@ CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=y CONFIG_ESP_COREDUMP_DATA_FORMAT_BIN=y CONFIG_ESP_COREDUMP_CHECKSUM_CRC32=y +CONFIG_LOG_DEFAULT_LEVEL_INFO=y # static D/IRAM usage 97%, add this to reduce CONFIG_HAL_ASSERTION_DISABLE=y diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_elf_sha b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_elf_sha index 6af50e4f1c..43d6fdc6e6 100644 --- a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_elf_sha +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_elf_sha @@ -1,3 +1,4 @@ CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=y CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=y CONFIG_ESP_COREDUMP_CHECKSUM_SHA256=y +CONFIG_LOG_DEFAULT_LEVEL_INFO=y diff --git a/tools/test_apps/system/panic/test_panic_util/panic_dut.py b/tools/test_apps/system/panic/test_panic_util/panic_dut.py index d738c307ae..b19b32d539 100644 --- a/tools/test_apps/system/panic/test_panic_util/panic_dut.py +++ b/tools/test_apps/system/panic/test_panic_util/panic_dut.py @@ -80,15 +80,13 @@ class PanicTestDut(IdfDut): except pexpect.TIMEOUT: pass - def expect_backtrace(self) -> None: + def expect_backtrace(self, corrupted: bool = False) -> None: assert self.is_xtensa, 'Backtrace can be printed only on Xtensa' match = self.expect(r'Backtrace:( 0x[0-9a-fA-F]{8}:0x[0-9a-fA-F]{8})+(?P \|<-CORRUPTED)?') - assert not match.group('corrupted') - - def expect_corrupted_backtrace(self) -> None: - assert self.is_xtensa, 'Backtrace can be printed only on Xtensa' - self.expect_exact('Backtrace:') - self.expect_exact('CORRUPTED') + if corrupted: + assert match.group('corrupted') + else: + assert match def expect_stack_dump(self) -> None: assert not self.is_xtensa, 'Stack memory dump is only printed on RISC-V' @@ -153,11 +151,16 @@ class PanicTestDut(IdfDut): self.coredump_output.flush() self.coredump_output.seek(0) - def process_coredump_uart(self, expected: Optional[List[Union[str, re.Pattern]]] = None) -> None: + def process_coredump_uart( + self, expected: Optional[List[Union[str, re.Pattern]]] = None, wait_reboot: bool = True + ) -> None: """Extract the core dump from UART output of the test, run espcoredump on it""" self.expect(self.COREDUMP_UART_START) - res = self.expect('(.+)' + self.COREDUMP_UART_END) - coredump_base64 = res.group(1).decode('utf8') + uart_data = self.expect('(.+)' + self.COREDUMP_UART_END) + self.expect(re.compile(r"Coredump checksum='([a-fA-F0-9]+)'")) + if wait_reboot: + self.expect('Rebooting...') + coredump_base64 = uart_data.group(1).decode('utf8') with open(os.path.join(self.logdir, 'coredump_data.b64'), 'w') as coredump_file: logging.info('Writing UART base64 core dump to %s', coredump_file.name) coredump_file.write(coredump_base64) From dfc248d66d9faedafbf89a1be5db3f0ba333ad3e Mon Sep 17 00:00:00 2001 From: Erhan Kurubas Date: Fri, 29 Nov 2024 13:36:07 +0100 Subject: [PATCH 5/5] change(tools): enhance `expect_reg_dump` to support any or specific core values --- tools/test_apps/system/panic/pytest_panic.py | 4 ++-- .../system/panic/test_panic_util/panic_dut.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py index 28f541cfa1..6f37f36ef0 100644 --- a/tools/test_apps/system/panic/pytest_panic.py +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -1066,11 +1066,11 @@ def test_tcb_corrupted(dut: PanicTestDut, target: str, config: str, test_func_na dut.run_test_func(test_func_name) if dut.is_xtensa: dut.expect_gme('LoadProhibited') - dut.expect_reg_dump(0) + dut.expect_reg_dump() dut.expect_backtrace() else: dut.expect_gme('Load access fault') - dut.expect_reg_dump(0) + dut.expect_reg_dump() dut.expect_stack_dump() dut.expect_elf_sha256() diff --git a/tools/test_apps/system/panic/test_panic_util/panic_dut.py b/tools/test_apps/system/panic/test_panic_util/panic_dut.py index b19b32d539..2619008fe0 100644 --- a/tools/test_apps/system/panic/test_panic_util/panic_dut.py +++ b/tools/test_apps/system/panic/test_panic_util/panic_dut.py @@ -96,9 +96,13 @@ class PanicTestDut(IdfDut): """Expect method for Guru Meditation Errors""" self.expect_exact(f"Guru Meditation Error: Core 0 panic'ed ({reason})") - def expect_reg_dump(self, core: int = 0) -> None: - """Expect method for the register dump""" - self.expect(r'Core\s+%d register dump:' % core) + def expect_reg_dump(self, core: Optional[int] = None) -> None: + if core is None: + # Match any core num + self.expect(r'Core\s+\d+\s+register dump:') + else: + # Match the exact core num provided + self.expect(r'Core\s+%d\s+register dump:' % core) def expect_cpu_reset(self) -> None: # no digital system reset for panic handling restarts (see IDF-7255)