Merge branch 'feature/coredump_new_ci_tests' into 'master'

coredump: Add CI tests

See merge request espressif/esp-idf!8998

tools/ci/config/target-test.yml

@@ -373,7 +373,7 @@ example_test_016:
 
 test_app_test_001:
   extends: .test_app_template
-  parallel: 2
+  parallel: 4
   tags:
     - ESP32
     - test_jtag_arm

tools/test_apps/system/panic/app_test.py

@@ -1,175 +1,249 @@
 #!/usr/bin/env python
-import re
+import panic_tests as test
-from test_panic_util.test_panic_util import panic_test, get_dut, run_all
+from test_panic_util.test_panic_util import panic_test, run_all
 
 
+# test_task_wdt
+
 @panic_test()
 def test_panic_task_wdt(env, extra_data):
-    with get_dut(env, "panic", "test_task_wdt", qemu_wdt_enable=True) as dut:
+    test.task_wdt_inner(env, "panic")
-        dut.expect("Task watchdog got triggered. The following tasks did not reset the watchdog in time:")
-        dut.expect("CPU 0: main")
-        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
-        dut.expect_none("register dump:")
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
 
 
+@panic_test()
+def test_coredump_task_wdt_uart_elf_crc(env, extra_data):
+    test.task_wdt_inner(env, "coredump_uart_elf_crc")
+
+
+@panic_test()
+def test_coredump_task_wdt_uart_bin_crc(env, extra_data):
+    test.task_wdt_inner(env, "coredump_uart_bin_crc")
+
+
+@panic_test()
+def test_coredump_task_wdt_flash_elf_sha(env, extra_data):
+    test.task_wdt_inner(env, "coredump_flash_elf_sha")
+
+
+@panic_test()
+def test_coredump_task_wdt_flash_bin_crc(env, extra_data):
+    test.task_wdt_inner(env, "coredump_flash_bin_crc")
+
+
+# test_int_wdt
+
 @panic_test()
 def test_panic_int_wdt(env, extra_data):
-    with get_dut(env, "panic", "test_int_wdt", qemu_wdt_enable=True) as dut:
+    test.int_wdt_inner(env, "panic")
-        dut.expect_gme("Interrupt wdt timeout on CPU0")
-        dut.expect_reg_dump(0)
-        dut.expect("Backtrace:")
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect_reg_dump(1)
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
 
 
 @panic_test()
-def test_cache_error(env, extra_data):
+def test_coredump_int_wdt_uart_elf_crc(env, extra_data):
-    with get_dut(env, "panic", "test_cache_error") as dut:
+    test.int_wdt_inner(env, "coredump_uart_elf_crc")
-        dut.expect("Re-enable cpu cache.")
-        dut.expect_gme("Cache disabled but cached memory region accessed")
-        dut.expect_reg_dump(0)
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
 
 
+@panic_test()
+def test_coredump_int_wdt_uart_bin_crc(env, extra_data):
+    test.int_wdt_inner(env, "coredump_uart_bin_crc")
+
+
+@panic_test()
+def test_coredump_int_wdt_flash_elf_sha(env, extra_data):
+    test.int_wdt_inner(env, "coredump_flash_elf_sha")
+
+
+@panic_test()
+def test_coredump_int_wdt_flash_bin_crc(env, extra_data):
+    test.int_wdt_inner(env, "coredump_flash_bin_crc")
+
+
+# test_int_wdt_cache_disabled
+
 @panic_test()
 def test_panic_int_wdt_cache_disabled(env, extra_data):
-    with get_dut(env, "panic", "test_int_wdt_cache_disabled", qemu_wdt_enable=True) as dut:
+    test.int_wdt_cache_disabled_inner(env, "panic")
-        dut.expect("Re-enable cpu cache.")
-        dut.expect_gme("Interrupt wdt timeout on CPU0")
-        dut.expect_reg_dump(0)
-        dut.expect("Backtrace:")
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect_reg_dump(1)
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
 
 
 @panic_test()
-def test_panic_abort(env, extra_data):
+def test_coredump_int_wdt_cache_disabled_uart_elf_crc(env, extra_data):
-    with get_dut(env, "panic", "test_abort") as dut:
+    test.int_wdt_cache_disabled_inner(env, "coredump_uart_elf_crc")
-        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
-        dut.expect_none("register dump:")
-        dut.expect("Backtrace:")
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
 
 
 @panic_test()
-def test_panic_storeprohibited(env, extra_data):
+def test_coredump_int_wdt_cache_disabled_uart_bin_crc(env, extra_data):
-    with get_dut(env, "panic", "test_storeprohibited") as dut:
+    test.int_wdt_cache_disabled_inner(env, "coredump_uart_bin_crc")
-        dut.expect_gme("StoreProhibited")
-        dut.expect_reg_dump(0)
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
 
 
+@panic_test()
+def test_coredump_int_wdt_cache_disabled_flash_elf_sha(env, extra_data):
+    test.int_wdt_cache_disabled_inner(env, "coredump_flash_elf_sha")
+
+
+@panic_test()
+def test_coredump_int_wdt_cache_disabled_flash_bin_crc(env, extra_data):
+    test.int_wdt_cache_disabled_inner(env, "coredump_flash_bin_crc")
+
+
+# test_cache_error
+
+@panic_test()
+def test_panic_cache_error(env, extra_data):
+    test.cache_error_inner(env, "panic")
+
+
+@panic_test()
+def test_coredump_cache_error_uart_elf_crc(env, extra_data):
+    test.cache_error_inner(env, "coredump_uart_elf_crc")
+
+
+@panic_test()
+def test_coredump_cache_error_uart_bin_crc(env, extra_data):
+    test.cache_error_inner(env, "coredump_uart_bin_crc")
+
+
+@panic_test()
+def test_coredump_cache_error_flash_elf_sha(env, extra_data):
+    test.cache_error_inner(env, "coredump_flash_elf_sha")
+
+
+@panic_test()
+def test_coredump_cache_error_flash_bin_crc(env, extra_data):
+    test.cache_error_inner(env, "coredump_flash_bin_crc")
+
+
+# test_stack_overflow
+
 @panic_test()
 def test_panic_stack_overflow(env, extra_data):
-    with get_dut(env, "panic", "test_stack_overflow") as dut:
+    test.stack_overflow_inner(env, "panic")
-        dut.expect_gme("Unhandled debug exception")
-        dut.expect("Stack canary watchpoint triggered (main)")
-        dut.expect_reg_dump(0)
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
 
 
 @panic_test()
-def test_panic_illegal_instruction(env, extra_data):
+def test_coredump_stack_overflow_uart_elf_crc(env, extra_data):
-    with get_dut(env, "panic", "test_illegal_instruction") as dut:
+    test.stack_overflow_inner(env, "coredump_uart_elf_crc")
-        dut.expect_gme("IllegalInstruction")
-        dut.expect_reg_dump(0)
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
 
 
+@panic_test()
+def test_coredump_stack_overflow_uart_bin_crc(env, extra_data):
+    test.stack_overflow_inner(env, "coredump_uart_bin_crc")
+
+
+@panic_test()
+def test_coredump_stack_overflow_flash_elf_sha(env, extra_data):
+    test.stack_overflow_inner(env, "coredump_flash_elf_sha")
+
+
+@panic_test()
+def test_coredump_stack_overflow_flash_bin_crc(env, extra_data):
+    test.stack_overflow_inner(env, "coredump_flash_bin_crc")
+
+
+# test_instr_fetch_prohibited
+
 @panic_test()
 def test_panic_instr_fetch_prohibited(env, extra_data):
-    with get_dut(env, "panic", "test_instr_fetch_prohibited") as dut:
+    test.instr_fetch_prohibited_inner(env, "panic")
-        dut.expect_gme("InstrFetchProhibited")
-        dut.expect_reg_dump(0)
-        dut.expect("Backtrace:")
-        # At the moment the backtrace is corrupted, need to jump over the first PC in case of InstrFetchProhibited.
-        # Fix this and change expect to expect_none.
-        dut.expect("CORRUPTED")
-        dut.expect_elf_sha256()
-        dut.expect_none("Guru Meditation")
-        dut.expect("Rebooting...")
 
 
 @panic_test()
-def test_coredump_uart_abort(env, extra_data):
+def test_coredump_instr_fetch_prohibited_uart_elf_crc(env, extra_data):
-    with get_dut(env, "coredump_uart", "test_abort") as dut:
+    test.instr_fetch_prohibited_inner(env, "coredump_uart_elf_crc")
-        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation", "Re-entered core dump")
-        dut.expect(dut.COREDUMP_UART_END)
-        dut.expect("Rebooting...")
-        dut.process_coredump_uart()
-        # TODO: check the contents of core dump output
 
 
 @panic_test()
-def test_coredump_uart_int_wdt(env, extra_data):
+def test_coredump_instr_fetch_prohibited_uart_bin_crc(env, extra_data):
-    with get_dut(env, "coredump_uart", "test_int_wdt") as dut:
+    test.instr_fetch_prohibited_inner(env, "coredump_uart_bin_crc")
-        dut.expect_gme("Interrupt wdt timeout on CPU0")
-        dut.expect_reg_dump(0)
-        dut.expect("Backtrace:")
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect_reg_dump(1)
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation")
-        dut.expect("Rebooting...")
-        dut.process_coredump_uart()
-        # TODO: check the contents of core dump output
 
 
 @panic_test()
-def test_coredump_flash_abort(env, extra_data):
+def test_coredump_instr_fetch_prohibited_flash_elf_sha(env, extra_data):
-    with get_dut(env, "coredump_flash", "test_abort") as dut:
+    test.instr_fetch_prohibited_inner(env, "coredump_flash_elf_sha")
-        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
-        dut.expect("Backtrace:")
-        dut.expect_elf_sha256()
-        dut.expect_none("CORRUPTED", "Guru Meditation", "Re-entered core dump")
-        dut.expect("Rebooting...")
-        dut.process_coredump_flash()
-        # TODO: check the contents of core dump output
 
 
 @panic_test()
-def test_coredump_flash_int_wdt(env, extra_data):
+def test_coredump_instr_fetch_prohibited_flash_bin_crc(env, extra_data):
-    with get_dut(env, "coredump_flash", "test_int_wdt") as dut:
+    test.instr_fetch_prohibited_inner(env, "coredump_flash_bin_crc")
-        dut.expect_gme("Interrupt wdt timeout on CPU0")
+
-        dut.expect_reg_dump(0)
+
-        dut.expect("Backtrace:")
+# test_illegal_instruction
-        dut.expect_none("CORRUPTED", "Guru Meditation")
+
-        dut.expect_reg_dump(1)
+@panic_test()
-        dut.expect("Backtrace:")
+def test_panic_illegal_instruction(env, extra_data):
-        dut.expect_elf_sha256()
+    test.illegal_instruction_inner(env, "panic")
-        dut.expect_none("CORRUPTED", "Guru Meditation")
+
-        dut.expect("Rebooting...")
+
-        dut.process_coredump_flash()
+@panic_test()
-        # TODO: check the contents of core dump output
+def test_coredump_illegal_instruction_uart_elf_crc(env, extra_data):
+    test.illegal_instruction_inner(env, "coredump_uart_elf_crc")
+
+
+@panic_test()
+def test_coredump_illegal_instruction_uart_bin_crc(env, extra_data):
+    test.illegal_instruction_inner(env, "coredump_uart_bin_crc")
+
+
+@panic_test()
+def test_coredump_illegal_instruction_flash_elf_sha(env, extra_data):
+    test.illegal_instruction_inner(env, "coredump_flash_elf_sha")
+
+
+@panic_test()
+def test_coredump_illegal_instruction_flash_bin_crc(env, extra_data):
+    test.illegal_instruction_inner(env, "coredump_flash_bin_crc")
+
+
+# test_storeprohibited
+
+@panic_test()
+def test_panic_storeprohibited(env, extra_data):
+    test.storeprohibited_inner(env, "panic")
+
+
+@panic_test()
+def test_coredump_storeprohibited_uart_elf_crc(env, extra_data):
+    test.storeprohibited_inner(env, "coredump_uart_elf_crc")
+
+
+@panic_test()
+def test_coredump_storeprohibited_uart_bin_crc(env, extra_data):
+    test.storeprohibited_inner(env, "coredump_uart_bin_crc")
+
+
+@panic_test()
+def test_coredump_storeprohibited_flash_elf_sha(env, extra_data):
+    test.storeprohibited_inner(env, "coredump_flash_elf_sha")
+
+
+@panic_test()
+def test_coredump_storeprohibited_flash_bin_crc(env, extra_data):
+    test.storeprohibited_inner(env, "coredump_flash_bin_crc")
+
+
+# test_abort
+
+@panic_test()
+def test_panic_abort(env, extra_data):
+    test.abort_inner(env, "panic")
+
+
+@panic_test()
+def test_coredump_abort_uart_elf_crc(env, extra_data):
+    test.abort_inner(env, "coredump_uart_elf_crc")
+
+
+@panic_test()
+def test_coredump_abort_uart_bin_crc(env, extra_data):
+    test.abort_inner(env, "coredump_uart_bin_crc")
+
+
+@panic_test()
+def test_coredump_abort_flash_elf_sha(env, extra_data):
+    test.abort_inner(env, "coredump_flash_elf_sha")
+
+
+@panic_test()
+def test_coredump_abort_flash_bin_crc(env, extra_data):
+    test.abort_inner(env, "coredump_flash_bin_crc")
 
 
 if __name__ == '__main__':

tools/test_apps/system/panic/main/test_panic_main.c

@@ -102,7 +102,7 @@ static void IRAM_ATTR test_int_wdt_cache_disabled(void)
 
 static void test_stack_overflow(void)
 {
-    volatile uint8_t stuff[CONFIG_ESP_MAIN_TASK_STACK_SIZE * 2];
+    volatile uint8_t stuff[CONFIG_ESP_MAIN_TASK_STACK_SIZE + 1000];
     for (int i = 0; i < sizeof(stuff); ++i) {
         stuff[i] = rand();
     }

tools/test_apps/system/panic/panic_tests.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+import re
+from test_panic_util.test_panic_util import get_dut
+
+
+def test_common(dut, test_name):
+    if "uart" in test_name:
+        dut.expect(dut.COREDUMP_UART_END)
+    dut.expect("Rebooting...")
+    if "uart" in test_name:
+        dut.process_coredump_uart
+    elif "flash" in test_name:
+        dut.process_coredump_flash
+
+
+def task_wdt_inner(env, test_name):
+    with get_dut(env, test_name, "test_task_wdt", qemu_wdt_enable=True) as dut:
+        dut.expect("Task watchdog got triggered. The following tasks did not reset the watchdog in time:")
+        dut.expect("CPU 0: main")
+        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
+        dut.expect_none("register dump:")
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        test_common(dut, test_name)
+
+
+def int_wdt_inner(env, test_name):
+    with get_dut(env, test_name, "test_int_wdt", qemu_wdt_enable=True) as dut:
+        dut.expect_gme("Interrupt wdt timeout on CPU0")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect_reg_dump(1)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        test_common(dut, test_name)
+
+
+def int_wdt_cache_disabled_inner(env, test_name):
+    with get_dut(env, test_name, "test_int_wdt_cache_disabled", qemu_wdt_enable=True) as dut:
+        dut.expect("Re-enable cpu cache.")
+        dut.expect_gme("Interrupt wdt timeout on CPU0")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect_reg_dump(1)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        test_common(dut, test_name)
+
+
+def cache_error_inner(env, test_name):
+    with get_dut(env, test_name, "test_cache_error") as dut:
+        dut.expect("Re-enable cpu cache.")
+        dut.expect_gme("Cache disabled but cached memory region accessed")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        test_common(dut, test_name)
+
+
+def abort_inner(env, test_name):
+    with get_dut(env, test_name, "test_abort") as dut:
+        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation", "Re-entered core dump")
+        test_common(dut, test_name)
+
+
+def storeprohibited_inner(env, test_name):
+    with get_dut(env, test_name, "test_storeprohibited") as dut:
+        dut.expect_gme("StoreProhibited")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        test_common(dut, test_name)
+
+
+def stack_overflow_inner(env, test_name):
+    with get_dut(env, test_name, "test_stack_overflow") as dut:
+        dut.expect_gme("Unhandled debug exception")
+        dut.expect("Stack canary watchpoint triggered (main)")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        test_common(dut, test_name)
+
+
+def illegal_instruction_inner(env, test_name):
+    with get_dut(env, test_name, "test_illegal_instruction") as dut:
+        dut.expect_gme("IllegalInstruction")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        test_common(dut, test_name)
+
+
+def instr_fetch_prohibited_inner(env, test_name):
+    with get_dut(env, test_name, "test_instr_fetch_prohibited") as dut:
+        dut.expect_gme("InstrFetchProhibited")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        # At the moment the backtrace is corrupted, need to jump over the first PC in case of InstrFetchProhibited.
+        # Fix this and change expect to expect_none.
+        dut.expect("CORRUPTED")
+        dut.expect_elf_sha256()
+        dut.expect_none("Guru Meditation")
+        test_common(dut, test_name)

tools/test_apps/system/panic/sdkconfig.ci.coredump_flash

@@ -1 +0,0 @@
-CONFIG_ESP32_ENABLE_COREDUMP_TO_FLASH=y

tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_bin_crc

@@ -0,0 +1,3 @@
+CONFIG_ESP32_ENABLE_COREDUMP_TO_FLASH=y
+CONFIG_ESP32_COREDUMP_DATA_FORMAT_BIN=y
+CONFIG_ESP32_COREDUMP_CHECKSUM_CRC32=y

tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_elf_sha

@@ -0,0 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
+CONFIG_ESP32_ENABLE_COREDUMP_TO_FLASH=y
+CONFIG_ESP32_COREDUMP_DATA_FORMAT_ELF=y
+CONFIG_ESP32_COREDUMP_CHECKSUM_SHA256=y

tools/test_apps/system/panic/sdkconfig.ci.coredump_uart

@@ -1 +0,0 @@
-CONFIG_ESP32_ENABLE_COREDUMP_TO_UART=y

tools/test_apps/system/panic/sdkconfig.ci.coredump_uart_bin_crc

@@ -0,0 +1,3 @@
+CONFIG_ESP32_ENABLE_COREDUMP_TO_UART=y
+CONFIG_ESP32_COREDUMP_DATA_FORMAT_BIN=y
+CONFIG_ESP32_COREDUMP_CHECKSUM_CRC32=y

tools/test_apps/system/panic/sdkconfig.ci.coredump_uart_elf_crc

@@ -0,0 +1,3 @@
+CONFIG_ESP32_ENABLE_COREDUMP_TO_UART=y
+CONFIG_ESP32_COREDUMP_DATA_FORMAT_ELF=y
+CONFIG_ESP32_COREDUMP_CHECKSUM_CRC32=y