Merge branch 'feature/test_panic' into 'master'

ci: panic handling test app

See merge request espressif/esp-idf!8593

tools/ci/config/build.yml

@@ -272,6 +272,12 @@ build_docs_zh_CN_esp32s2:
 
 test_build_system:
   extends: .build_template
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - rm -rf test_build_system
@@ -281,6 +287,12 @@ test_build_system:
 
 test_build_system_cmake:
   extends: .build_template
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - rm -rf test_build_system

tools/ci/python_packages/tiny_test_fw/DUT.py

@@ -751,7 +751,9 @@ class SerialDUT(BaseDUT):
     def __init__(self, name, port, log_file, app, **kwargs):
         self.port_inst = None
         self.serial_configs = self.DEFAULT_UART_CONFIG.copy()
-        self.serial_configs.update(kwargs)
+        for uart_config_name in self.serial_configs.keys():
+            if uart_config_name in kwargs:
+                self.serial_configs[uart_config_name] = kwargs[uart_config_name]
         super(SerialDUT, self).__init__(name, port, log_file, app, **kwargs)
 
     def _format_data(self, data):

tools/ci/python_packages/tiny_test_fw/Utility/__init__.py

@@ -61,11 +61,15 @@ def load_source(path):
         return __LOADED_MODULES[path]
     except KeyError:
         try:
+            dir = os.path.dirname(path)
+            sys.path.append(dir)
             from importlib.machinery import SourceFileLoader
             ret = SourceFileLoader(load_name, path).load_module()
         except ImportError:
             # importlib.machinery doesn't exists in Python 2 so we will use imp (deprecated in Python 3)
             import imp
             ret = imp.load_source(load_name, path)
+        finally:
+            sys.path.remove(dir)
         __LOADED_MODULES[path] = ret
         return ret

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -14,7 +14,7 @@
 
 """ IDF Test Applications """
 import subprocess
-
+import hashlib
 import json
 import os
 import sys
@@ -151,6 +151,7 @@ class IDFApp(App.BaseApp):
         self.idf_path = self.get_sdk_path()
         self.binary_path = self.get_binary_path(app_path, config_name, target)
         self.elf_file = self._get_elf_file_path(self.binary_path)
+        self._elf_file_sha256 = None
         assert os.path.exists(self.binary_path)
         if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path):
             if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path):
@@ -307,6 +308,7 @@ class IDFApp(App.BaseApp):
                         _size = int(_size[:-1]) * 1024 * 1024
                     else:
                         _size = int(_size)
+                    _offset = int(_offset, 0)
                 except ValueError:
                     continue
                 partition_table[_name] = {
@@ -319,6 +321,16 @@ class IDFApp(App.BaseApp):
 
         return partition_table
 
+    def get_elf_sha256(self):
+        if self._elf_file_sha256:
+            return self._elf_file_sha256
+
+        sha256 = hashlib.sha256()
+        with open(self.elf_file, 'rb') as f:
+            sha256.update(f.read())
+        self._elf_file_sha256 = sha256.hexdigest()
+        return self._elf_file_sha256
+
 
 class Example(IDFApp):
     def _get_sdkconfig_paths(self):

tools/ci/python_packages/ttfw_idf/IDFDUT.py

@@ -226,7 +226,9 @@ class IDFDUT(DUT.SerialDUT):
                 nvs_file = tempfile.TemporaryFile()
                 nvs_file.write(b'\xff' * size)
                 nvs_file.seek(0)
-                flash_files.append((int(address, 0), nvs_file))
+                if not isinstance(address, int):
+                    address = int(address, 0)
+                flash_files.append((address, nvs_file))
 
             # fake flasher args object, this is a hack until
             # esptool Python API is improved

tools/test_apps/system/panic/CMakeLists.txt

@@ -0,0 +1,6 @@
+# The following lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(test_panic)

tools/test_apps/system/panic/app_test.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+import re
+from test_panic_util.test_panic_util import panic_test, get_dut, run_all
+
+
+@panic_test()
+def test_panic_task_wdt(env, extra_data):
+    with get_dut(env, "panic", "test_task_wdt", qemu_wdt_enable=True) as dut:
+        dut.expect("Task watchdog got triggered. The following tasks did not reset the watchdog in time:")
+        dut.expect("CPU 0: main")
+        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
+        dut.expect_none("register dump:")
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_panic_int_wdt(env, extra_data):
+    with get_dut(env, "panic", "test_int_wdt", qemu_wdt_enable=True) as dut:
+        dut.expect_gme("Interrupt wdt timeout on CPU0")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect_reg_dump(1)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_cache_error(env, extra_data):
+    with get_dut(env, "panic", "test_cache_error") as dut:
+        dut.expect("Re-enable cpu cache.")
+        dut.expect_gme("Cache disabled but cached memory region accessed")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_panic_int_wdt_cache_disabled(env, extra_data):
+    with get_dut(env, "panic", "test_int_wdt_cache_disabled", qemu_wdt_enable=True) as dut:
+        dut.expect("Re-enable cpu cache.")
+        dut.expect_gme("Interrupt wdt timeout on CPU0")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect_reg_dump(1)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_panic_abort(env, extra_data):
+    with get_dut(env, "panic", "test_abort") as dut:
+        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
+        dut.expect_none("register dump:")
+        dut.expect("Backtrace:")
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_panic_storeprohibited(env, extra_data):
+    with get_dut(env, "panic", "test_storeprohibited") as dut:
+        dut.expect_gme("StoreProhibited")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_panic_stack_overflow(env, extra_data):
+    with get_dut(env, "panic", "test_stack_overflow") as dut:
+        dut.expect_gme("Unhandled debug exception")
+        dut.expect("Stack canary watchpoint triggered (main)")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_panic_illegal_instruction(env, extra_data):
+    with get_dut(env, "panic", "test_illegal_instruction") as dut:
+        dut.expect_gme("IllegalInstruction")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_panic_instr_fetch_prohibited(env, extra_data):
+    with get_dut(env, "panic", "test_instr_fetch_prohibited") as dut:
+        dut.expect_gme("InstrFetchProhibited")
+        dut.expect_reg_dump(0)
+        dut.expect("Backtrace:")
+        # At the moment the backtrace is corrupted, need to jump over the first PC in case of InstrFetchProhibited.
+        # Fix this and change expect to expect_none.
+        dut.expect("CORRUPTED")
+        dut.expect_elf_sha256()
+        dut.expect_none("Guru Meditation")
+        dut.expect("Rebooting...")
+
+
+@panic_test()
+def test_coredump_uart_abort(env, extra_data):
+    with get_dut(env, "coredump_uart", "test_abort") as dut:
+        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation", "Re-entered core dump")
+        dut.expect(dut.COREDUMP_UART_END)
+        dut.expect("Rebooting...")
+        dut.process_coredump_uart()
+        # TODO: check the contents of core dump output
+
+
+@panic_test()
+def test_coredump_flash_abort(env, extra_data):
+    with get_dut(env, "coredump_flash", "test_abort") as dut:
+        dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0"))
+        dut.expect("Backtrace:")
+        dut.expect_elf_sha256()
+        dut.expect_none("CORRUPTED", "Guru Meditation", "Re-entered core dump")
+        dut.expect("Rebooting...")
+        dut.process_coredump_flash()
+        # TODO: check the contents of core dump output
+
+
+if __name__ == '__main__':
+    run_all(__file__)

tools/test_apps/system/panic/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "test_panic_main.c"
+                       INCLUDE_DIRS ".")

tools/test_apps/system/panic/main/test_panic_main.c

@@ -0,0 +1,167 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "esp_partition.h"
+#include "esp_flash.h"
+#include "esp_system.h"
+
+/* utility functions */
+static void die(const char* msg) __attribute__ ((noreturn));
+static const char* get_test_name();
+
+/* functions which cause an exception/panic in different ways */
+static void test_abort();
+static void test_int_wdt();
+static void test_task_wdt();
+static void test_storeprohibited();
+static void test_cache_error();
+static void test_int_wdt_cache_disabled();
+static void test_stack_overflow();
+static void test_illegal_instruction();
+static void test_instr_fetch_prohibited();
+
+
+void app_main(void)
+{
+    /* Needed to allow the tick hook to set correct INT WDT timeouts */
+    vTaskDelay(2);
+
+    /* Test script sends to command over UART. Read it and determine how to proceed. */
+    const char* test_name = get_test_name();
+    if (test_name == NULL) {
+        /* Nothing to do */
+        return;
+    }
+    printf("Got test name: %s\n", test_name);
+
+    #define HANDLE_TEST(name_) \
+        if (strcmp(test_name, #name_) == 0) { \
+            name_(); \
+            die("Test function has returned"); \
+        }
+
+    HANDLE_TEST(test_abort);
+    HANDLE_TEST(test_int_wdt);
+    HANDLE_TEST(test_task_wdt);
+    HANDLE_TEST(test_storeprohibited);
+    HANDLE_TEST(test_cache_error);
+    HANDLE_TEST(test_int_wdt_cache_disabled);
+    HANDLE_TEST(test_stack_overflow);
+    HANDLE_TEST(test_illegal_instruction);
+    HANDLE_TEST(test_instr_fetch_prohibited);
+
+    #undef HANDLE_TEST
+
+    die("Unknown test name");
+}
+
+/* implementations of the test functions */
+
+static void test_abort()
+{
+    abort();
+}
+
+static void test_int_wdt()
+{
+    portDISABLE_INTERRUPTS();
+    while (true) {
+        ;
+    }
+}
+
+static void test_task_wdt()
+{
+    while (true) {
+        ;
+    }
+}
+
+static void test_storeprohibited()
+{
+    *(int*) 0x1 = 0;
+}
+
+static IRAM_ATTR void test_cache_error()
+{
+    esp_flash_default_chip->os_func->start(esp_flash_default_chip->os_func_data);
+    die("this should not be printed");
+}
+
+static void IRAM_ATTR test_int_wdt_cache_disabled()
+{
+    esp_flash_default_chip->os_func->start(esp_flash_default_chip->os_func_data);
+    portDISABLE_INTERRUPTS();
+    while (true) {
+        ;
+    }
+}
+
+static void test_stack_overflow()
+{
+    volatile uint8_t stuff[CONFIG_ESP_MAIN_TASK_STACK_SIZE * 2];
+    for (int i = 0; i < sizeof(stuff); ++i) {
+        stuff[i] = rand();
+    }
+}
+
+static void test_illegal_instruction()
+{
+    __asm__ __volatile__("ill");
+}
+
+static void test_instr_fetch_prohibited()
+{
+    typedef void (*fptr_t)(void);
+    volatile fptr_t fptr = (fptr_t) 0x4;
+    fptr();
+}
+
+/* implementations of the utility functions */
+
+#define BOOT_CMD_MAX_LEN (128)
+
+static const char* get_test_name()
+{
+    static char test_name_str[BOOT_CMD_MAX_LEN] = {0};
+
+    printf("Enter test name: ");
+    fflush(stdout);
+
+    /* Not using blocking fgets(stdin) here, as QEMU doesn't yet implement RX timeout interrupt,
+     * which is required for the UART driver and blocking stdio to work.
+     */
+    int c = EOF;
+    char *p = test_name_str;
+    const char *end = test_name_str + sizeof(test_name_str) - 1;
+    while (p < end) {
+        c = getchar();
+        if (c == EOF) {
+            vTaskDelay(pdMS_TO_TICKS(10));
+        } else if (c == '\r') {
+            continue;
+        } else if (c == '\n') {
+            *p = '\0';
+            break;
+        } else {
+            *p = c;
+            ++p;
+        }
+    }
+
+    return test_name_str;
+}
+
+extern void esp_restart_noos(void) __attribute__ ((noreturn));
+
+static void die(const char* msg)
+{
+    printf("Test error: %s\n\n", msg);
+    fflush(stdout);
+    fsync(fileno(stdout));
+    /* Don't use abort here as it would enter the panic handler */
+    esp_restart_noos();
+}

tools/test_apps/system/panic/sdkconfig.ci.coredump_flash

@@ -0,0 +1 @@
+CONFIG_ESP32_ENABLE_COREDUMP_TO_FLASH=y

tools/test_apps/system/panic/sdkconfig.ci.coredump_uart

@@ -0,0 +1 @@
+CONFIG_ESP32_ENABLE_COREDUMP_TO_UART=y

tools/test_apps/system/panic/sdkconfig.defaults

@@ -0,0 +1,12 @@
+# Flash DOUT mode (QEMU limitation)
+CONFIG_ESPTOOLPY_FLASHMODE_DOUT=y
+
+# Less noisy output
+CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y
+CONFIG_LOG_DEFAULT_LEVEL_WARN=y
+
+# To check for stack overflows
+CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y
+
+# To panic on task WDT
+CONFIG_ESP_TASK_WDT_PANIC=y

tools/test_apps/system/panic/test_panic_util/test_panic_util.py

@@ -0,0 +1,176 @@
+import os
+import sys
+import re
+import subprocess
+import ttfw_idf
+from tiny_test_fw import Utility, TinyFW, DUT
+from tiny_test_fw.Utility import SearchCases, CaseConfig
+
+
+# hard-coded to the path one level above - only intended to be used from the panic test app
+TEST_PATH = os.path.relpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."), os.getenv("IDF_PATH"))
+TEST_SUITE = "Panic"
+
+
+def ok(data):
+    """ Helper function used with dut.expect_any """
+    pass
+
+
+def unexpected(data):
+    """ Helper function used with dut.expect_any """
+    raise AssertionError("Unexpected: {}".format(data))
+
+
+class PanicTestApp(ttfw_idf.TestApp):
+    pass
+
+
+class PanicTestMixin(object):
+    """ Provides custom functionality for the panic test DUT """
+    BOOT_CMD_ADDR = 0x9000
+    BOOT_CMD_SIZE = 0x1000
+    DEFAULT_EXPECT_TIMEOUT = 10
+    COREDUMP_UART_START = "================= CORE DUMP START ================="
+    COREDUMP_UART_END = "================= CORE DUMP END ================="
+
+    def start_test(self, test_name):
+        """ Starts the app and sends it the test name """
+        self.test_name = test_name
+        # Start the app and verify that it has started up correctly
+        self.start_capture_raw_data()
+        self.start_app()
+        self.expect("Enter test name: ")
+        Utility.console_log("Setting boot command: " + test_name)
+        self.write(test_name)
+        self.expect("Got test name: " + test_name)
+
+    def expect_none(self, *patterns, **timeout_args):
+        """ like dut.expect_all, but with an inverse logic """
+        found_data = []
+        if "timeout" not in timeout_args:
+            timeout_args["timeout"] = 1
+
+        def found(data):
+            raise AssertionError("Unexpected: {}".format(data))
+            found_data.append(data)
+        try:
+            expect_items = [(pattern, found) for pattern in patterns]
+            self.expect_any(*expect_items, **timeout_args)
+            raise AssertionError("Unexpected: {}".format(found_data))
+        except DUT.ExpectTimeout:
+            return True
+
+    def expect_gme(self, reason):
+        """ Expect method for Guru Meditation Errors """
+        self.expect(r"Guru Meditation Error: Core  0 panic'ed (%s)" % reason)
+
+    def expect_reg_dump(self, core=0):
+        """ Expect method for the register dump """
+        self.expect(re.compile(r"Core\s+%d register dump:" % core))
+
+    def expect_elf_sha256(self):
+        """ Expect method for ELF SHA256 line """
+        elf_sha256 = self.app.get_elf_sha256()
+        sdkconfig = self.app.get_sdkconfig()
+        elf_sha256_len = int(sdkconfig.get("CONFIG_APP_RETRIEVE_LEN_ELF_SHA", "16"))
+        self.expect("ELF file SHA256: " + elf_sha256[0:elf_sha256_len])
+
+    def __enter__(self):
+        self._raw_data = None
+        return self
+
+    def __exit__(self, type, value, traceback):
+        log_folder = self.app.get_log_folder(TEST_SUITE)
+        with open(os.path.join(log_folder, "log_" + self.test_name + ".txt"), "w") as log_file:
+            Utility.console_log("Writing output of {} to {}".format(self.test_name, log_file.name))
+            log_file.write(self.get_raw_data())
+
+        self.close()
+
+    def get_raw_data(self):
+        if not self._raw_data:
+            self._raw_data = self.stop_capture_raw_data()
+        return self._raw_data
+
+    def _call_espcoredump(self, extra_args, coredump_file_name, output_file_name):
+        # no "with" here, since we need the file to be open for later inspection by the test case
+        self.coredump_output = open(output_file_name, "w")
+        espcoredump_script = os.path.join(os.environ["IDF_PATH"], "components", "espcoredump", "espcoredump.py")
+        espcoredump_args = [
+            sys.executable,
+            espcoredump_script,
+            "info_corefile",
+            "--core", coredump_file_name,
+        ]
+        espcoredump_args += extra_args
+        espcoredump_args.append(self.app.elf_file)
+        Utility.console_log("Running " + " ".join(espcoredump_args))
+        Utility.console_log("espcoredump output is written to " + self.coredump_output.name)
+
+        subprocess.check_call(espcoredump_args, stdout=self.coredump_output)
+        self.coredump_output.flush()
+        self.coredump_output.seek(0)
+
+    def process_coredump_uart(self):
+        """ Extract the core dump from UART output of the test, run espcoredump on it """
+        log_folder = self.app.get_log_folder(TEST_SUITE)
+        data = self.get_raw_data()
+        coredump_start = data.find(self.COREDUMP_UART_START)
+        coredump_end = data.find(self.COREDUMP_UART_END)
+        coredump_base64 = data[coredump_start + len(self.COREDUMP_UART_START):coredump_end]
+        with open(os.path.join(log_folder, "coredump_data_" + self.test_name + ".b64"), "w") as coredump_file:
+            Utility.console_log("Writing UART base64 core dump to " + coredump_file.name)
+            coredump_file.write(coredump_base64)
+
+        output_file_name = os.path.join(log_folder, "coredump_uart_result_" + self.test_name + ".txt")
+        self._call_espcoredump(["--core-format", "b64"], coredump_file.name, output_file_name)
+
+    def process_coredump_flash(self):
+        """ Extract the core dump from flash, run espcoredump on it """
+        log_folder = self.app.get_log_folder(TEST_SUITE)
+        coredump_file_name = os.path.join(log_folder, "coredump_data_" + self.test_name + ".bin")
+        Utility.console_log("Writing flash binary core dump to " + coredump_file_name)
+        self.dump_flush(coredump_file_name, partition="coredump")
+
+        output_file_name = os.path.join(log_folder, "coredump_flash_result_" + self.test_name + ".txt")
+        self._call_espcoredump(["--core-format", "raw"], coredump_file_name, output_file_name)
+
+
+class ESP32PanicTestDUT(ttfw_idf.ESP32DUT, PanicTestMixin):
+    def get_gdb_remote(self):
+        return self.port
+
+
+def panic_test(**kwargs):
+    """ Decorator for the panic tests, sets correct App and DUT classes """
+    return ttfw_idf.idf_custom_test(app=PanicTestApp, dut=ESP32PanicTestDUT, env_tag="test_jtag_arm", **kwargs)
+
+
+def get_dut(env, app_config_name, test_name, qemu_wdt_enable=False):
+    dut = env.get_dut("panic", TEST_PATH, app_config_name=app_config_name, allow_dut_exception=True)
+    dut.qemu_wdt_enable = qemu_wdt_enable
+    """ Wrapper for getting the DUT and starting the test """
+    dut.start_test(test_name)
+    return dut
+
+
+def run_all(filename):
+    """ Helper function to run all test cases defined in a file; to be called from __main__. """
+    TinyFW.set_default_config(env_config_file=None, test_suite_name=TEST_SUITE)
+    test_methods = SearchCases.Search.search_test_cases(filename)
+    test_methods = filter(lambda m: not m.case_info["ignore"], test_methods)
+    test_cases = CaseConfig.Parser.apply_config(test_methods, None)
+    tests_failed = []
+    for case in test_cases:
+        result = case.run()
+        if not result:
+            tests_failed.append(case)
+
+    if tests_failed:
+        print("The following tests have failed:")
+        for case in tests_failed:
+            print(" - " + case.test_method.__name__)
+        raise SystemExit(1)
+
+    print("Tests pass")