From b1d64d1a617b60db620dcc2d6197da3c371cd5d6 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 30 Sep 2020 22:41:14 +0200 Subject: [PATCH] test/panic: add gdbstub test configuration --- tools/test_apps/system/panic/app_test.py | 135 ++++++++++++------ tools/test_apps/system/panic/panic_tests.py | 50 ++++++- .../system/panic/sdkconfig.ci.gdbstub | 1 + .../panic/test_panic_util/test_panic_util.py | 102 ++++++++++++- 4 files changed, 232 insertions(+), 56 deletions(-) create mode 100644 tools/test_apps/system/panic/sdkconfig.ci.gdbstub diff --git a/tools/test_apps/system/panic/app_test.py b/tools/test_apps/system/panic/app_test.py index 231f42daf8..15ce730a3d 100644 --- a/tools/test_apps/system/panic/app_test.py +++ b/tools/test_apps/system/panic/app_test.py @@ -7,245 +7,290 @@ from test_panic_util.test_panic_util import panic_test, run_all # test_task_wdt @panic_test() -def test_panic_task_wdt(env, extra_data): +def test_panic_task_wdt(env, _extra_data): test.task_wdt_inner(env, "panic") @panic_test() -def test_coredump_task_wdt_uart_elf_crc(env, extra_data): +def test_coredump_task_wdt_uart_elf_crc(env, _extra_data): test.task_wdt_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_task_wdt_uart_bin_crc(env, extra_data): +def test_coredump_task_wdt_uart_bin_crc(env, _extra_data): test.task_wdt_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_task_wdt_flash_elf_sha(env, extra_data): +def test_coredump_task_wdt_flash_elf_sha(env, _extra_data): test.task_wdt_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_task_wdt_flash_bin_crc(env, extra_data): +def test_coredump_task_wdt_flash_bin_crc(env, _extra_data): test.task_wdt_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_task_wdt(env, _extra_data): + test.task_wdt_inner(env, "gdbstub") + + # test_int_wdt @panic_test() -def test_panic_int_wdt(env, extra_data): +def test_panic_int_wdt(env, _extra_data): test.int_wdt_inner(env, "panic") @panic_test() -def test_coredump_int_wdt_uart_elf_crc(env, extra_data): +def test_coredump_int_wdt_uart_elf_crc(env, _extra_data): test.int_wdt_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_int_wdt_uart_bin_crc(env, extra_data): +def test_coredump_int_wdt_uart_bin_crc(env, _extra_data): test.int_wdt_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_int_wdt_flash_elf_sha(env, extra_data): +def test_coredump_int_wdt_flash_elf_sha(env, _extra_data): test.int_wdt_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_int_wdt_flash_bin_crc(env, extra_data): +def test_coredump_int_wdt_flash_bin_crc(env, _extra_data): test.int_wdt_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_int_wdt(env, _extra_data): + test.int_wdt_inner(env, "gdbstub") + + # test_int_wdt_cache_disabled @panic_test() -def test_panic_int_wdt_cache_disabled(env, extra_data): +def test_panic_int_wdt_cache_disabled(env, _extra_data): test.int_wdt_cache_disabled_inner(env, "panic") @panic_test() -def test_coredump_int_wdt_cache_disabled_uart_elf_crc(env, extra_data): +def test_coredump_int_wdt_cache_disabled_uart_elf_crc(env, _extra_data): test.int_wdt_cache_disabled_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_int_wdt_cache_disabled_uart_bin_crc(env, extra_data): +def test_coredump_int_wdt_cache_disabled_uart_bin_crc(env, _extra_data): test.int_wdt_cache_disabled_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_int_wdt_cache_disabled_flash_elf_sha(env, extra_data): +def test_coredump_int_wdt_cache_disabled_flash_elf_sha(env, _extra_data): test.int_wdt_cache_disabled_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_int_wdt_cache_disabled_flash_bin_crc(env, extra_data): +def test_coredump_int_wdt_cache_disabled_flash_bin_crc(env, _extra_data): test.int_wdt_cache_disabled_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_int_wdt_cache_disabled(env, _extra_data): + test.int_wdt_cache_disabled_inner(env, "gdbstub") + + # test_cache_error @panic_test() -def test_panic_cache_error(env, extra_data): +def test_panic_cache_error(env, _extra_data): test.cache_error_inner(env, "panic") @panic_test() -def test_coredump_cache_error_uart_elf_crc(env, extra_data): +def test_coredump_cache_error_uart_elf_crc(env, _extra_data): test.cache_error_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_cache_error_uart_bin_crc(env, extra_data): +def test_coredump_cache_error_uart_bin_crc(env, _extra_data): test.cache_error_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_cache_error_flash_elf_sha(env, extra_data): +def test_coredump_cache_error_flash_elf_sha(env, _extra_data): test.cache_error_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_cache_error_flash_bin_crc(env, extra_data): +def test_coredump_cache_error_flash_bin_crc(env, _extra_data): test.cache_error_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_cache_error(env, _extra_data): + test.cache_error_inner(env, "gdbstub") + + # test_stack_overflow @panic_test() -def test_panic_stack_overflow(env, extra_data): +def test_panic_stack_overflow(env, _extra_data): test.stack_overflow_inner(env, "panic") @panic_test() -def test_coredump_stack_overflow_uart_elf_crc(env, extra_data): +def test_coredump_stack_overflow_uart_elf_crc(env, _extra_data): test.stack_overflow_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_stack_overflow_uart_bin_crc(env, extra_data): +def test_coredump_stack_overflow_uart_bin_crc(env, _extra_data): test.stack_overflow_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_stack_overflow_flash_elf_sha(env, extra_data): +def test_coredump_stack_overflow_flash_elf_sha(env, _extra_data): test.stack_overflow_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_stack_overflow_flash_bin_crc(env, extra_data): +def test_coredump_stack_overflow_flash_bin_crc(env, _extra_data): test.stack_overflow_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_stack_overflow(env, _extra_data): + test.stack_overflow_inner(env, "gdbstub") + + # test_instr_fetch_prohibited @panic_test() -def test_panic_instr_fetch_prohibited(env, extra_data): +def test_panic_instr_fetch_prohibited(env, _extra_data): test.instr_fetch_prohibited_inner(env, "panic") @panic_test() -def test_coredump_instr_fetch_prohibited_uart_elf_crc(env, extra_data): +def test_coredump_instr_fetch_prohibited_uart_elf_crc(env, _extra_data): test.instr_fetch_prohibited_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_instr_fetch_prohibited_uart_bin_crc(env, extra_data): +def test_coredump_instr_fetch_prohibited_uart_bin_crc(env, _extra_data): test.instr_fetch_prohibited_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_instr_fetch_prohibited_flash_elf_sha(env, extra_data): +def test_coredump_instr_fetch_prohibited_flash_elf_sha(env, _extra_data): test.instr_fetch_prohibited_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_instr_fetch_prohibited_flash_bin_crc(env, extra_data): +def test_coredump_instr_fetch_prohibited_flash_bin_crc(env, _extra_data): test.instr_fetch_prohibited_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_instr_fetch_prohibited(env, _extra_data): + test.instr_fetch_prohibited_inner(env, "gdbstub") + + # test_illegal_instruction @panic_test() -def test_panic_illegal_instruction(env, extra_data): +def test_panic_illegal_instruction(env, _extra_data): test.illegal_instruction_inner(env, "panic") @panic_test() -def test_coredump_illegal_instruction_uart_elf_crc(env, extra_data): +def test_coredump_illegal_instruction_uart_elf_crc(env, _extra_data): test.illegal_instruction_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_illegal_instruction_uart_bin_crc(env, extra_data): +def test_coredump_illegal_instruction_uart_bin_crc(env, _extra_data): test.illegal_instruction_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_illegal_instruction_flash_elf_sha(env, extra_data): +def test_coredump_illegal_instruction_flash_elf_sha(env, _extra_data): test.illegal_instruction_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_illegal_instruction_flash_bin_crc(env, extra_data): +def test_coredump_illegal_instruction_flash_bin_crc(env, _extra_data): test.illegal_instruction_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_illegal_instruction(env, _extra_data): + test.illegal_instruction_inner(env, "gdbstub") + + # test_storeprohibited @panic_test() -def test_panic_storeprohibited(env, extra_data): +def test_panic_storeprohibited(env, _extra_data): test.storeprohibited_inner(env, "panic") @panic_test() -def test_coredump_storeprohibited_uart_elf_crc(env, extra_data): +def test_coredump_storeprohibited_uart_elf_crc(env, _extra_data): test.storeprohibited_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_storeprohibited_uart_bin_crc(env, extra_data): +def test_coredump_storeprohibited_uart_bin_crc(env, _extra_data): test.storeprohibited_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_storeprohibited_flash_elf_sha(env, extra_data): +def test_coredump_storeprohibited_flash_elf_sha(env, _extra_data): test.storeprohibited_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_storeprohibited_flash_bin_crc(env, extra_data): +def test_coredump_storeprohibited_flash_bin_crc(env, _extra_data): test.storeprohibited_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_storeprohibited(env, _extra_data): + test.storeprohibited_inner(env, "gdbstub") + + # test_abort @panic_test() -def test_panic_abort(env, extra_data): +def test_panic_abort(env, _extra_data): test.abort_inner(env, "panic") @panic_test() -def test_coredump_abort_uart_elf_crc(env, extra_data): +def test_coredump_abort_uart_elf_crc(env, _extra_data): test.abort_inner(env, "coredump_uart_elf_crc") @panic_test() -def test_coredump_abort_uart_bin_crc(env, extra_data): +def test_coredump_abort_uart_bin_crc(env, _extra_data): test.abort_inner(env, "coredump_uart_bin_crc") @panic_test() -def test_coredump_abort_flash_elf_sha(env, extra_data): +def test_coredump_abort_flash_elf_sha(env, _extra_data): test.abort_inner(env, "coredump_flash_elf_sha") @panic_test() -def test_coredump_abort_flash_bin_crc(env, extra_data): +def test_coredump_abort_flash_bin_crc(env, _extra_data): test.abort_inner(env, "coredump_flash_bin_crc") +@panic_test() +def test_gdbstub_abort(env, _extra_data): + test.abort_inner(env, "gdbstub") + + if __name__ == '__main__': run_all(__file__, sys.argv[1:]) diff --git a/tools/test_apps/system/panic/panic_tests.py b/tools/test_apps/system/panic/panic_tests.py index 6fb93f13b8..efb629391f 100644 --- a/tools/test_apps/system/panic/panic_tests.py +++ b/tools/test_apps/system/panic/panic_tests.py @@ -1,16 +1,43 @@ #!/usr/bin/env python +from pprint import pformat import re from test_panic_util.test_panic_util import get_dut -def test_common(dut, test_name): +def get_default_backtrace(test_name): + return [ + test_name, + "app_main", + "main_task", + "vPortTaskWrapper" + ] + + +def test_common(dut, test_name, expected_backtrace=None): + if expected_backtrace is None: + expected_backtrace = get_default_backtrace(dut.test_name) + + if "gdbstub" in test_name: + dut.start_gdb() + frames = dut.gdb_backtrace() + if not dut.match_backtrace(frames, expected_backtrace): + raise AssertionError("Unexpected backtrace in test {}:\n{}".format(test_name, pformat(frames))) + return + if "uart" in test_name: dut.expect(dut.COREDUMP_UART_END) + dut.expect("Rebooting...") + if "uart" in test_name: - dut.process_coredump_uart + dut.process_coredump_uart() + # TODO: check backtrace elif "flash" in test_name: - dut.process_coredump_flash + dut.process_coredump_flash() + # TODO: check backtrace + elif "panic" in test_name: + # TODO: check backtrace + pass def task_wdt_inner(env, test_name): @@ -22,7 +49,11 @@ def task_wdt_inner(env, test_name): dut.expect_backtrace() dut.expect_elf_sha256() dut.expect_none("Guru Meditation") - test_common(dut, test_name) + test_common(dut, test_name, expected_backtrace=[ + # Backtrace interrupted when abort is called, IDF-842. + # Task WDT calls abort internally. + "panic_abort", "esp_system_abort" + ]) def int_wdt_inner(env, test_name): @@ -60,7 +91,8 @@ def cache_error_inner(env, test_name): dut.expect_backtrace() dut.expect_elf_sha256() dut.expect_none("Guru Meditation") - test_common(dut, test_name) + test_common(dut, test_name, + expected_backtrace=["die"] + get_default_backtrace(dut.test_name)) def abort_inner(env, test_name): @@ -69,7 +101,10 @@ def abort_inner(env, test_name): dut.expect_backtrace() dut.expect_elf_sha256() dut.expect_none("Guru Meditation", "Re-entered core dump") - test_common(dut, test_name) + test_common(dut, test_name, expected_backtrace=[ + # Backtrace interrupted when abort is called, IDF-842 + "panic_abort", "esp_system_abort" + ]) def storeprohibited_inner(env, test_name): @@ -110,4 +145,5 @@ def instr_fetch_prohibited_inner(env, test_name): dut.expect_backtrace() dut.expect_elf_sha256() dut.expect_none("Guru Meditation") - test_common(dut, test_name) + test_common(dut, test_name, + expected_backtrace=["_init"] + get_default_backtrace(dut.test_name)) diff --git a/tools/test_apps/system/panic/sdkconfig.ci.gdbstub b/tools/test_apps/system/panic/sdkconfig.ci.gdbstub new file mode 100644 index 0000000000..38830f8dd6 --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.ci.gdbstub @@ -0,0 +1 @@ +CONFIG_ESP_SYSTEM_PANIC_GDBSTUB=y diff --git a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py index 6229f6c402..f2de09a327 100644 --- a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py +++ b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py @@ -1,7 +1,9 @@ +import logging import os -import sys +from pygdbmi.gdbcontroller import GdbController import re import subprocess +import sys import ttfw_idf from tiny_test_fw import Utility, TinyFW, DUT from tiny_test_fw.Utility import SearchCases, CaseConfig @@ -82,6 +84,7 @@ class PanicTestMixin(object): def __enter__(self): self._raw_data = None + self.gdb = None return self def __exit__(self, type, value, traceback): @@ -89,7 +92,8 @@ class PanicTestMixin(object): with open(os.path.join(log_folder, "log_" + self.test_name + ".txt"), "w") as log_file: Utility.console_log("Writing output of {} to {}".format(self.test_name, log_file.name)) log_file.write(self.get_raw_data()) - + if self.gdb: + self.gdb.exit() self.close() def get_raw_data(self): @@ -140,6 +144,90 @@ class PanicTestMixin(object): output_file_name = os.path.join(log_folder, "coredump_flash_result_" + self.test_name + ".txt") self._call_espcoredump(["--core-format", "raw"], coredump_file_name, output_file_name) + def start_gdb(self): + """ + Runs GDB and connects it to the "serial" port of the DUT. + After this, the DUT expect methods can no longer be used to capture output. + """ + self.stop_receive() + self._port_close() + + Utility.console_log("Starting GDB...", "orange") + self.gdb = GdbController(gdb_path=self.TOOLCHAIN_PREFIX + "gdb") + + # pygdbmi logs to console by default, make it log to a file instead + log_folder = self.app.get_log_folder(TEST_SUITE) + pygdbmi_log_file_name = os.path.join(log_folder, "pygdbmi_log_" + self.test_name + ".txt") + pygdbmi_logger = self.gdb.logger + pygdbmi_logger.setLevel(logging.DEBUG) + while pygdbmi_logger.hasHandlers(): + pygdbmi_logger.removeHandler(pygdbmi_logger.handlers[0]) + log_handler = logging.FileHandler(pygdbmi_log_file_name) + log_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s")) + pygdbmi_logger.addHandler(log_handler) + + # Set up logging for GDB remote protocol + gdb_remotelog_file_name = os.path.join(log_folder, "gdb_remote_log_" + self.test_name + ".txt") + self.gdb.write("-gdb-set remotelogfile " + gdb_remotelog_file_name) + + # Load the ELF file + self.gdb.write("-file-exec-and-symbols {}".format(self.app.elf_file)) + + # Connect GDB to UART + Utility.console_log("Connecting to GDB Stub...", "orange") + self.gdb.write("-gdb-set serial baud 115200") + responses = self.gdb.write("-target-select remote " + self.get_gdb_remote(), timeout_sec=3) + + # Make sure we get the 'stopped' notification + stop_response = self.find_gdb_response('stopped', 'notify', responses) + if not stop_response: + responses = self.gdb.write("-exec-interrupt", timeout_sec=3) + stop_response = self.find_gdb_response('stopped', 'notify', responses) + assert stop_response + frame = stop_response["payload"]["frame"] + if "file" not in frame: + frame["file"] = "?" + if "line" not in frame: + frame["line"] = "?" + Utility.console_log("Stopped in {func} at {addr} ({file}:{line})".format(**frame), "orange") + + # Drain remaining responses + self.gdb.get_gdb_response(raise_error_on_timeout=False) + + def gdb_backtrace(self): + """ + Returns the list of stack frames for the current thread. + Each frame is a dictionary, refer to pygdbmi docs for the format. + """ + assert self.gdb + + responses = self.gdb.write("-stack-list-frames", timeout_sec=3) + return self.find_gdb_response("done", "result", responses)["payload"]["stack"] + + @staticmethod + def match_backtrace(gdb_backtrace, expected_functions_list): + """ + Returns True if the function names listed in expected_functions_list match the backtrace + given by gdb_backtrace argument. The latter is in the same format as returned by gdb_backtrace() + function. + """ + return all([frame["func"] == expected_functions_list[i] for i, frame in enumerate(gdb_backtrace)]) + + @staticmethod + def find_gdb_response(message, response_type, responses): + """ + Helper function which extracts one response from an array of GDB responses, filtering + by message and type. Returned message is a dictionary, refer to pygdbmi docs for the format. + """ + def match_response(response): + return (response["message"] == message and + response["type"] == response_type) + + filtered_responses = [r for r in responses if match_response(r)] + if not filtered_responses: + return None + return filtered_responses[0] + class ESP32PanicTestDUT(ttfw_idf.ESP32DUT, PanicTestMixin): def get_gdb_remote(self): @@ -170,8 +258,14 @@ def run_all(filename, case_filter=[]): test_cases = CaseConfig.Parser.apply_config(test_methods, None) tests_failed = [] for case in test_cases: - if case_filter and case.test_method.__name__ not in case_filter: - continue + test_name = case.test_method.__name__ + if case_filter: + if case_filter[0].endswith("*"): + if not test_name.startswith(case_filter[0][:-1]): + continue + else: + if test_name not in case_filter: + continue result = case.run() if not result: tests_failed.append(case)