From 441d1306be2e1716c398963b5d0c751af8d40a57 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 31 Dec 2020 10:58:28 +1100 Subject: [PATCH 1/2] ci ttfw: Consistently handle unexpected exceptions in test cases --- tools/ci/python_packages/tiny_test_fw/DUT.py | 2 +- tools/ci/python_packages/tiny_test_fw/TinyFW.py | 6 +----- .../tiny_test_fw/Utility/__init__.py | 14 ++++++++++++++ tools/unit-test-app/unit_test.py | 7 ++++--- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tools/ci/python_packages/tiny_test_fw/DUT.py b/tools/ci/python_packages/tiny_test_fw/DUT.py index ca54269a72..f2aa809edc 100644 --- a/tools/ci/python_packages/tiny_test_fw/DUT.py +++ b/tools/ci/python_packages/tiny_test_fw/DUT.py @@ -439,7 +439,7 @@ class BaseDUT(object): if isinstance(data, type(u'')): try: data = data.encode('utf-8') - except Exception as e: + except UnicodeEncodeError as e: print(u'Cannot encode {} of type {}'.format(data, type(data))) raise e return data diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py index 6e2d03d38e..c45d678205 100644 --- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py +++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py @@ -15,7 +15,6 @@ """ Interface for test cases. """ import os import time -import traceback import functools import socket from datetime import datetime @@ -195,10 +194,7 @@ def test_method(**kwargs): # if finish without exception, test result is True result = True except Exception as e: - # handle all the exceptions here - traceback.print_exc() - # log failure - junit_test_case.add_failure_info(str(e) + ":\r\n" + traceback.format_exc()) + Utility.handle_unexpected_exception(junit_test_case, e) finally: # do close all DUTs, if result is False then print DUT debug info close_errors = env_inst.close(dut_debug=(not result)) diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py b/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py index 026674d5c5..6548869de0 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py @@ -2,6 +2,7 @@ from __future__ import print_function import os.path import sys import time +import traceback from .. import Env @@ -95,3 +96,16 @@ def load_source(path): sys.path.remove(dir) __LOADED_MODULES[path] = ret return ret + + +def handle_unexpected_exception(junit_test_case, exception): + """ + Helper to log & add junit result details for an unexpected exception encountered + when running a test case. + + Should always be called from inside an except: block + """ + traceback.print_exc() + # AssertionError caused by an 'assert' statement has an empty string as its 'str' form + e_str = str(exception) if str(exception) else repr(exception) + junit_test_case.add_failure_info("Unexpected exception: {}\n{}".format(e_str, traceback.format_exc())) diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 5ad632a77a..565ff46de0 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -24,6 +24,7 @@ import argparse import threading from tiny_test_fw import TinyFW, Utility, Env, DUT +from tiny_test_fw.Utility import handle_unexpected_exception import ttfw_idf UT_APP_BOOT_UP_DONE = "Press ENTER to see the list of tests." @@ -317,7 +318,7 @@ def run_unit_test_cases(env, extra_data): except TestCaseFailed: failed_cases.append(format_case_name(one_case)) except Exception as e: - junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + handle_unexpected_exception(junit_test_case, e) failed_cases.append(format_case_name(one_case)) finally: TinyFW.JunitReport.update_performance(performance_items) @@ -517,7 +518,7 @@ def run_multiple_devices_cases(env, extra_data): result = run_one_multiple_devices_case(duts, ut_config, env, one_case, one_case.get('app_bin'), junit_test_case) except Exception as e: - junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + handle_unexpected_exception(junit_test_case, e) finally: if result: Utility.console_log("Success: " + format_case_name(one_case), color="green") @@ -677,7 +678,7 @@ def run_multiple_stage_cases(env, extra_data): except TestCaseFailed: failed_cases.append(format_case_name(one_case)) except Exception as e: - junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + handle_unexpected_exception(junit_test_case, e) failed_cases.append(format_case_name(one_case)) finally: TinyFW.JunitReport.update_performance(performance_items) From 6d3e06a9a3a3e1ef17b6fb6b6c15083f1b3f543d Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 5 Jan 2021 10:38:39 +1100 Subject: [PATCH 2/2] ttfw: Move TestCaseFailed exception and handle differently to other exceptions But also ensure the string form of this exception is never empty, if it ends up somewhere else. --- tools/ci/python_packages/tiny_test_fw/TinyFW.py | 16 ++++++++++++++++ tools/unit-test-app/unit_test.py | 17 ++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py index c45d678205..d9e6abb10b 100644 --- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py +++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py @@ -27,6 +27,20 @@ from . import App from . import Utility +class TestCaseFailed(AssertionError): + def __init__(self, *cases): + """ + Raise this exception if one or more test cases fail in a 'normal' way (ie the test runs but fails, no unexpected exceptions) + + This will avoid dumping the Python stack trace, because the assumption is the junit error info and full job log already has + enough information for a developer to debug. + + 'cases' argument is the names of one or more test cases + """ + message = "Test case{} failed: {}".format("s" if len(cases) > 1 else "", ", ".join(str(c) for c in cases)) + super(TestCaseFailed, self).__init__(self, message) + + class DefaultEnvConfig(object): """ default test configs. There're 3 places to set configs, priority is (high -> low): @@ -193,6 +207,8 @@ def test_method(**kwargs): test_func(env_inst, extra_data) # if finish without exception, test result is True result = True + except TestCaseFailed as e: + junit_test_case.add_failure_info(str(e)) except Exception as e: Utility.handle_unexpected_exception(junit_test_case, e) finally: diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 565ff46de0..8f9c827596 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -24,6 +24,7 @@ import argparse import threading from tiny_test_fw import TinyFW, Utility, Env, DUT +from tiny_test_fw.TinyFW import TestCaseFailed from tiny_test_fw.Utility import handle_unexpected_exception import ttfw_idf @@ -72,10 +73,6 @@ def reset_reason_matches(reported_str, expected_str): return False -class TestCaseFailed(AssertionError): - pass - - def format_test_case_config(test_case_data): """ convert the test case data to unified format. @@ -222,7 +219,7 @@ def run_one_normal_case(dut, one_case, junit_test_case): else: Utility.console_log("Failed: " + format_case_name(one_case), color="red") junit_test_case.add_failure_info(output) - raise TestCaseFailed() + raise TestCaseFailed(format_case_name(one_case)) def handle_exception_reset(data): """ @@ -331,7 +328,7 @@ def run_unit_test_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) class Handler(threading.Thread): @@ -517,6 +514,8 @@ def run_multiple_devices_cases(env, extra_data): try: result = run_one_multiple_devices_case(duts, ut_config, env, one_case, one_case.get('app_bin'), junit_test_case) + except TestCaseFailed: + pass # result is False, this is handled by the finally block except Exception as e: handle_unexpected_exception(junit_test_case, e) finally: @@ -535,7 +534,7 @@ def run_multiple_devices_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) def run_one_multiple_stage_case(dut, one_case, junit_test_case): @@ -590,7 +589,7 @@ def run_one_multiple_stage_case(dut, one_case, junit_test_case): else: Utility.console_log("Failed: " + format_case_name(one_case), color="red") junit_test_case.add_failure_info(output) - raise TestCaseFailed() + raise TestCaseFailed(format_case_name(one_case)) stage_finish.append("break") def handle_exception_reset(data): @@ -691,7 +690,7 @@ def run_multiple_stage_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) def detect_update_unit_test_info(env, extra_data, app_bin):