diff --git a/tools/ci/python_packages/tiny_test_fw/DUT.py b/tools/ci/python_packages/tiny_test_fw/DUT.py index ca54269a72..f2aa809edc 100644 --- a/tools/ci/python_packages/tiny_test_fw/DUT.py +++ b/tools/ci/python_packages/tiny_test_fw/DUT.py @@ -439,7 +439,7 @@ class BaseDUT(object): if isinstance(data, type(u'')): try: data = data.encode('utf-8') - except Exception as e: + except UnicodeEncodeError as e: print(u'Cannot encode {} of type {}'.format(data, type(data))) raise e return data diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py index 6e2d03d38e..d9e6abb10b 100644 --- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py +++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py @@ -15,7 +15,6 @@ """ Interface for test cases. """ import os import time -import traceback import functools import socket from datetime import datetime @@ -28,6 +27,20 @@ from . import App from . import Utility +class TestCaseFailed(AssertionError): + def __init__(self, *cases): + """ + Raise this exception if one or more test cases fail in a 'normal' way (ie the test runs but fails, no unexpected exceptions) + + This will avoid dumping the Python stack trace, because the assumption is the junit error info and full job log already has + enough information for a developer to debug. + + 'cases' argument is the names of one or more test cases + """ + message = "Test case{} failed: {}".format("s" if len(cases) > 1 else "", ", ".join(str(c) for c in cases)) + super(TestCaseFailed, self).__init__(self, message) + + class DefaultEnvConfig(object): """ default test configs. There're 3 places to set configs, priority is (high -> low): @@ -194,11 +207,10 @@ def test_method(**kwargs): test_func(env_inst, extra_data) # if finish without exception, test result is True result = True + except TestCaseFailed as e: + junit_test_case.add_failure_info(str(e)) except Exception as e: - # handle all the exceptions here - traceback.print_exc() - # log failure - junit_test_case.add_failure_info(str(e) + ":\r\n" + traceback.format_exc()) + Utility.handle_unexpected_exception(junit_test_case, e) finally: # do close all DUTs, if result is False then print DUT debug info close_errors = env_inst.close(dut_debug=(not result)) diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py b/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py index 026674d5c5..6548869de0 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py @@ -2,6 +2,7 @@ from __future__ import print_function import os.path import sys import time +import traceback from .. import Env @@ -95,3 +96,16 @@ def load_source(path): sys.path.remove(dir) __LOADED_MODULES[path] = ret return ret + + +def handle_unexpected_exception(junit_test_case, exception): + """ + Helper to log & add junit result details for an unexpected exception encountered + when running a test case. + + Should always be called from inside an except: block + """ + traceback.print_exc() + # AssertionError caused by an 'assert' statement has an empty string as its 'str' form + e_str = str(exception) if str(exception) else repr(exception) + junit_test_case.add_failure_info("Unexpected exception: {}\n{}".format(e_str, traceback.format_exc())) diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 5ad632a77a..8f9c827596 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -24,6 +24,8 @@ import argparse import threading from tiny_test_fw import TinyFW, Utility, Env, DUT +from tiny_test_fw.TinyFW import TestCaseFailed +from tiny_test_fw.Utility import handle_unexpected_exception import ttfw_idf UT_APP_BOOT_UP_DONE = "Press ENTER to see the list of tests." @@ -71,10 +73,6 @@ def reset_reason_matches(reported_str, expected_str): return False -class TestCaseFailed(AssertionError): - pass - - def format_test_case_config(test_case_data): """ convert the test case data to unified format. @@ -221,7 +219,7 @@ def run_one_normal_case(dut, one_case, junit_test_case): else: Utility.console_log("Failed: " + format_case_name(one_case), color="red") junit_test_case.add_failure_info(output) - raise TestCaseFailed() + raise TestCaseFailed(format_case_name(one_case)) def handle_exception_reset(data): """ @@ -317,7 +315,7 @@ def run_unit_test_cases(env, extra_data): except TestCaseFailed: failed_cases.append(format_case_name(one_case)) except Exception as e: - junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + handle_unexpected_exception(junit_test_case, e) failed_cases.append(format_case_name(one_case)) finally: TinyFW.JunitReport.update_performance(performance_items) @@ -330,7 +328,7 @@ def run_unit_test_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) class Handler(threading.Thread): @@ -516,8 +514,10 @@ def run_multiple_devices_cases(env, extra_data): try: result = run_one_multiple_devices_case(duts, ut_config, env, one_case, one_case.get('app_bin'), junit_test_case) + except TestCaseFailed: + pass # result is False, this is handled by the finally block except Exception as e: - junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + handle_unexpected_exception(junit_test_case, e) finally: if result: Utility.console_log("Success: " + format_case_name(one_case), color="green") @@ -534,7 +534,7 @@ def run_multiple_devices_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) def run_one_multiple_stage_case(dut, one_case, junit_test_case): @@ -589,7 +589,7 @@ def run_one_multiple_stage_case(dut, one_case, junit_test_case): else: Utility.console_log("Failed: " + format_case_name(one_case), color="red") junit_test_case.add_failure_info(output) - raise TestCaseFailed() + raise TestCaseFailed(format_case_name(one_case)) stage_finish.append("break") def handle_exception_reset(data): @@ -677,7 +677,7 @@ def run_multiple_stage_cases(env, extra_data): except TestCaseFailed: failed_cases.append(format_case_name(one_case)) except Exception as e: - junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + handle_unexpected_exception(junit_test_case, e) failed_cases.append(format_case_name(one_case)) finally: TinyFW.JunitReport.update_performance(performance_items) @@ -690,7 +690,7 @@ def run_multiple_stage_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) def detect_update_unit_test_info(env, extra_data, app_bin):