Merge branch 'ci/ttfw_exception_details' into 'master'

ci ttfw: Consistently handle unexpected exceptions in test cases

See merge request espressif/esp-idf!11843

tools/ci/python_packages/tiny_test_fw/DUT.py

@@ -439,7 +439,7 @@ class BaseDUT(object):
         if isinstance(data, type(u'')):
             try:
                 data = data.encode('utf-8')
-            except Exception as e:
+            except UnicodeEncodeError as e:
                 print(u'Cannot encode {} of type {}'.format(data, type(data)))
                 raise e
         return data

tools/ci/python_packages/tiny_test_fw/TinyFW.py

@@ -15,7 +15,6 @@
 """ Interface for test cases. """
 import os
 import time
-import traceback
 import functools
 import socket
 from datetime import datetime
@@ -28,6 +27,20 @@ from . import App
 from . import Utility
 
 
+class TestCaseFailed(AssertionError):
+    def __init__(self, *cases):
+        """
+        Raise this exception if one or more test cases fail in a 'normal' way (ie the test runs but fails, no unexpected exceptions)
+
+        This will avoid dumping the Python stack trace, because the assumption is the junit error info and full job log already has
+        enough information for a developer to debug.
+
+        'cases' argument is the names of one or more test cases
+        """
+        message = "Test case{} failed: {}".format("s" if len(cases) > 1 else "", ", ".join(str(c) for c in cases))
+        super(TestCaseFailed, self).__init__(self, message)
+
+
 class DefaultEnvConfig(object):
     """
     default test configs. There're 3 places to set configs, priority is (high -> low):
@@ -194,11 +207,10 @@ def test_method(**kwargs):
                 test_func(env_inst, extra_data)
                 # if finish without exception, test result is True
                 result = True
+            except TestCaseFailed as e:
+                junit_test_case.add_failure_info(str(e))
             except Exception as e:
-                # handle all the exceptions here
+                Utility.handle_unexpected_exception(junit_test_case, e)
-                traceback.print_exc()
-                # log failure
-                junit_test_case.add_failure_info(str(e) + ":\r\n" + traceback.format_exc())
             finally:
                 # do close all DUTs, if result is False then print DUT debug info
                 close_errors = env_inst.close(dut_debug=(not result))

tools/ci/python_packages/tiny_test_fw/Utility/__init__.py

@@ -2,6 +2,7 @@ from __future__ import print_function
 import os.path
 import sys
 import time
+import traceback
 
 from .. import Env
 
@@ -95,3 +96,16 @@ def load_source(path):
             sys.path.remove(dir)
         __LOADED_MODULES[path] = ret
         return ret
+
+
+def handle_unexpected_exception(junit_test_case, exception):
+    """
+    Helper to log & add junit result details for an unexpected exception encountered
+    when running a test case.
+
+    Should always be called from inside an except: block
+    """
+    traceback.print_exc()
+    # AssertionError caused by an 'assert' statement has an empty string as its 'str' form
+    e_str = str(exception) if str(exception) else repr(exception)
+    junit_test_case.add_failure_info("Unexpected exception: {}\n{}".format(e_str, traceback.format_exc()))

tools/unit-test-app/unit_test.py

@@ -24,6 +24,8 @@ import argparse
 import threading
 
 from tiny_test_fw import TinyFW, Utility, Env, DUT
+from tiny_test_fw.TinyFW import TestCaseFailed
+from tiny_test_fw.Utility import handle_unexpected_exception
 import ttfw_idf
 
 UT_APP_BOOT_UP_DONE = "Press ENTER to see the list of tests."
@@ -71,10 +73,6 @@ def reset_reason_matches(reported_str, expected_str):
     return False
 
 
-class TestCaseFailed(AssertionError):
-    pass
-
-
 def format_test_case_config(test_case_data):
     """
     convert the test case data to unified format.
@@ -221,7 +219,7 @@ def run_one_normal_case(dut, one_case, junit_test_case):
         else:
             Utility.console_log("Failed: " + format_case_name(one_case), color="red")
             junit_test_case.add_failure_info(output)
-            raise TestCaseFailed()
+            raise TestCaseFailed(format_case_name(one_case))
 
     def handle_exception_reset(data):
         """
@@ -317,7 +315,7 @@ def run_unit_test_cases(env, extra_data):
             except TestCaseFailed:
                 failed_cases.append(format_case_name(one_case))
             except Exception as e:
-                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+                handle_unexpected_exception(junit_test_case, e)
                 failed_cases.append(format_case_name(one_case))
             finally:
                 TinyFW.JunitReport.update_performance(performance_items)
@@ -330,7 +328,7 @@ def run_unit_test_cases(env, extra_data):
         Utility.console_log("Failed Cases:", color="red")
         for _case_name in failed_cases:
             Utility.console_log("\t" + _case_name, color="red")
-        raise AssertionError("Unit Test Failed")
+        raise TestCaseFailed(*failed_cases)
 
 
 class Handler(threading.Thread):
@@ -516,8 +514,10 @@ def run_multiple_devices_cases(env, extra_data):
             try:
                 result = run_one_multiple_devices_case(duts, ut_config, env, one_case,
                                                        one_case.get('app_bin'), junit_test_case)
+            except TestCaseFailed:
+                pass  # result is False, this is handled by the finally block
             except Exception as e:
-                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+                handle_unexpected_exception(junit_test_case, e)
             finally:
                 if result:
                     Utility.console_log("Success: " + format_case_name(one_case), color="green")
@@ -534,7 +534,7 @@ def run_multiple_devices_cases(env, extra_data):
         Utility.console_log("Failed Cases:", color="red")
         for _case_name in failed_cases:
             Utility.console_log("\t" + _case_name, color="red")
-        raise AssertionError("Unit Test Failed")
+        raise TestCaseFailed(*failed_cases)
 
 
 def run_one_multiple_stage_case(dut, one_case, junit_test_case):
@@ -589,7 +589,7 @@ def run_one_multiple_stage_case(dut, one_case, junit_test_case):
             else:
                 Utility.console_log("Failed: " + format_case_name(one_case), color="red")
                 junit_test_case.add_failure_info(output)
-                raise TestCaseFailed()
+                raise TestCaseFailed(format_case_name(one_case))
             stage_finish.append("break")
 
         def handle_exception_reset(data):
@@ -677,7 +677,7 @@ def run_multiple_stage_cases(env, extra_data):
             except TestCaseFailed:
                 failed_cases.append(format_case_name(one_case))
             except Exception as e:
-                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+                handle_unexpected_exception(junit_test_case, e)
                 failed_cases.append(format_case_name(one_case))
             finally:
                 TinyFW.JunitReport.update_performance(performance_items)
@@ -690,7 +690,7 @@ def run_multiple_stage_cases(env, extra_data):
         Utility.console_log("Failed Cases:", color="red")
         for _case_name in failed_cases:
             Utility.console_log("\t" + _case_name, color="red")
-        raise AssertionError("Unit Test Failed")
+        raise TestCaseFailed(*failed_cases)
 
 
 def detect_update_unit_test_info(env, extra_data, app_bin):