Merge branch 'feature/panic_test_multi_target_support' into 'master'

CI: Add multi-target support for panic test

See merge request espressif/esp-idf!11451

tools/ci/config/target-test.yml

@@ -315,7 +315,6 @@ example_test_016:
 
 test_app_test_001:
   extends: .test_app_template
-  parallel: 4
   tags:
     - ESP32
     - test_jtag_arm
@@ -324,7 +323,6 @@ test_app_test_001:
       - $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
   variables:
     SETUP_TOOLS: "1"
-    PYTHON_VER: 3
 
 test_app_test_002:
   extends: .test_app_template
@@ -344,6 +342,15 @@ test_app_test_004:
     - ESP32S2
     - Example_GENERIC
 
+test_app_test_esp32_generic:
+  extends: .test_app_template
+  parallel: 4
+  tags:
+    - ESP32
+    - Example_GENERIC
+  variables:
+    SETUP_TOOLS: "1"
+
 component_ut_test_001:
   extends: .component_ut_32_template
   tags:

tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py

@@ -49,7 +49,7 @@ class Search(object):
             test_functions_out += cls.replicate_case(case)
 
         for i, test_function in enumerate(test_functions_out):
-            print("\t{}. ".format(i + 1) + test_function.case_info["name"])
+            print("\t{}. {} <{}>".format(i + 1, test_function.case_info["name"], test_function.case_info["target"]))
             test_function.case_info['app_dir'] = os.path.dirname(file_name)
         return test_functions_out
 

tools/ci/python_packages/ttfw_idf/CIScanTests.py

@@ -4,6 +4,7 @@ import json
 import logging
 import os
 from collections import defaultdict
+from copy import deepcopy
 
 from find_apps import find_apps
 from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE
@@ -124,7 +125,7 @@ def main():
     scan_info_dict = defaultdict(dict)
     # store the test cases dir, exclude these folders when scan for standalone apps
     default_exclude = args.exclude if args.exclude else []
-    exclude_apps = default_exclude
+    exclude_apps = deepcopy(default_exclude)
 
     build_system = args.build_system.lower()
     build_system_class = BUILD_SYSTEMS[build_system]

tools/ci/python_packages/ttfw_idf/__init__.py

@@ -16,6 +16,7 @@ import json
 import logging
 import os
 import re
+from copy import deepcopy
 
 import junit_xml
 
@@ -73,11 +74,11 @@ def local_test_check(decorator_target):
         try:
             idf_target = sdkconfig['IDF_TARGET'].upper()
         except KeyError:
-            logging.warning('IDF_TARGET not in {}. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
+            logging.debug('IDF_TARGET not in {}. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
         else:
-            logging.info('IDF_TARGET: {}'.format(idf_target))
+            logging.debug('IDF_TARGET: {}'.format(idf_target))
     else:
-        logging.warning('{} not found. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
+        logging.debug('{} not found. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
 
     if isinstance(decorator_target, list):
         if idf_target not in decorator_target:
@@ -88,11 +89,11 @@ def local_test_check(decorator_target):
     return idf_target
 
 
-def get_dut_class(target, erase_nvs=None):
+def get_dut_class(target, dut_class_dict, erase_nvs=None):
-    if target not in TARGET_DUT_CLS_DICT:
+    if target not in dut_class_dict:
-        raise Exception('target can only be {%s} (case insensitive)' % ', '.join(TARGET_DUT_CLS_DICT.keys()))
+        raise Exception('target can only be {%s} (case insensitive)' % ', '.join(dut_class_dict.keys()))
 
-    dut = TARGET_DUT_CLS_DICT[target.upper()]
+    dut = dut_class_dict[target.upper()]
     if erase_nvs:
         dut.ERASE_NVS = 'erase_nvs'
     return dut
@@ -111,16 +112,19 @@ def ci_target_check(func):
     return wrapper
 
 
-def test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, drop_kwargs_dut=False, **kwargs):
+def test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, **kwargs):
+    target = upper_list_or_str(target)
     test_target = local_test_check(target)
-    dut = get_dut_class(test_target, erase_nvs)
+    if 'additional_duts' in kwargs:
-    if drop_kwargs_dut and 'dut' in kwargs:  # panic_test() will inject dut, resolve conflicts here
+        dut_classes = deepcopy(TARGET_DUT_CLS_DICT)
-        dut = kwargs['dut']
+        dut_classes.update(kwargs['additional_duts'])
-        del kwargs['dut']
+    else:
+        dut_classes = TARGET_DUT_CLS_DICT
+    dut = get_dut_class(test_target, dut_classes, erase_nvs)
     original_method = TinyFW.test_method(
-        app=app, dut=dut, target=upper_list_or_str(target), ci_target=upper_list_or_str(ci_target),
+        app=app, dut=dut, target=target, ci_target=upper_list_or_str(ci_target),
         module=module, execution_time=execution_time, level=level, erase_nvs=erase_nvs,
-        dut_dict=TARGET_DUT_CLS_DICT, **kwargs
+        dut_dict=dut_classes, **kwargs
     )
     test_func = original_method(func)
     test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
@@ -188,7 +192,7 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc",
     :return: test method
     """
     def test(func):
-        return test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, drop_kwargs_dut=True, **kwargs)
+        return test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, **kwargs)
     return test
 
 

tools/test_apps/system/panic/app_test.py

@@ -6,7 +6,7 @@ from test_panic_util.test_panic_util import panic_test, run_all
 
 # test_task_wdt
 
-@panic_test()
+@panic_test(target=['ESP32', 'ESP32S2'])
 def test_panic_task_wdt(env, _extra_data):
     test.task_wdt_inner(env, "panic")
 
@@ -134,7 +134,7 @@ def test_gdbstub_cache_error(env, _extra_data):
 
 # test_stack_overflow
 
-@panic_test()
+@panic_test(target=['ESP32', 'ESP32S2'])
 def test_panic_stack_overflow(env, _extra_data):
     test.stack_overflow_inner(env, "panic")
 
@@ -166,7 +166,7 @@ def test_gdbstub_stack_overflow(env, _extra_data):
 
 # test_instr_fetch_prohibited
 
-@panic_test()
+@panic_test(target=['ESP32', 'ESP32S2'])
 def test_panic_instr_fetch_prohibited(env, _extra_data):
     test.instr_fetch_prohibited_inner(env, "panic")
 
@@ -198,7 +198,7 @@ def test_gdbstub_instr_fetch_prohibited(env, _extra_data):
 
 # test_illegal_instruction
 
-@panic_test()
+@panic_test(target=['ESP32', 'ESP32S2'])
 def test_panic_illegal_instruction(env, _extra_data):
     test.illegal_instruction_inner(env, "panic")
 
@@ -230,7 +230,7 @@ def test_gdbstub_illegal_instruction(env, _extra_data):
 
 # test_storeprohibited
 
-@panic_test()
+@panic_test(target=['ESP32', 'ESP32S2'])
 def test_panic_storeprohibited(env, _extra_data):
     test.storeprohibited_inner(env, "panic")
 
@@ -262,7 +262,7 @@ def test_gdbstub_storeprohibited(env, _extra_data):
 
 # test_abort
 
-@panic_test()
+@panic_test(target=['ESP32', 'ESP32S2'])
 def test_panic_abort(env, _extra_data):
     test.abort_inner(env, "panic")
 

tools/test_apps/system/panic/test_panic_util/test_panic_util.py

@@ -234,9 +234,25 @@ class ESP32PanicTestDUT(ttfw_idf.ESP32DUT, PanicTestMixin):
         return self.port
 
 
+class ESP32S2PanicTestDUT(ttfw_idf.ESP32S2DUT, PanicTestMixin):
+    def get_gdb_remote(self):
+        return self.port
+
+
+PANIC_TEST_DUT_DICT = {
+    'ESP32': ESP32PanicTestDUT,
+    'ESP32S2': ESP32S2PanicTestDUT
+}
+
+
 def panic_test(**kwargs):
     """ Decorator for the panic tests, sets correct App and DUT classes """
-    return ttfw_idf.idf_custom_test(app=PanicTestApp, dut=ESP32PanicTestDUT, env_tag="test_jtag_arm", **kwargs)
+    if 'target' not in kwargs:
+        kwargs['target'] = ['ESP32']
+
+    if 'additional_duts' not in kwargs:
+        kwargs['additional_duts'] = PANIC_TEST_DUT_DICT
+    return ttfw_idf.idf_custom_test(app=PanicTestApp, env_tag="Example_GENERIC", **kwargs)
 
 
 def get_dut(env, app_config_name, test_name, qemu_wdt_enable=False):

tools/test_apps/system/startup/app_test.py

@@ -6,7 +6,7 @@ import ttfw_idf
 from tiny_test_fw import Utility
 
 
-@ttfw_idf.idf_custom_test(env_tag="test_jtag_arm", group="test-apps")
+@ttfw_idf.idf_custom_test(env_tag="Example_GENERIC", group="test-apps")
 def test_startup(env, extra_data):
     config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*"))
     config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files]