From a5bcaea9cc69f888198f8af2a9552557e6e23356 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 3 Dec 2020 16:32:54 +0800 Subject: [PATCH 1/6] use "additional_duts" to overwrite dut class for different targets --- tools/ci/python_packages/ttfw_idf/__init__.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index d033cbd878..03673807b6 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -16,6 +16,7 @@ import json import logging import os import re +from copy import deepcopy import junit_xml @@ -73,11 +74,11 @@ def local_test_check(decorator_target): try: idf_target = sdkconfig['IDF_TARGET'].upper() except KeyError: - logging.warning('IDF_TARGET not in {}. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path))) + logging.debug('IDF_TARGET not in {}. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path))) else: - logging.info('IDF_TARGET: {}'.format(idf_target)) + logging.debug('IDF_TARGET: {}'.format(idf_target)) else: - logging.warning('{} not found. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path))) + logging.debug('{} not found. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path))) if isinstance(decorator_target, list): if idf_target not in decorator_target: @@ -88,11 +89,11 @@ def local_test_check(decorator_target): return idf_target -def get_dut_class(target, erase_nvs=None): - if target not in TARGET_DUT_CLS_DICT: - raise Exception('target can only be {%s} (case insensitive)' % ', '.join(TARGET_DUT_CLS_DICT.keys())) +def get_dut_class(target, dut_class_dict, erase_nvs=None): + if target not in dut_class_dict: + raise Exception('target can only be {%s} (case insensitive)' % ', '.join(dut_class_dict.keys())) - dut = TARGET_DUT_CLS_DICT[target.upper()] + dut = dut_class_dict[target.upper()] if erase_nvs: dut.ERASE_NVS = 'erase_nvs' return dut @@ -111,16 +112,19 @@ def ci_target_check(func): return wrapper -def test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, drop_kwargs_dut=False, **kwargs): +def test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, **kwargs): + target = upper_list_or_str(target) test_target = local_test_check(target) - dut = get_dut_class(test_target, erase_nvs) - if drop_kwargs_dut and 'dut' in kwargs: # panic_test() will inject dut, resolve conflicts here - dut = kwargs['dut'] - del kwargs['dut'] + if 'additional_duts' in kwargs: + dut_classes = deepcopy(TARGET_DUT_CLS_DICT) + dut_classes.update(kwargs['additional_duts']) + else: + dut_classes = TARGET_DUT_CLS_DICT + dut = get_dut_class(test_target, dut_classes, erase_nvs) original_method = TinyFW.test_method( - app=app, dut=dut, target=upper_list_or_str(target), ci_target=upper_list_or_str(ci_target), + app=app, dut=dut, target=target, ci_target=upper_list_or_str(ci_target), module=module, execution_time=execution_time, level=level, erase_nvs=erase_nvs, - dut_dict=TARGET_DUT_CLS_DICT, **kwargs + dut_dict=dut_classes, **kwargs ) test_func = original_method(func) test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"]) @@ -188,7 +192,7 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc", :return: test method """ def test(func): - return test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, drop_kwargs_dut=True, **kwargs) + return test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, **kwargs) return test From 2547670477d5a18aa36efb762544e2bdf951a6b7 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 3 Dec 2020 16:33:32 +0800 Subject: [PATCH 2/6] Add esp32s2 support for panic test, run as default --- .../panic/test_panic_util/test_panic_util.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py index f2de09a327..520256d546 100644 --- a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py +++ b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py @@ -234,9 +234,25 @@ class ESP32PanicTestDUT(ttfw_idf.ESP32DUT, PanicTestMixin): return self.port +class ESP32S2PanicTestDUT(ttfw_idf.ESP32S2DUT, PanicTestMixin): + def get_gdb_remote(self): + return self.port + + +PANIC_TEST_DUT_DICT = { + 'ESP32': ESP32PanicTestDUT, + 'ESP32S2': ESP32S2PanicTestDUT +} + + def panic_test(**kwargs): """ Decorator for the panic tests, sets correct App and DUT classes """ - return ttfw_idf.idf_custom_test(app=PanicTestApp, dut=ESP32PanicTestDUT, env_tag="test_jtag_arm", **kwargs) + if 'target' not in kwargs: + kwargs['target'] = ['ESP32', 'ESP32S2'] + + if 'additional_duts' not in kwargs: + kwargs['additional_duts'] = PANIC_TEST_DUT_DICT + return ttfw_idf.idf_custom_test(app=PanicTestApp, env_tag="test_jtag_arm", **kwargs) def get_dut(env, app_config_name, test_name, qemu_wdt_enable=False): From 3a88626ec3573dc1a4f7ea1e19a6ca30f1c4995c Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 3 Dec 2020 16:34:10 +0800 Subject: [PATCH 3/6] search case: print target as well --- tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py index 669a1a980a..9404c0698e 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py @@ -49,7 +49,7 @@ class Search(object): test_functions_out += cls.replicate_case(case) for i, test_function in enumerate(test_functions_out): - print("\t{}. ".format(i + 1) + test_function.case_info["name"]) + print("\t{}. {} <{}>".format(i + 1, test_function.case_info["name"], test_function.case_info["target"])) test_function.case_info['app_dir'] = os.path.dirname(file_name) return test_functions_out From eda42035d2ab18b78f645936dbc236876e0d353c Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 3 Dec 2020 17:12:54 +0800 Subject: [PATCH 4/6] scan test: bugfix: now can build multi target for one app --- tools/ci/python_packages/ttfw_idf/CIScanTests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index 462249d157..20273378ee 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -4,6 +4,7 @@ import json import logging import os from collections import defaultdict +from copy import deepcopy from find_apps import find_apps from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE @@ -124,7 +125,7 @@ def main(): scan_info_dict = defaultdict(dict) # store the test cases dir, exclude these folders when scan for standalone apps default_exclude = args.exclude if args.exclude else [] - exclude_apps = default_exclude + exclude_apps = deepcopy(default_exclude) build_system = args.build_system.lower() build_system_class = BUILD_SYSTEMS[build_system] From e32885d6992227690df6c48878fa45df8690fef1 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 16 Jun 2020 21:06:36 +0200 Subject: [PATCH 5/6] tools/test_apps: run tests which don't need JTAG on Example_GENERIC --- tools/ci/config/target-test.yml | 7 ++++--- .../system/panic/test_panic_util/test_panic_util.py | 2 +- tools/test_apps/system/startup/app_test.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index a506db4349..f17b6b70cd 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -315,7 +315,6 @@ example_test_016: test_app_test_001: extends: .test_app_template - parallel: 4 tags: - ESP32 - test_jtag_arm @@ -324,7 +323,6 @@ test_app_test_001: - $CI_PROJECT_DIR/tools/test_apps/system/*/*.log variables: SETUP_TOOLS: "1" - PYTHON_VER: 3 test_app_test_002: extends: .test_app_template @@ -340,9 +338,12 @@ test_app_test_003: test_app_test_004: extends: .test_app_template + parallel: 4 tags: - - ESP32S2 + - ESP32 - Example_GENERIC + variables: + SETUP_TOOLS: "1" component_ut_test_001: extends: .component_ut_32_template diff --git a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py index 520256d546..66cab91283 100644 --- a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py +++ b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py @@ -252,7 +252,7 @@ def panic_test(**kwargs): if 'additional_duts' not in kwargs: kwargs['additional_duts'] = PANIC_TEST_DUT_DICT - return ttfw_idf.idf_custom_test(app=PanicTestApp, env_tag="test_jtag_arm", **kwargs) + return ttfw_idf.idf_custom_test(app=PanicTestApp, env_tag="Example_GENERIC", **kwargs) def get_dut(env, app_config_name, test_name, qemu_wdt_enable=False): diff --git a/tools/test_apps/system/startup/app_test.py b/tools/test_apps/system/startup/app_test.py index ebd4cf3306..6d3950ca1f 100644 --- a/tools/test_apps/system/startup/app_test.py +++ b/tools/test_apps/system/startup/app_test.py @@ -6,7 +6,7 @@ import ttfw_idf from tiny_test_fw import Utility -@ttfw_idf.idf_custom_test(env_tag="test_jtag_arm", group="test-apps") +@ttfw_idf.idf_custom_test(env_tag="Example_GENERIC", group="test-apps") def test_startup(env, extra_data): config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*")) config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files] From 0f96b70294e9d84dc149fda70c309995ec139470 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 3 Dec 2020 17:53:27 +0800 Subject: [PATCH 6/6] ci: add esp32s2 support for panic test --- tools/ci/config/target-test.yml | 6 ++++++ tools/test_apps/system/panic/app_test.py | 12 ++++++------ .../system/panic/test_panic_util/test_panic_util.py | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index f17b6b70cd..dd479ae896 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -337,6 +337,12 @@ test_app_test_003: - Example_PPP test_app_test_004: + extends: .test_app_template + tags: + - ESP32S2 + - Example_GENERIC + +test_app_test_esp32_generic: extends: .test_app_template parallel: 4 tags: diff --git a/tools/test_apps/system/panic/app_test.py b/tools/test_apps/system/panic/app_test.py index 15ce730a3d..bb0869f738 100644 --- a/tools/test_apps/system/panic/app_test.py +++ b/tools/test_apps/system/panic/app_test.py @@ -6,7 +6,7 @@ from test_panic_util.test_panic_util import panic_test, run_all # test_task_wdt -@panic_test() +@panic_test(target=['ESP32', 'ESP32S2']) def test_panic_task_wdt(env, _extra_data): test.task_wdt_inner(env, "panic") @@ -134,7 +134,7 @@ def test_gdbstub_cache_error(env, _extra_data): # test_stack_overflow -@panic_test() +@panic_test(target=['ESP32', 'ESP32S2']) def test_panic_stack_overflow(env, _extra_data): test.stack_overflow_inner(env, "panic") @@ -166,7 +166,7 @@ def test_gdbstub_stack_overflow(env, _extra_data): # test_instr_fetch_prohibited -@panic_test() +@panic_test(target=['ESP32', 'ESP32S2']) def test_panic_instr_fetch_prohibited(env, _extra_data): test.instr_fetch_prohibited_inner(env, "panic") @@ -198,7 +198,7 @@ def test_gdbstub_instr_fetch_prohibited(env, _extra_data): # test_illegal_instruction -@panic_test() +@panic_test(target=['ESP32', 'ESP32S2']) def test_panic_illegal_instruction(env, _extra_data): test.illegal_instruction_inner(env, "panic") @@ -230,7 +230,7 @@ def test_gdbstub_illegal_instruction(env, _extra_data): # test_storeprohibited -@panic_test() +@panic_test(target=['ESP32', 'ESP32S2']) def test_panic_storeprohibited(env, _extra_data): test.storeprohibited_inner(env, "panic") @@ -262,7 +262,7 @@ def test_gdbstub_storeprohibited(env, _extra_data): # test_abort -@panic_test() +@panic_test(target=['ESP32', 'ESP32S2']) def test_panic_abort(env, _extra_data): test.abort_inner(env, "panic") diff --git a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py index 66cab91283..2c68b91475 100644 --- a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py +++ b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py @@ -248,7 +248,7 @@ PANIC_TEST_DUT_DICT = { def panic_test(**kwargs): """ Decorator for the panic tests, sets correct App and DUT classes """ if 'target' not in kwargs: - kwargs['target'] = ['ESP32', 'ESP32S2'] + kwargs['target'] = ['ESP32'] if 'additional_duts' not in kwargs: kwargs['additional_duts'] = PANIC_TEST_DUT_DICT