From 6c98d7e4bdb2c7ddb34325cb1882bc1605bf24a2 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 3 Apr 2020 16:16:40 +0800 Subject: [PATCH] Add multi target support for search examples drop keyword `dut`, use `target` instead to assign`dut_class` to `Env` --- .../ci/python_packages/tiny_test_fw/TinyFW.py | 24 +++++++- .../tiny_test_fw/Utility/SearchCases.py | 32 ++++++---- .../ttfw_idf/CIAssignExampleTest.py | 6 +- tools/ci/python_packages/ttfw_idf/__init__.py | 58 +++++++------------ 4 files changed, 68 insertions(+), 52 deletions(-) diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py index 6e2d03d38e..77eaab1540 100644 --- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py +++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py @@ -63,7 +63,6 @@ class DefaultEnvConfig(object): set_default_config = DefaultEnvConfig.set_default_config get_default_config = DefaultEnvConfig.get_default_config - MANDATORY_INFO = { "execution_time": 1, "env_tag": "default", @@ -158,6 +157,7 @@ def test_method(**kwargs): In some cases, one test function might test many test cases. If this flag is set, test case can update junit report by its own. """ + def test(test_func): case_info = MANDATORY_INFO.copy() @@ -181,6 +181,27 @@ def test_method(**kwargs): env_config[key] = kwargs[key] env_config.update(overwrite) + + # FIXME: CI need more variable here. add `if CI_TARGET: ...` later with CI. + target = env_config['target'] if 'target' in env_config else kwargs['target'] + dut_dict = kwargs['dut_dict'] + if isinstance(target, list): + target = target[0] + elif isinstance(target, str): + target = target + else: + raise TypeError('keyword targets can only be list or str') + if target not in dut_dict: + raise Exception('target can only be {%s}' % ', '.join(dut_dict.keys())) + + dut = dut_dict[target] + try: + # try to config the default behavior of erase nvs + dut.ERASE_NVS = kwargs['erase_nvs'] + except AttributeError: + pass + + env_config['dut'] = dut env_inst = Env.Env(**env_config) # prepare for xunit test results @@ -227,4 +248,5 @@ def test_method(**kwargs): handle_test.case_info = case_info handle_test.test_method = True return handle_test + return test diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py index 984083a143..248b692aab 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py @@ -42,9 +42,14 @@ class Search(object): continue except ImportError as e: print("ImportError: \r\n\tFile:" + file_name + "\r\n\tError:" + str(e)) - for i, test_function in enumerate(test_functions): + + test_functions_out = [] + for case in test_functions: + test_functions_out += cls.replicate_case(case) + + for i, test_function in enumerate(test_functions_out): print("\t{}. ".format(i + 1) + test_function.case_info["name"]) - return test_functions + return test_functions_out @classmethod def _search_test_case_files(cls, test_case, file_pattern): @@ -77,17 +82,26 @@ class Search(object): if isinstance(case.case_info[key], (list, tuple)): replicate_config.append(key) - def _replicate_for_key(case_list, replicate_key, replicate_list): + def _replicate_for_key(cases, replicate_key, replicate_list): + def deepcopy_func(f, name=None): + fn = types.FunctionType(f.__code__, f.__globals__, name if name else f.__name__, + f.__defaults__, f.__closure__) + fn.__dict__.update(copy.deepcopy(f.__dict__)) + return fn + case_out = [] - for _case in case_list: + for inner_case in cases: for value in replicate_list: - new_case = copy.deepcopy(_case) + new_case = deepcopy_func(inner_case) new_case.case_info[replicate_key] = value case_out.append(new_case) return case_out replicated_cases = [case] - for key in replicate_config: + while replicate_config: + if not replicate_config: + break + key = replicate_config.pop() replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key]) return replicated_cases @@ -104,8 +118,4 @@ class Search(object): test_cases = [] for test_case_file in test_case_files: test_cases += cls._search_cases_from_file(test_case_file) - # handle replicate cases - test_case_out = [] - for case in test_cases: - test_case_out += cls.replicate_case(case) - return test_case_out + return test_cases diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index d887ae1d0e..3acdd78d06 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -29,7 +29,7 @@ IDF_PATH_FROM_ENV = os.getenv("IDF_PATH") class ExampleGroup(CIAssignTest.Group): - SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"] + SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "target"] BUILD_LOCAL_DIR = "build_examples" BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] @@ -61,7 +61,7 @@ def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=Exa artifact_index_list = [] def format_build_log_path(): - parallel = job_info["parallel_num"] # Could be None if "parallel_num" not defined for the job + parallel = job_info["parallel_num"] # Could be None if "parallel_num" not defined for the job return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1) for build_job_name in case_group.BUILD_JOB_NAMES: @@ -99,7 +99,7 @@ if __name__ == '__main__': help="file name pattern used to find Python test case files") parser.add_argument('--custom-group', help='select custom-group for the test cases, if other than ExampleTest', - choices=['example','test-apps'], default='example') + choices=['example', 'test-apps'], default='example') args = parser.parse_args() diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index efbde25e75..15c1a7d392 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -19,19 +19,24 @@ from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp # noqa: ex from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT # noqa: export DUTs for users from .DebugUtils import OCDProcess, GDBProcess, TelnetProcess, CustomProcess # noqa: export DebugUtils for users +# pass TARGET_DUT_CLS_DICT to Env.py to avoid circular dependency issue. +TARGET_DUT_CLS_DICT = { + 'ESP32': ESP32DUT, + 'ESP32S2': ESP32S2DUT, +} + def format_case_id(chip, case_name): return "{}.{}".format(chip, case_name) -def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1, +def idf_example_test(app=Example, target="ESP32", module="examples", execution_time=1, level="example", erase_nvs=True, config_name=None, **kwargs): """ decorator for testing idf examples (with default values for some keyword args). :param app: test application class - :param dut: dut class - :param chip: chip supported, string or tuple + :param target: target supported, string or iterable :param module: module, string :param execution_time: execution time in minutes, int :param level: test level, could be used to filter test cases, string @@ -40,31 +45,24 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", e :param kwargs: other keyword args :return: test method """ - try: - # try to config the default behavior of erase nvs - dut.ERASE_NVS = erase_nvs - except AttributeError: - pass - - original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module, - execution_time=execution_time, level=level, **kwargs) def test(func): + original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time, + level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs) test_func = original_method(func) - test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"]) + test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"]) return test_func return test -def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", execution_time=1, +def idf_unit_test(app=UT, target="ESP32", module="unit-test", execution_time=1, level="unit", erase_nvs=True, **kwargs): """ decorator for testing idf unit tests (with default values for some keyword args). :param app: test application class - :param dut: dut class - :param chip: chip supported, string or tuple + :param target: target supported, string or iterable :param module: module, string :param execution_time: execution time in minutes, int :param level: test level, could be used to filter test cases, string @@ -72,32 +70,24 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio :param kwargs: other keyword args :return: test method """ - try: - # try to config the default behavior of erase nvs - dut.ERASE_NVS = erase_nvs - except AttributeError: - pass - - original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module, - execution_time=execution_time, level=level, **kwargs) def test(func): + original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time, + level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs) test_func = original_method(func) - test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"]) + test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"]) return test_func return test -def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1, +def idf_custom_test(app=TestApp, target="ESP32", module="misc", execution_time=1, level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs): - """ decorator for idf custom tests (with default values for some keyword args). :param app: test application class - :param dut: dut class - :param chip: chip supported, string or tuple + :param target: target supported, string or iterable :param module: module, string :param execution_time: execution time in minutes, int :param level: test level, could be used to filter test cases, string @@ -107,18 +97,12 @@ def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execut :param kwargs: other keyword args :return: test method """ - try: - # try to config the default behavior of erase nvs - dut.ERASE_NVS = erase_nvs - except AttributeError: - pass - - original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module, - execution_time=execution_time, level=level, **kwargs) def test(func): + original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time, + level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs) test_func = original_method(func) - test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"]) + test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"]) return test_func return test