From 6c98d7e4bdb2c7ddb34325cb1882bc1605bf24a2 Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Fri, 3 Apr 2020 16:16:40 +0800
Subject: [PATCH 01/11] Add multi target support for search examples

drop keyword `dut`, use `target` instead to assign`dut_class` to `Env`
---
 .../ci/python_packages/tiny_test_fw/TinyFW.py | 24 +++++++-
 .../tiny_test_fw/Utility/SearchCases.py       | 32 ++++++----
 .../ttfw_idf/CIAssignExampleTest.py           |  6 +-
 tools/ci/python_packages/ttfw_idf/__init__.py | 58 +++++++------------
 4 files changed, 68 insertions(+), 52 deletions(-)

diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
index 6e2d03d38e..77eaab1540 100644
--- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py
+++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
@@ -63,7 +63,6 @@ class DefaultEnvConfig(object):
 set_default_config = DefaultEnvConfig.set_default_config
 get_default_config = DefaultEnvConfig.get_default_config
 
-
 MANDATORY_INFO = {
     "execution_time": 1,
     "env_tag": "default",
@@ -158,6 +157,7 @@ def test_method(**kwargs):
                                    In some cases, one test function might test many test cases.
                                    If this flag is set, test case can update junit report by its own.
     """
+
     def test(test_func):
 
         case_info = MANDATORY_INFO.copy()
@@ -181,6 +181,27 @@ def test_method(**kwargs):
                     env_config[key] = kwargs[key]
 
             env_config.update(overwrite)
+
+            # FIXME: CI need more variable here. add `if CI_TARGET: ...` later with CI.
+            target = env_config['target'] if 'target' in env_config else kwargs['target']
+            dut_dict = kwargs['dut_dict']
+            if isinstance(target, list):
+                target = target[0]
+            elif isinstance(target, str):
+                target = target
+            else:
+                raise TypeError('keyword targets can only be list or str')
+            if target not in dut_dict:
+                raise Exception('target can only be {%s}' % ', '.join(dut_dict.keys()))
+
+            dut = dut_dict[target]
+            try:
+                # try to config the default behavior of erase nvs
+                dut.ERASE_NVS = kwargs['erase_nvs']
+            except AttributeError:
+                pass
+
+            env_config['dut'] = dut
             env_inst = Env.Env(**env_config)
 
             # prepare for xunit test results
@@ -227,4 +248,5 @@ def test_method(**kwargs):
         handle_test.case_info = case_info
         handle_test.test_method = True
         return handle_test
+
     return test
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
index 984083a143..248b692aab 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
@@ -42,9 +42,14 @@ class Search(object):
                     continue
         except ImportError as e:
             print("ImportError: \r\n\tFile:" + file_name + "\r\n\tError:" + str(e))
-        for i, test_function in enumerate(test_functions):
+
+        test_functions_out = []
+        for case in test_functions:
+            test_functions_out += cls.replicate_case(case)
+
+        for i, test_function in enumerate(test_functions_out):
             print("\t{}. ".format(i + 1) + test_function.case_info["name"])
-        return test_functions
+        return test_functions_out
 
     @classmethod
     def _search_test_case_files(cls, test_case, file_pattern):
@@ -77,17 +82,26 @@ class Search(object):
             if isinstance(case.case_info[key], (list, tuple)):
                 replicate_config.append(key)
 
-        def _replicate_for_key(case_list, replicate_key, replicate_list):
+        def _replicate_for_key(cases, replicate_key, replicate_list):
+            def deepcopy_func(f, name=None):
+                fn = types.FunctionType(f.__code__, f.__globals__, name if name else f.__name__,
+                                        f.__defaults__, f.__closure__)
+                fn.__dict__.update(copy.deepcopy(f.__dict__))
+                return fn
+
             case_out = []
-            for _case in case_list:
+            for inner_case in cases:
                 for value in replicate_list:
-                    new_case = copy.deepcopy(_case)
+                    new_case = deepcopy_func(inner_case)
                     new_case.case_info[replicate_key] = value
                     case_out.append(new_case)
             return case_out
 
         replicated_cases = [case]
-        for key in replicate_config:
+        while replicate_config:
+            if not replicate_config:
+                break
+            key = replicate_config.pop()
             replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key])
 
         return replicated_cases
@@ -104,8 +118,4 @@ class Search(object):
         test_cases = []
         for test_case_file in test_case_files:
             test_cases += cls._search_cases_from_file(test_case_file)
-        # handle replicate cases
-        test_case_out = []
-        for case in test_cases:
-            test_case_out += cls.replicate_case(case)
-        return test_case_out
+        return test_cases
diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py
index d887ae1d0e..3acdd78d06 100644
--- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py
+++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py
@@ -29,7 +29,7 @@ IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
 
 
 class ExampleGroup(CIAssignTest.Group):
-    SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
+    SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "target"]
     BUILD_LOCAL_DIR = "build_examples"
     BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
 
@@ -61,7 +61,7 @@ def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=Exa
     artifact_index_list = []
 
     def format_build_log_path():
-        parallel = job_info["parallel_num"]    # Could be None if "parallel_num" not defined for the job
+        parallel = job_info["parallel_num"]  # Could be None if "parallel_num" not defined for the job
         return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1)
 
     for build_job_name in case_group.BUILD_JOB_NAMES:
@@ -99,7 +99,7 @@ if __name__ == '__main__':
                         help="file name pattern used to find Python test case files")
     parser.add_argument('--custom-group',
                         help='select custom-group for the test cases, if other than ExampleTest',
-                        choices=['example','test-apps'], default='example')
+                        choices=['example', 'test-apps'], default='example')
 
     args = parser.parse_args()
 
diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py
index efbde25e75..15c1a7d392 100644
--- a/tools/ci/python_packages/ttfw_idf/__init__.py
+++ b/tools/ci/python_packages/ttfw_idf/__init__.py
@@ -19,19 +19,24 @@ from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp  # noqa: ex
 from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT  # noqa: export DUTs for users
 from .DebugUtils import OCDProcess, GDBProcess, TelnetProcess, CustomProcess  # noqa: export DebugUtils for users
 
+# pass TARGET_DUT_CLS_DICT to Env.py to avoid circular dependency issue.
+TARGET_DUT_CLS_DICT = {
+    'ESP32': ESP32DUT,
+    'ESP32S2': ESP32S2DUT,
+}
+
 
 def format_case_id(chip, case_name):
     return "{}.{}".format(chip, case_name)
 
 
-def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1,
+def idf_example_test(app=Example, target="ESP32", module="examples", execution_time=1,
                      level="example", erase_nvs=True, config_name=None, **kwargs):
     """
     decorator for testing idf examples (with default values for some keyword args).
 
     :param app: test application class
-    :param dut: dut class
-    :param chip: chip supported, string or tuple
+    :param target: target supported, string or iterable
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -40,31 +45,24 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", e
     :param kwargs: other keyword args
     :return: test method
     """
-    try:
-        # try to config the default behavior of erase nvs
-        dut.ERASE_NVS = erase_nvs
-    except AttributeError:
-        pass
-
-    original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                                         execution_time=execution_time, level=level, **kwargs)
 
     def test(func):
+        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
+                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
-        test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
+        test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
 
     return test
 
 
-def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", execution_time=1,
+def idf_unit_test(app=UT, target="ESP32", module="unit-test", execution_time=1,
                   level="unit", erase_nvs=True, **kwargs):
     """
     decorator for testing idf unit tests (with default values for some keyword args).
 
     :param app: test application class
-    :param dut: dut class
-    :param chip: chip supported, string or tuple
+    :param target: target supported, string or iterable
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -72,32 +70,24 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio
     :param kwargs: other keyword args
     :return: test method
     """
-    try:
-        # try to config the default behavior of erase nvs
-        dut.ERASE_NVS = erase_nvs
-    except AttributeError:
-        pass
-
-    original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                                         execution_time=execution_time, level=level, **kwargs)
 
     def test(func):
+        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
+                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
-        test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
+        test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
 
     return test
 
 
-def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
+def idf_custom_test(app=TestApp, target="ESP32", module="misc", execution_time=1,
                     level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
-
     """
     decorator for idf custom tests (with default values for some keyword args).
 
     :param app: test application class
-    :param dut: dut class
-    :param chip: chip supported, string or tuple
+    :param target: target supported, string or iterable
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -107,18 +97,12 @@ def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execut
     :param kwargs: other keyword args
     :return: test method
     """
-    try:
-        # try to config the default behavior of erase nvs
-        dut.ERASE_NVS = erase_nvs
-    except AttributeError:
-        pass
-
-    original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                                         execution_time=execution_time, level=level, **kwargs)
 
     def test(func):
+        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
+                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
-        test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
+        test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
 
     return test

From 9f8b63da387ea3f31a4abcbfc3489f6171c016a8 Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Wed, 8 Apr 2020 11:32:25 +0800
Subject: [PATCH 02/11] Pass current target to test_method **overwrite.

---
 tools/ci/python_packages/tiny_test_fw/TinyFW.py          | 7 +++++--
 .../python_packages/tiny_test_fw/Utility/CaseConfig.py   | 9 +++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
index 77eaab1540..d5287f7c21 100644
--- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py
+++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
@@ -180,17 +180,20 @@ def test_method(**kwargs):
                 if key in env_config:
                     env_config[key] = kwargs[key]
 
+            # Runner.py should overwrite target with the current target.
             env_config.update(overwrite)
 
-            # FIXME: CI need more variable here. add `if CI_TARGET: ...` later with CI.
+            # This code block is used to run test script locally without
+            # Runner.py
             target = env_config['target'] if 'target' in env_config else kwargs['target']
-            dut_dict = kwargs['dut_dict']
             if isinstance(target, list):
                 target = target[0]
             elif isinstance(target, str):
                 target = target
             else:
                 raise TypeError('keyword targets can only be list or str')
+
+            dut_dict = kwargs['dut_dict']
             if target not in dut_dict:
                 raise Exception('target can only be {%s}' % ', '.join(dut_dict.keys()))
 
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
index 8f373bce9f..6205a8e851 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
@@ -190,6 +190,15 @@ class Parser(object):
             _overwrite = cls.handle_overwrite_args(_config.pop("overwrite", dict()))
             _extra_data = _config.pop("extra_data", None)
             _filter.update(_config)
+
+            # Try get target from yml
+            try:
+                _target = _filter['target']
+            except KeyError:
+                pass
+            else:
+                _overwrite.update({'target': _target})
+
             for test_method in test_methods:
                 if _filter_one_case(test_method, _filter):
                     test_case_list.append(TestCase.TestCase(test_method, _extra_data, **_overwrite))

From bc026133c5319727a25473518a0fae792951fb87 Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Thu, 9 Apr 2020 13:42:13 +0800
Subject: [PATCH 03/11] add ci_target filter for target and local target check

---
 .../ci/python_packages/tiny_test_fw/TinyFW.py | 51 ++++++++++++++++---
 tools/ci/python_packages/ttfw_idf/__init__.py | 30 ++++++-----
 2 files changed, 61 insertions(+), 20 deletions(-)

diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
index d5287f7c21..45dbefe23c 100644
--- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py
+++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 """ Interface for test cases. """
+import json
+import logging
 import os
 import time
 import traceback
@@ -163,6 +165,22 @@ def test_method(**kwargs):
         case_info = MANDATORY_INFO.copy()
         case_info["name"] = case_info["ID"] = test_func.__name__
         case_info["junit_report_by_case"] = False
+
+        def _filter_ci_target(target, ci_target):
+            if not ci_target:
+                return target
+            if isinstance(target, str):
+                if isinstance(ci_target, str) and target == ci_target:
+                    return ci_target
+            else:
+                if isinstance(ci_target, str) and ci_target in target:
+                    return ci_target
+                elif isinstance(ci_target, list) and set(ci_target).issubset(set(target)):
+                    return ci_target
+            raise ValueError('ci_target must be a subset of target')
+
+        if os.getenv('CI_JOB_NAME') and 'ci_target' in kwargs:
+            kwargs['target'] = _filter_ci_target(kwargs['target'], kwargs['ci_target'])
         case_info.update(kwargs)
 
         @functools.wraps(test_func)
@@ -183,15 +201,32 @@ def test_method(**kwargs):
             # Runner.py should overwrite target with the current target.
             env_config.update(overwrite)
 
-            # This code block is used to run test script locally without
-            # Runner.py
+            # if target not in the default_config or the overwrite, then take it from kwargs passed from the decorator
             target = env_config['target'] if 'target' in env_config else kwargs['target']
-            if isinstance(target, list):
-                target = target[0]
-            elif isinstance(target, str):
-                target = target
-            else:
-                raise TypeError('keyword targets can only be list or str')
+
+            # This code block is used to do run local test script target set check
+            if not os.getenv('CI_JOB_NAME'):
+                idf_target = 'ESP32'  # default if sdkconfig not found or not readable
+                expected_json_path = os.path.join('build', 'config', 'sdkconfig.json')
+                if os.path.exists(expected_json_path):
+                    sdkconfig = json.load(open(expected_json_path))
+                    try:
+                        idf_target = sdkconfig['IDF_TARGET'].upper()
+                    except KeyError:
+                        pass
+                    else:
+                        logging.info('IDF_TARGET: {}'.format(idf_target))
+                else:
+                    logging.warning('{} not found. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
+
+                if isinstance(target, list):
+                    if idf_target in target:
+                        target = idf_target
+                    else:
+                        raise ValueError('IDF_TARGET set to {}, not in decorator target value'.format(idf_target))
+                else:
+                    if idf_target != target:
+                        raise ValueError('IDF_TARGET set to {}, not equal to decorator target value'.format(idf_target))
 
             dut_dict = kwargs['dut_dict']
             if target not in dut_dict:
diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py
index 15c1a7d392..5aa4f3426e 100644
--- a/tools/ci/python_packages/ttfw_idf/__init__.py
+++ b/tools/ci/python_packages/ttfw_idf/__init__.py
@@ -30,13 +30,14 @@ def format_case_id(chip, case_name):
     return "{}.{}".format(chip, case_name)
 
 
-def idf_example_test(app=Example, target="ESP32", module="examples", execution_time=1,
+def idf_example_test(app=Example, target="ESP32", ci_target=None, module="examples", execution_time=1,
                      level="example", erase_nvs=True, config_name=None, **kwargs):
     """
     decorator for testing idf examples (with default values for some keyword args).
 
     :param app: test application class
-    :param target: target supported, string or iterable
+    :param target: target supported, string or list
+    :param ci_target: target auto run in CI, if None than all target will be tested, None, string or list
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -47,8 +48,9 @@ def idf_example_test(app=Example, target="ESP32", module="examples", execution_t
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
-                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
+        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+                                             execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
+                                             erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
         test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
@@ -56,13 +58,14 @@ def idf_example_test(app=Example, target="ESP32", module="examples", execution_t
     return test
 
 
-def idf_unit_test(app=UT, target="ESP32", module="unit-test", execution_time=1,
+def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", execution_time=1,
                   level="unit", erase_nvs=True, **kwargs):
     """
     decorator for testing idf unit tests (with default values for some keyword args).
 
     :param app: test application class
-    :param target: target supported, string or iterable
+    :param target: target supported, string or list
+    :param ci_target: target auto run in CI, if None than all target will be tested, None, string or list
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -72,8 +75,9 @@ def idf_unit_test(app=UT, target="ESP32", module="unit-test", execution_time=1,
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
-                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
+        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+                                             execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
+                                             erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
         test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
@@ -81,13 +85,14 @@ def idf_unit_test(app=UT, target="ESP32", module="unit-test", execution_time=1,
     return test
 
 
-def idf_custom_test(app=TestApp, target="ESP32", module="misc", execution_time=1,
+def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc", execution_time=1,
                     level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
     """
     decorator for idf custom tests (with default values for some keyword args).
 
     :param app: test application class
-    :param target: target supported, string or iterable
+    :param target: target supported, string or list
+    :param ci_target: target auto run in CI, if None than all target will be tested, None, string or list
     :param module: module, string
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
@@ -99,8 +104,9 @@ def idf_custom_test(app=TestApp, target="ESP32", module="misc", execution_time=1
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, module=module, execution_time=execution_time,
-                                             level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs)
+        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+                                             execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
+                                             erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
         test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func

From 38c288bc0ee81ba5bccffa4bc9b441867ef264b4 Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Thu, 9 Apr 2020 18:19:03 +0800
Subject: [PATCH 04/11] Add `BOT_TARGET_FILTER` to `_apply_bot_filter`

fix the bug that will calculate twice if ci_target is a list.
---
 .../tiny_test_fw/Utility/CIAssignTest.py              | 11 +++++------
 .../tiny_test_fw/Utility/SearchCases.py               |  2 ++
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
index 367de54dc5..6b584c31c0 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
@@ -234,12 +234,11 @@ class AssignTest(object):
 
         :return: filter for search test cases
         """
-        bot_filter = os.getenv("BOT_CASE_FILTER")
-        if bot_filter:
-            bot_filter = json.loads(bot_filter)
-        else:
-            bot_filter = dict()
-        return bot_filter
+        res = dict()
+        for bot_filter in [os.getenv('BOT_CASE_FILTER'), os.getenv('BOT_TARGET_FILTER')]:
+            if bot_filter:
+                res.update(json.loads(bot_filter))
+        return res
 
     def _apply_bot_test_count(self):
         """
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
index 248b692aab..075f7a1cca 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
@@ -79,6 +79,8 @@ class Search(object):
         """
         replicate_config = []
         for key in case.case_info:
+            if key == 'ci_target':  # ci_target is used to filter target, should not be duplicated.
+                continue
             if isinstance(case.case_info[key], (list, tuple)):
                 replicate_config.append(key)
 

From e553092d62dee835b722b76a51cdfec06172c07a Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Mon, 20 Apr 2020 15:12:03 +0800
Subject: [PATCH 05/11] move ci_target_check to ttfw_idf, move ci_target_filter
 to AssignTest

mark `supported_in_ci` for AssignTest filter.
---
 .../ci/python_packages/tiny_test_fw/TinyFW.py | 17 +---------
 .../tiny_test_fw/Utility/CIAssignTest.py      |  1 +
 .../tiny_test_fw/Utility/CaseConfig.py        |  1 -
 .../tiny_test_fw/Utility/SearchCases.py       | 10 ++++++
 tools/ci/python_packages/ttfw_idf/__init__.py | 33 +++++++++++++++++--
 5 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
index 45dbefe23c..37b8b8ed5b 100644
--- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py
+++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
@@ -166,21 +166,6 @@ def test_method(**kwargs):
         case_info["name"] = case_info["ID"] = test_func.__name__
         case_info["junit_report_by_case"] = False
 
-        def _filter_ci_target(target, ci_target):
-            if not ci_target:
-                return target
-            if isinstance(target, str):
-                if isinstance(ci_target, str) and target == ci_target:
-                    return ci_target
-            else:
-                if isinstance(ci_target, str) and ci_target in target:
-                    return ci_target
-                elif isinstance(ci_target, list) and set(ci_target).issubset(set(target)):
-                    return ci_target
-            raise ValueError('ci_target must be a subset of target')
-
-        if os.getenv('CI_JOB_NAME') and 'ci_target' in kwargs:
-            kwargs['target'] = _filter_ci_target(kwargs['target'], kwargs['ci_target'])
         case_info.update(kwargs)
 
         @functools.wraps(test_func)
@@ -230,7 +215,7 @@ def test_method(**kwargs):
 
             dut_dict = kwargs['dut_dict']
             if target not in dut_dict:
-                raise Exception('target can only be {%s}' % ', '.join(dut_dict.keys()))
+                raise Exception('target can only be {%s} (case insensitive)' % ', '.join(dut_dict.keys()))
 
             dut = dut_dict[target]
             try:
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
index 6b584c31c0..b89379992f 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
@@ -143,6 +143,7 @@ class AssignTest(object):
     DEFAULT_FILTER = {
         "category": "function",
         "ignore": False,
+        "supported_in_ci": True,
     }
 
     def __init__(self, test_case_path, ci_config_file, case_group=Group):
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
index 6205a8e851..d016043aba 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
@@ -127,7 +127,6 @@ def filter_test_cases(test_methods, case_filter):
         * user case filter is ``chip: ["esp32", "esp32c"]``, case attribute is ``chip: "esp32"``
         * user case filter is ``chip: "esp32"``, case attribute is ``chip: "esp32"``
 
-
     :param test_methods: a list of test methods functions
     :param case_filter: case filter
     :return: filtered test methods
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
index 075f7a1cca..e9a8e29b49 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
@@ -106,6 +106,16 @@ class Search(object):
             key = replicate_config.pop()
             replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key])
 
+        # mark the cases with targets not in ci_target
+        for case in replicated_cases:
+            ci_target = case.case_info['ci_target']
+            if isinstance(ci_target, str):
+                ci_target = [ci_target]
+            if not ci_target or case.case_info['target'] in ci_target:
+                case.case_info['supported_in_ci'] = True
+            else:
+                case.case_info['supported_in_ci'] = False
+
         return replicated_cases
 
     @classmethod
diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py
index 5aa4f3426e..d5d94956da 100644
--- a/tools/ci/python_packages/ttfw_idf/__init__.py
+++ b/tools/ci/python_packages/ttfw_idf/__init__.py
@@ -11,9 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import functools
 import os
 import re
 
+from typing import Union
+
 from tiny_test_fw import TinyFW, Utility
 from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp  # noqa: export all Apps for users
 from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT  # noqa: export DUTs for users
@@ -30,6 +33,28 @@ def format_case_id(chip, case_name):
     return "{}.{}".format(chip, case_name)
 
 
+def upper_list(text):  # type: (Union[str, unicode, list]) -> list
+    if isinstance(text, basestring):  # It's not working in python3
+        res = [text.upper()]
+    else:
+        res = [item.upper() for item in text]
+    return res
+
+
+def ci_target_check(func):
+    @functools.wraps(func)
+    def wrapper(**kwargs):
+        target = upper_list(kwargs.get('target', []))
+        ci_target = upper_list(kwargs.get('ci_target', []))
+        if not set(ci_target).issubset(set(target)):
+            raise ValueError('ci_target must be a subset of target')
+
+        return func(**kwargs)
+
+    return wrapper
+
+
+@ci_target_check
 def idf_example_test(app=Example, target="ESP32", ci_target=None, module="examples", execution_time=1,
                      level="example", erase_nvs=True, config_name=None, **kwargs):
     """
@@ -48,7 +73,7 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
                                              execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
                                              erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
@@ -58,6 +83,7 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl
     return test
 
 
+@ci_target_check
 def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", execution_time=1,
                   level="unit", erase_nvs=True, **kwargs):
     """
@@ -75,7 +101,7 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
                                              execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
                                              erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)
@@ -85,6 +111,7 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex
     return test
 
 
+@ci_target_check
 def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc", execution_time=1,
                     level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
     """
@@ -104,7 +131,7 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc",
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module,
+        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
                                              execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
                                              erase_nvs=erase_nvs, **kwargs)
         test_func = original_method(func)

From 5c92d3607851b171de9ecec4dc40226c3d263ecb Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Thu, 30 Apr 2020 11:49:51 +0800
Subject: [PATCH 06/11] Add list support for ttfw_idf test decorators. Only
 replicate supported keys

---
 .../tiny_test_fw/Utility/CIAssignTest.py           | 13 ++++++++++++-
 .../tiny_test_fw/Utility/SearchCases.py            |  4 +++-
 tools/ci/python_packages/ttfw_idf/__init__.py      | 14 +++++++++++---
 3 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
index b89379992f..6277f07819 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
@@ -44,6 +44,7 @@ import re
 import json
 
 import yaml
+
 try:
     from yaml import CLoader as Loader
 except ImportError:
@@ -67,7 +68,7 @@ class Group(object):
         self.case_list = [case]
         self.filters = dict(zip(self.SORT_KEYS, [self._get_case_attr(case, x) for x in self.SORT_KEYS]))
         # we use ci_job_match_keys to match CI job tags. It's a set of required tags.
-        self.ci_job_match_keys = set([self._get_case_attr(case, x) for x in self.CI_JOB_MATCH_KEYS])
+        self.ci_job_match_keys = self._get_match_keys(case)
 
     @staticmethod
     def _get_case_attr(case, attr):
@@ -75,6 +76,16 @@ class Group(object):
         # this method will do get attribute form cases
         return case.case_info[attr]
 
+    def _get_match_keys(self, case):
+        keys = []
+        for attr in self.CI_JOB_MATCH_KEYS:
+            val = self._get_case_attr(case, attr)
+            if isinstance(val, list):
+                keys.extend(val)
+            else:
+                keys.append(val)
+        return set(keys)
+
     def accept_new_case(self):
         """
         check if allowed to add any case to this group
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
index e9a8e29b49..a77f3b5798 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
@@ -23,6 +23,7 @@ from . import load_source
 
 class Search(object):
     TEST_CASE_FILE_PATTERN = "*_test.py"
+    SUPPORT_REPLICATE_CASES_KEY = ['target']
 
     @classmethod
     def _search_cases_from_file(cls, file_name):
@@ -104,7 +105,8 @@ class Search(object):
             if not replicate_config:
                 break
             key = replicate_config.pop()
-            replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key])
+            if key in cls.SUPPORT_REPLICATE_CASES_KEY:
+                replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key])
 
         # mark the cases with targets not in ci_target
         for case in replicated_cases:
diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py
index d5d94956da..96d5f804d0 100644
--- a/tools/ci/python_packages/ttfw_idf/__init__.py
+++ b/tools/ci/python_packages/ttfw_idf/__init__.py
@@ -15,7 +15,7 @@ import functools
 import os
 import re
 
-from typing import Union
+from typing import Optional
 
 from tiny_test_fw import TinyFW, Utility
 from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp  # noqa: export all Apps for users
@@ -33,9 +33,17 @@ def format_case_id(chip, case_name):
     return "{}.{}".format(chip, case_name)
 
 
-def upper_list(text):  # type: (Union[str, unicode, list]) -> list
-    if isinstance(text, basestring):  # It's not working in python3
+try:
+    string_type = basestring
+except NameError:
+    string_type = str
+
+
+def upper_list(text):  # type: (Optional[string_type, list]) -> list
+    if isinstance(text, string_type):
         res = [text.upper()]
+    elif text is None:
+        res = []
     else:
         res = [item.upper() for item in text]
     return res

From d06d2a2b9f8f832fead244ed315c40ec524a2848 Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Thu, 30 Apr 2020 18:11:25 +0800
Subject: [PATCH 07/11] remove typing as a dependency.

return None if the input is None
---
 tools/ci/python_packages/ttfw_idf/__init__.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py
index 96d5f804d0..96fc07cdd5 100644
--- a/tools/ci/python_packages/ttfw_idf/__init__.py
+++ b/tools/ci/python_packages/ttfw_idf/__init__.py
@@ -15,8 +15,6 @@ import functools
 import os
 import re
 
-from typing import Optional
-
 from tiny_test_fw import TinyFW, Utility
 from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp  # noqa: export all Apps for users
 from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT  # noqa: export DUTs for users
@@ -39,11 +37,11 @@ except NameError:
     string_type = str
 
 
-def upper_list(text):  # type: (Optional[string_type, list]) -> list
+def upper_list(text):
+    if not text:
+        return text
     if isinstance(text, string_type):
         res = [text.upper()]
-    elif text is None:
-        res = []
     else:
         res = [item.upper() for item in text]
     return res

From e99172fbac3540c6d5afee74c182dc512fdd6d67 Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Wed, 6 May 2020 12:00:52 +0800
Subject: [PATCH 08/11] make upper_list do not make str into list

---
 tools/ci/python_packages/ttfw_idf/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py
index 96fc07cdd5..235dbdab2c 100644
--- a/tools/ci/python_packages/ttfw_idf/__init__.py
+++ b/tools/ci/python_packages/ttfw_idf/__init__.py
@@ -41,7 +41,7 @@ def upper_list(text):
     if not text:
         return text
     if isinstance(text, string_type):
-        res = [text.upper()]
+        res = text.upper()
     else:
         res = [item.upper() for item in text]
     return res

From 377d3eaaa5657522de6682439752022142ad4303 Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Fri, 29 May 2020 12:39:58 +0800
Subject: [PATCH 09/11] Move local target detection to `ttfw_idf`

rename upper_list to upper_list_or_str
minor fix for `unit_test.py` `is 'name'` -> `== 'name`
---
 .../ci/python_packages/tiny_test_fw/TinyFW.py | 47 +--------
 .../tiny_test_fw/Utility/CaseConfig.py        | 13 ++-
 .../tiny_test_fw/Utility/SearchCases.py       |  2 -
 tools/ci/python_packages/ttfw_idf/__init__.py | 97 +++++++++++++++----
 tools/unit-test-app/unit_test.py              | 10 +-
 5 files changed, 94 insertions(+), 75 deletions(-)

diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
index 37b8b8ed5b..6e2d03d38e 100644
--- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py
+++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py
@@ -13,8 +13,6 @@
 # limitations under the License.
 
 """ Interface for test cases. """
-import json
-import logging
 import os
 import time
 import traceback
@@ -65,6 +63,7 @@ class DefaultEnvConfig(object):
 set_default_config = DefaultEnvConfig.set_default_config
 get_default_config = DefaultEnvConfig.get_default_config
 
+
 MANDATORY_INFO = {
     "execution_time": 1,
     "env_tag": "default",
@@ -159,13 +158,11 @@ def test_method(**kwargs):
                                    In some cases, one test function might test many test cases.
                                    If this flag is set, test case can update junit report by its own.
     """
-
     def test(test_func):
 
         case_info = MANDATORY_INFO.copy()
         case_info["name"] = case_info["ID"] = test_func.__name__
         case_info["junit_report_by_case"] = False
-
         case_info.update(kwargs)
 
         @functools.wraps(test_func)
@@ -183,48 +180,7 @@ def test_method(**kwargs):
                 if key in env_config:
                     env_config[key] = kwargs[key]
 
-            # Runner.py should overwrite target with the current target.
             env_config.update(overwrite)
-
-            # if target not in the default_config or the overwrite, then take it from kwargs passed from the decorator
-            target = env_config['target'] if 'target' in env_config else kwargs['target']
-
-            # This code block is used to do run local test script target set check
-            if not os.getenv('CI_JOB_NAME'):
-                idf_target = 'ESP32'  # default if sdkconfig not found or not readable
-                expected_json_path = os.path.join('build', 'config', 'sdkconfig.json')
-                if os.path.exists(expected_json_path):
-                    sdkconfig = json.load(open(expected_json_path))
-                    try:
-                        idf_target = sdkconfig['IDF_TARGET'].upper()
-                    except KeyError:
-                        pass
-                    else:
-                        logging.info('IDF_TARGET: {}'.format(idf_target))
-                else:
-                    logging.warning('{} not found. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
-
-                if isinstance(target, list):
-                    if idf_target in target:
-                        target = idf_target
-                    else:
-                        raise ValueError('IDF_TARGET set to {}, not in decorator target value'.format(idf_target))
-                else:
-                    if idf_target != target:
-                        raise ValueError('IDF_TARGET set to {}, not equal to decorator target value'.format(idf_target))
-
-            dut_dict = kwargs['dut_dict']
-            if target not in dut_dict:
-                raise Exception('target can only be {%s} (case insensitive)' % ', '.join(dut_dict.keys()))
-
-            dut = dut_dict[target]
-            try:
-                # try to config the default behavior of erase nvs
-                dut.ERASE_NVS = kwargs['erase_nvs']
-            except AttributeError:
-                pass
-
-            env_config['dut'] = dut
             env_inst = Env.Env(**env_config)
 
             # prepare for xunit test results
@@ -271,5 +227,4 @@ def test_method(**kwargs):
         handle_test.case_info = case_info
         handle_test.test_method = True
         return handle_test
-
     return test
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
index d016043aba..636af08ae0 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
@@ -45,6 +45,7 @@ Template Config File::
 import importlib
 
 import yaml
+
 try:
     from yaml import CLoader as Loader
 except ImportError:
@@ -194,12 +195,22 @@ class Parser(object):
             try:
                 _target = _filter['target']
             except KeyError:
-                pass
+                _target = None
             else:
                 _overwrite.update({'target': _target})
 
             for test_method in test_methods:
                 if _filter_one_case(test_method, _filter):
+                    try:
+                        dut_dict = test_method.case_info['dut_dict']
+                    except (AttributeError, KeyError):
+                        dut_dict = None
+
+                    if dut_dict and _target:
+                        if _target.upper() in dut_dict:
+                            _overwrite.update({'dut': dut_dict[_target.upper()]})
+                        else:
+                            raise ValueError('target {} is not in the specified dut_dict'.format(_target))
                     test_case_list.append(TestCase.TestCase(test_method, _extra_data, **_overwrite))
         return test_case_list
 
diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
index a77f3b5798..699bb57ece 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
@@ -111,8 +111,6 @@ class Search(object):
         # mark the cases with targets not in ci_target
         for case in replicated_cases:
             ci_target = case.case_info['ci_target']
-            if isinstance(ci_target, str):
-                ci_target = [ci_target]
             if not ci_target or case.case_info['target'] in ci_target:
                 case.case_info['supported_in_ci'] = True
             else:
diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py
index 235dbdab2c..f969ec8de1 100644
--- a/tools/ci/python_packages/ttfw_idf/__init__.py
+++ b/tools/ci/python_packages/ttfw_idf/__init__.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import functools
+import json
+import logging
 import os
 import re
 
@@ -27,8 +29,8 @@ TARGET_DUT_CLS_DICT = {
 }
 
 
-def format_case_id(chip, case_name):
-    return "{}.{}".format(chip, case_name)
+def format_case_id(target, case_name):
+    return "{}.{}".format(target, case_name)
 
 
 try:
@@ -37,21 +39,66 @@ except NameError:
     string_type = str
 
 
-def upper_list(text):
-    if not text:
-        return text
+def upper_list_or_str(text):
+    """
+    Return the uppercase of list of string or string. Return itself for other
+    data types
+    :param text: list or string, other instance will be returned immediately
+    :return: uppercase of list of string
+    """
     if isinstance(text, string_type):
-        res = text.upper()
+        return [text.upper()]
+    elif isinstance(text, list):
+        return [item.upper() for item in text]
     else:
-        res = [item.upper() for item in text]
-    return res
+        return text
+
+
+def local_test_check(decorator_target):
+    # Try to get the sdkconfig.json to read the IDF_TARGET value.
+    # If not set, will set to ESP32.
+    # For CI jobs, this is a fake procedure, the true target and dut will be
+    # overwritten by the job config YAML file.
+    idf_target = 'ESP32'  # default if sdkconfig not found or not readable
+    if os.getenv('CI_JOB_ID'):  # Only auto-detect target when running locally
+        return idf_target
+
+    expected_json_path = os.path.join('build', 'config', 'sdkconfig.json')
+    if os.path.exists(expected_json_path):
+        sdkconfig = json.load(open(expected_json_path))
+        try:
+            idf_target = sdkconfig['IDF_TARGET'].upper()
+        except KeyError:
+            logging.warning('IDF_TARGET not in {}. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
+        else:
+            logging.info('IDF_TARGET: {}'.format(idf_target))
+    else:
+        logging.warning('{} not found. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
+
+    if isinstance(decorator_target, list):
+        if idf_target not in decorator_target:
+            raise ValueError('IDF_TARGET set to {}, not in decorator target value'.format(idf_target))
+    else:
+        if idf_target != decorator_target:
+            raise ValueError('IDF_TARGET set to {}, not equal to decorator target value'.format(idf_target))
+    return idf_target
+
+
+def get_dut_class(target, erase_nvs=None):
+    if target not in TARGET_DUT_CLS_DICT:
+        raise Exception('target can only be {%s} (case insensitive)' % ', '.join(TARGET_DUT_CLS_DICT.keys()))
+
+    dut = TARGET_DUT_CLS_DICT[target.upper()]
+    if erase_nvs:
+        dut.ERASE_NVS = 'erase_nvs'
+    return dut
 
 
 def ci_target_check(func):
     @functools.wraps(func)
     def wrapper(**kwargs):
-        target = upper_list(kwargs.get('target', []))
-        ci_target = upper_list(kwargs.get('ci_target', []))
+        target = upper_list_or_str(kwargs.get('target', []))
+        ci_target = upper_list_or_str(kwargs.get('ci_target', []))
         if not set(ci_target).issubset(set(target)):
             raise ValueError('ci_target must be a subset of target')
 
@@ -79,9 +126,13 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
-                                             execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
-                                             erase_nvs=erase_nvs, **kwargs)
+        test_target = local_test_check(target)
+        dut = get_dut_class(test_target, erase_nvs)
+        original_method = TinyFW.test_method(
+            app=app, dut=dut, target=upper_list_or_str(target), ci_target=upper_list_or_str(ci_target),
+            module=module, execution_time=execution_time, level=level, erase_nvs=erase_nvs,
+            dut_dict=TARGET_DUT_CLS_DICT, **kwargs
+        )
         test_func = original_method(func)
         test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
@@ -107,9 +158,13 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
-                                             execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
-                                             erase_nvs=erase_nvs, **kwargs)
+        test_target = local_test_check(target)
+        dut = get_dut_class(test_target, erase_nvs)
+        original_method = TinyFW.test_method(
+            app=app, dut=dut, target=upper_list_or_str(target), ci_target=upper_list_or_str(ci_target),
+            module=module, execution_time=execution_time, level=level, erase_nvs=erase_nvs,
+            dut_dict=TARGET_DUT_CLS_DICT, **kwargs
+        )
         test_func = original_method(func)
         test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
@@ -137,9 +192,13 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc",
     """
 
     def test(func):
-        original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module,
-                                             execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT,
-                                             erase_nvs=erase_nvs, **kwargs)
+        test_target = local_test_check(target)
+        dut = get_dut_class(test_target, erase_nvs)
+        original_method = TinyFW.test_method(
+            app=app, dut=dut, target=upper_list_or_str(target), ci_target=upper_list_or_str(ci_target),
+            module=module, execution_time=execution_time, level=level, erase_nvs=erase_nvs,
+            dut_dict=TARGET_DUT_CLS_DICT, **kwargs
+        )
         test_func = original_method(func)
         test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
         return test_func
diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py
index 564f8e22c3..8afcfb6e62 100755
--- a/tools/unit-test-app/unit_test.py
+++ b/tools/unit-test-app/unit_test.py
@@ -26,7 +26,6 @@ import threading
 from tiny_test_fw import TinyFW, Utility, Env, DUT
 import ttfw_idf
 
-
 UT_APP_BOOT_UP_DONE = "Press ENTER to see the list of tests."
 
 # matches e.g.: "rst:0xc (SW_CPU_RESET),boot:0x13 (SPI_FAST_FLASH_BOOT)"
@@ -188,12 +187,11 @@ def reset_dut(dut):
 
 
 def log_test_case(description, test_case, ut_config):
-            Utility.console_log("Running {} '{}' (config {})".format(description, test_case["name"], ut_config), color="orange")
-            Utility.console_log("Tags: %s" % ", ".join("%s=%s" % (k,v) for (k,v) in test_case.items() if k != "name" and v is not None), color="orange")
+    Utility.console_log("Running {} '{}' (config {})".format(description, test_case["name"], ut_config), color="orange")
+    Utility.console_log("Tags: %s" % ", ".join("%s=%s" % (k, v) for (k, v) in test_case.items() if k != "name" and v is not None), color="orange")
 
 
 def run_one_normal_case(dut, one_case, junit_test_case):
-
     reset_dut(dut)
 
     dut.start_capture_raw_data()
@@ -330,7 +328,6 @@ def run_unit_test_cases(env, extra_data):
 
 
 class Handler(threading.Thread):
-
     WAIT_SIGNAL_PATTERN = re.compile(r'Waiting for signal: \[(.+)]!')
     SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[([^]]+)](\[([^]]+)])?!')
     FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
@@ -691,7 +688,6 @@ def run_multiple_stage_cases(env, extra_data):
 
 
 def detect_update_unit_test_info(env, extra_data, app_bin):
-
     case_config = format_test_case_config(extra_data)
 
     for ut_config in case_config:
@@ -785,7 +781,7 @@ if __name__ == '__main__':
             if len(test_item) == 0:
                 continue
             pair = test_item.split(r':', 1)
-            if len(pair) == 1 or pair[0] is 'name':
+            if len(pair) == 1 or pair[0] == 'name':
                 test_dict['name'] = pair[0]
             elif len(pair) == 2:
                 if pair[0] == 'timeout' or pair[0] == 'child case num':

From 3b17b9e3935fdbe9ae6ba72b71055c7a3a48b66d Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Mon, 1 Jun 2020 10:26:38 +0800
Subject: [PATCH 10/11] panic_test contains some reassign dut cases

---
 tools/ci/python_packages/ttfw_idf/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py
index f969ec8de1..23114ef0c8 100644
--- a/tools/ci/python_packages/ttfw_idf/__init__.py
+++ b/tools/ci/python_packages/ttfw_idf/__init__.py
@@ -194,6 +194,9 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc",
     def test(func):
         test_target = local_test_check(target)
         dut = get_dut_class(test_target, erase_nvs)
+        if 'dut' in kwargs:  # panic_test() will inject dut, resolve conflicts here
+            dut = kwargs['dut']
+            del kwargs['dut']
         original_method = TinyFW.test_method(
             app=app, dut=dut, target=upper_list_or_str(target), ci_target=upper_list_or_str(ci_target),
             module=module, execution_time=execution_time, level=level, erase_nvs=erase_nvs,

From b76df309155c15be0e9574cd9992e2f6cf5cb48a Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Mon, 1 Jun 2020 13:39:09 +0800
Subject: [PATCH 11/11] don't overwrite special cases dut

---
 tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
index 636af08ae0..1334ab3ea1 100644
--- a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
@@ -207,8 +207,10 @@ class Parser(object):
                         dut_dict = None
 
                     if dut_dict and _target:
+                        dut = test_method.case_info.get('dut')
                         if _target.upper() in dut_dict:
-                            _overwrite.update({'dut': dut_dict[_target.upper()]})
+                            if dut and dut in dut_dict.values():  # don't overwrite special cases
+                                _overwrite.update({'dut': dut_dict[_target.upper()]})
                         else:
                             raise ValueError('target {} is not in the specified dut_dict'.format(_target))
                     test_case_list.append(TestCase.TestCase(test_method, _extra_data, **_overwrite))