diff --git a/tools/ci/python_packages/tiny_test_fw/bin/Runner.py b/tools/ci/python_packages/tiny_test_fw/bin/Runner.py index fc3c931c64..20e19cb3eb 100644 --- a/tools/ci/python_packages/tiny_test_fw/bin/Runner.py +++ b/tools/ci/python_packages/tiny_test_fw/bin/Runner.py @@ -26,8 +26,8 @@ import os import sys import threading -from tiny_test_fw import TinyFW -from tiny_test_fw.Utility import CaseConfig, SearchCases +from tiny_test_fw.TinyFW import JunitReport, set_default_config +from tiny_test_fw.Utility import CaseConfig, SearchCases, console_log class Runner(threading.Thread): @@ -37,25 +37,52 @@ class Runner(threading.Thread): :param env_config_file: env config file """ - def __init__(self, test_case_paths, case_config, env_config_file=None): + def __init__(self, test_case_paths, case_config, env_config_file=None, known_failure_cases_file=None): super(Runner, self).__init__() self.setDaemon(True) if case_config: test_suite_name = os.path.splitext(os.path.basename(case_config))[0] else: test_suite_name = 'TestRunner' - TinyFW.set_default_config(env_config_file=env_config_file, test_suite_name=test_suite_name) + set_default_config(env_config_file=env_config_file, test_suite_name=test_suite_name) test_methods = SearchCases.Search.search_test_cases(test_case_paths) self.test_cases = CaseConfig.Parser.apply_config(test_methods, case_config) - self.test_result = [] + self.known_failure_cases = self._get_config_cases(known_failure_cases_file) + + @staticmethod + def _get_config_cases(config_file): + res = set() + if not config_file or os.path.isfile(config_file): + return res + + for line in open(config_file).readlines(): + if not line: + continue + if not line.strip(): + continue + without_comments = line.split('#')[0].strip() + if without_comments: + res.add(without_comments) + return res def run(self): for case in self.test_cases: - result = case.run() - self.test_result.append(result) + case.run() def get_test_result(self): - return self.test_result and all(self.test_result) + _res = True + console_log('Test Results:') + for tc in JunitReport.JUNIT_TEST_SUITE.test_cases: + if tc.failures: + if tc.name in self.known_failure_cases: + console_log(' Known Failure: ' + tc.name, color='orange') + else: + console_log(' Test Fail: ' + tc.name, color='red') + _res = False + else: + console_log(' Test Succeed: ' + tc.name, color='green') + + return _res if __name__ == '__main__': @@ -66,10 +93,13 @@ if __name__ == '__main__': help='case filter/config file') parser.add_argument('--env_config_file', '-e', default=None, help='test env config file') + parser.add_argument('--known_failure_cases_file', default=None, + help='known failure cases file') args = parser.parse_args() - test_cases = [os.path.join(os.getenv('IDF_PATH'), path) if not os.path.isabs(path) else path for path in args.test_cases] - runner = Runner(test_cases, args.case_config, args.env_config_file) + test_cases = [os.path.join(os.getenv('IDF_PATH'), path) + if not os.path.isabs(path) else path for path in args.test_cases] + runner = Runner(test_cases, args.case_config, args.env_config_file, args.known_failure_cases_file) runner.start() while True: