feedc0de/esp-idf
1
0
Fork 0
forked from espressif/esp-idf
Code Pull Requests Activity

ci: make unity test case known failure work!

Browse Source 
Besides, it would also improve the test report summary
This commit is contained in:
Fu Hanxi
2023-02-23 10:16:53 +08:00
parent 2c8c8bfd2d
commit a25176d963
1 changed files with 38 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
conftest.py
View File

@@ -421,7 +421,9 @@ def pytest_addoption(parser: pytest.Parser) -> None:
base_group.addoption('--known-failure-cases-file', help='known failure cases file path') base_group.addoption('--known-failure-cases-file', help='known failure cases file path')
_idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded'] _idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded']()
_item_failed_cases_key = pytest.StashKey[list]()
_item_failed_key = pytest.StashKey[bool]()
def pytest_configure(config: Config) -> None: def pytest_configure(config: Config) -> None:
@@ -567,11 +569,31 @@ class IdfPytestEmbedded:
def pytest_runtest_makereport(self, item: Function, call: CallInfo[None]) -> Optional[TestReport]: def pytest_runtest_makereport(self, item: Function, call: CallInfo[None]) -> Optional[TestReport]:
report = TestReport.from_item_and_call(item, call) report = TestReport.from_item_and_call(item, call)
if item.stash.get(_item_failed_key, None) is None:
item.stash[_item_failed_key] = False
if report.outcome == 'failed': if report.outcome == 'failed':
test_case_name = item.funcargs.get('test_case_name', '') # Mark the failed test cases
is_known_failure = self._is_known_failure(test_case_name) #
is_xfail = report.keywords.get('xfail', False) # This hook function would be called in 3 phases, setup, call, teardown.
self._failed_cases.append((test_case_name, is_known_failure, is_xfail)) # the report.outcome is the outcome of the single call of current phase, which is independent
# the call phase outcome is the test result
item.stash[_item_failed_key] = True
if call.when == 'teardown':
item_failed = item.stash[_item_failed_key]
if item_failed:
# unity real test cases
failed_sub_cases = item.stash.get(_item_failed_cases_key, [])
if failed_sub_cases:
for test_case_name in failed_sub_cases:
self._failed_cases.append((test_case_name, self._is_known_failure(test_case_name), False))
else: # the case iteself is failing
test_case_name = item.funcargs.get('test_case_name', '')
if test_case_name:
self._failed_cases.append(
(test_case_name, self._is_known_failure(test_case_name), report.keywords.get('xfail', False))
)
return report return report
@@ -596,17 +618,27 @@ class IdfPytestEmbedded:
if not junits: if not junits:
return return
failed_sub_cases = []
target = item.funcargs['target'] target = item.funcargs['target']
config = item.funcargs['config'] config = item.funcargs['config']
for junit in junits: for junit in junits:
xml = ET.parse(junit) xml = ET.parse(junit)
testcases = xml.findall('.//testcase') testcases = xml.findall('.//testcase')
for case in testcases: for case in testcases:
case.attrib['name'] = format_case_id(target, config, case.attrib['name']) # modify the junit files
new_case_name = format_case_id(target, config, case.attrib['name'])
case.attrib['name'] = new_case_name
if 'file' in case.attrib: if 'file' in case.attrib:
case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework
# collect real failure cases
if case.find('failure') is not None:
failed_sub_cases.append(new_case_name)
xml.write(junit) xml.write(junit)
item.stash[_item_failed_cases_key] = failed_sub_cases
def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None:
if exitstatus != 0: if exitstatus != 0:
if exitstatus == ExitCode.NO_TESTS_COLLECTED: if exitstatus == ExitCode.NO_TESTS_COLLECTED: