espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-04 13:14:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ci/known-failure-fast-link' into 'master'

Browse Source 
ci: added known failure cases fast link to report

Closes RDT-1026

See merge request espressif/esp-idf!35566
This commit is contained in:
Igor Udot
2024-12-20 09:56:42 +08:00
parent b5ac4fbdf9 2c6e82c644
commit ca927b87d9
4 changed files with 53 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tools/ci/dynamic_pipelines/report.py
View File

@@ -31,6 +31,7 @@ from .utils import format_permalink
from .utils import get_artifacts_url from .utils import get_artifacts_url
from .utils import get_repository_file_url from .utils import get_repository_file_url
from .utils import is_url from .utils import is_url
from .utils import known_failure_issue_jira_fast_link
from .utils import load_known_failure_cases from .utils import load_known_failure_cases
@@ -542,6 +543,7 @@ class TargetTestReportGenerator(ReportGenerator):
'Failure Reason', 'Failure Reason',
f'Failures on your branch (40 latest testcases)', f'Failures on your branch (40 latest testcases)',
'Dut Log URL', 'Dut Log URL',
'Create Known Failure Case Jira',
'Job URL', 'Job URL',
'Grafana URL', 'Grafana URL',
], ],
@@ -550,6 +552,10 @@ class TargetTestReportGenerator(ReportGenerator):
( (
'Failures on your branch (40 latest testcases)', 'Failures on your branch (40 latest testcases)',
lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}",
),
(
'Create Known Failure Case Jira',
known_failure_issue_jira_fast_link
) )
], ],
) )
@@ -562,6 +568,7 @@ class TargetTestReportGenerator(ReportGenerator):
'Failure Reason', 'Failure Reason',
'Cases that failed in other branches as well (40 latest testcases)', 'Cases that failed in other branches as well (40 latest testcases)',
'Dut Log URL', 'Dut Log URL',
'Create Known Failure Case Jira',
'Job URL', 'Job URL',
'Grafana URL', 'Grafana URL',
], ],
@@ -570,6 +577,10 @@ class TargetTestReportGenerator(ReportGenerator):
( (
'Cases that failed in other branches as well (40 latest testcases)', 'Cases that failed in other branches as well (40 latest testcases)',
lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}",
),
(
'Create Known Failure Case Jira',
known_failure_issue_jira_fast_link
) )
], ],
) )

8
tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html
View File

@@ -63,6 +63,7 @@
<th>Failure Reason</th> <th>Failure Reason</th>
<th>Cases that failed in other branches as well (40 latest testcases)</th> <th>Cases that failed in other branches as well (40 latest testcases)</th>
<th>Dut Log URL</th> <th>Dut Log URL</th>
<th>Create Known Failure Case Jira</th>
<th>Job URL</th> <th>Job URL</th>
<th>Grafana URL</th> <th>Grafana URL</th>
</tr> </tr>
@@ -74,6 +75,7 @@
<td>failed on setup with "EOFError"</td> <td>failed on setup with "EOFError"</td>
<td>0 / 40</td> <td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device/dut.txt">link</a></td> <td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device/dut.txt">link</a></td>
<td><a href="https://jira.com/secure/CreateIssueDetails!init.jspa?pid=10514&issuetype=10004&summary=%5BTest+Case%5D%28%27esp32h2%27%2C+%27esp32h2%27%29.%28%27defaults%27%2C+%27defaults%27%29.test_i2c_multi_device&description=job_url%3A+Not%2520found%0A%0Adut_log_url%3A+https%3A%2F%2Furl%2Fesp%2Fesp-idf%2Fpytest-embedded%2F2024-07-01_10-53-05-207900%2F%2528%2527esp32h2%2527%252C%2520%2527esp32h2%2527%2529.%2528%2527defaults%2527%252C%2520%2527defaults%2527%2529.test_i2c_multi_device%2Fdut.txt%0A%0Aci_dashboard_url%3A+https%3A%2F%2Ftest_dashboard_host%2Fd%2FUcg477Fnz%2Fcase-list%3Fvar-case_id%3D%2528%2527esp32h2%2527%252C%2520%2527esp32h2%2527%2529.%2528%2527defaults%2527%252C%2520%2527defaults%2527%2529.test_i2c_multi_device%0A%0A&components=11909&priority=3&assignee=zhangjianwen&versions=17602">Create</a></td>
<td>Not found</td> <td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=%28%27esp32h2%27%2C%20%27esp32h2%27%29.%28%27defaults%27%2C%20%27defaults%27%29.test_i2c_multi_device">link</a></td> <td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=%28%27esp32h2%27%2C%20%27esp32h2%27%29.%28%27defaults%27%2C%20%27defaults%27%29.test_i2c_multi_device">link</a></td>
</tr> </tr>
@@ -83,6 +85,7 @@
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt</td> <td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt</td>
<td>0 / 40</td> <td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_esp_timer/dut.txt">link</a></td> <td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_esp_timer/dut.txt">link</a></td>
<td><a href="https://jira.com/secure/CreateIssueDetails!init.jspa?pid=10514&issuetype=10004&summary=%5BTest+Case%5Desp32c3.release.test_esp_timer&description=job_url%3A+Not%2520found%0A%0Adut_log_url%3A+https%3A%2F%2Furl%2Fesp%2Fesp-idf%2Fpytest-embedded%2F2024-07-01_10-53-05-207900%2Fesp32c3.release.test_esp_timer%2Fdut.txt%0A%0Aci_dashboard_url%3A+https%3A%2F%2Ftest_dashboard_host%2Fd%2FUcg477Fnz%2Fcase-list%3Fvar-case_id%3Desp32c3.release.test_esp_timer%0A%0A&components=11909&priority=3&assignee=zhangjianwen&versions=17602">Create</a></td>
<td>Not found</td> <td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.release.test_esp_timer">link</a></td> <td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.release.test_esp_timer">link</a></td>
</tr> </tr>
@@ -92,6 +95,7 @@
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt</td> <td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt</td>
<td>0 / 40</td> <td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.default.test_wpa_supplicant_ut/dut.txt">link</a></td> <td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.default.test_wpa_supplicant_ut/dut.txt">link</a></td>
<td><a href="https://jira.com/secure/CreateIssueDetails!init.jspa?pid=10514&issuetype=10004&summary=%5BTest+Case%5Desp32c3.default.test_wpa_supplicant_ut&description=job_url%3A+Not%2520found%0A%0Adut_log_url%3A+https%3A%2F%2Furl%2Fesp%2Fesp-idf%2Fpytest-embedded%2F2024-07-01_10-53-05-207900%2Fesp32c3.default.test_wpa_supplicant_ut%2Fdut.txt%0A%0Aci_dashboard_url%3A+https%3A%2F%2Ftest_dashboard_host%2Fd%2FUcg477Fnz%2Fcase-list%3Fvar-case_id%3Desp32c3.default.test_wpa_supplicant_ut%0A%0A&components=11909&priority=3&assignee=zhangjianwen&versions=17602">Create</a></td>
<td>Not found</td> <td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.default.test_wpa_supplicant_ut">link</a></td> <td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.default.test_wpa_supplicant_ut">link</a></td>
</tr> </tr>
@@ -101,6 +105,7 @@
<td>failed on setup with "EOFError"</td> <td>failed on setup with "EOFError"</td>
<td>3 / 40</td> <td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev/dut.txt">link</a></td> <td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev/dut.txt">link</a></td>
<td><a href="https://jira.com/secure/CreateIssueDetails!init.jspa?pid=10514&issuetype=10004&summary=%5BTest+Case%5D%28%27esp32h2%27%2C+%27esp32h2%27%29.%28%27default%27%2C+%27default%27%29.test_i2s_multi_dev&description=job_url%3A+Not%2520found%0A%0Adut_log_url%3A+https%3A%2F%2Furl%2Fesp%2Fesp-idf%2Fpytest-embedded%2F2024-07-01_10-53-05-207900%2F%2528%2527esp32h2%2527%252C%2520%2527esp32h2%2527%2529.%2528%2527default%2527%252C%2520%2527default%2527%2529.test_i2s_multi_dev%2Fdut.txt%0A%0Aci_dashboard_url%3A+https%3A%2F%2Ftest_dashboard_host%2Fd%2FUcg477Fnz%2Fcase-list%3Fvar-case_id%3D%2528%2527esp32h2%2527%252C%2520%2527esp32h2%2527%2529.%2528%2527default%2527%252C%2520%2527default%2527%2529.test_i2s_multi_dev%0A%0A&components=11909&priority=3&assignee=zhangjianwen&versions=17602">Create</a></td>
<td>Not found</td> <td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=%28%27esp32h2%27%2C%20%27esp32h2%27%29.%28%27default%27%2C%20%27default%27%29.test_i2s_multi_dev">link</a></td> <td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=%28%27esp32h2%27%2C%20%27esp32h2%27%29.%28%27default%27%2C%20%27default%27%29.test_i2s_multi_dev">link</a></td>
</tr> </tr>
@@ -110,6 +115,7 @@
<td>AssertionError: Unity test failed</td> <td>AssertionError: Unity test failed</td>
<td>3 / 40</td> <td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.default.test_wpa_supplicant_ut/dut.txt">link</a></td> <td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.default.test_wpa_supplicant_ut/dut.txt">link</a></td>
<td><a href="https://jira.com/secure/CreateIssueDetails!init.jspa?pid=10514&issuetype=10004&summary=%5BTest+Case%5Desp32c2.default.test_wpa_supplicant_ut&description=job_url%3A+Not%2520found%0A%0Adut_log_url%3A+https%3A%2F%2Furl%2Fesp%2Fesp-idf%2Fpytest-embedded%2F2024-07-01_10-53-05-207900%2Fesp32c2.default.test_wpa_supplicant_ut%2Fdut.txt%0A%0Aci_dashboard_url%3A+https%3A%2F%2Ftest_dashboard_host%2Fd%2FUcg477Fnz%2Fcase-list%3Fvar-case_id%3Desp32c2.default.test_wpa_supplicant_ut%0A%0A&components=11909&priority=3&assignee=zhangjianwen&versions=17602">Create</a></td>
<td>Not found</td> <td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.default.test_wpa_supplicant_ut">link</a></td> <td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.default.test_wpa_supplicant_ut">link</a></td>
</tr> </tr>
@@ -119,6 +125,7 @@
<td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt</td> <td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt</td>
<td>3 / 40</td> <td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.512safe.test_wear_levelling/dut.txt">link</a></td> <td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.512safe.test_wear_levelling/dut.txt">link</a></td>
<td><a href="https://jira.com/secure/CreateIssueDetails!init.jspa?pid=10514&issuetype=10004&summary=%5BTest+Case%5Desp32c3.512safe.test_wear_levelling&description=job_url%3A+Not%2520found%0A%0Adut_log_url%3A+https%3A%2F%2Furl%2Fesp%2Fesp-idf%2Fpytest-embedded%2F2024-07-01_10-53-05-207900%2Fesp32c3.512safe.test_wear_levelling%2Fdut.txt%0A%0Aci_dashboard_url%3A+https%3A%2F%2Ftest_dashboard_host%2Fd%2FUcg477Fnz%2Fcase-list%3Fvar-case_id%3Desp32c3.512safe.test_wear_levelling%0A%0A&components=11909&priority=3&assignee=zhangjianwen&versions=17602">Create</a></td>
<td>Not found</td> <td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.512safe.test_wear_levelling">link</a></td> <td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.512safe.test_wear_levelling">link</a></td>
</tr> </tr>
@@ -128,6 +135,7 @@
<td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt</td> <td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt</td>
<td>3 / 40</td> <td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_wear_levelling/dut.txt">link</a></td> <td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_wear_levelling/dut.txt">link</a></td>
<td><a href="https://jira.com/secure/CreateIssueDetails!init.jspa?pid=10514&issuetype=10004&summary=%5BTest+Case%5Desp32c3.release.test_wear_levelling&description=job_url%3A+Not%2520found%0A%0Adut_log_url%3A+https%3A%2F%2Furl%2Fesp%2Fesp-idf%2Fpytest-embedded%2F2024-07-01_10-53-05-207900%2Fesp32c3.release.test_wear_levelling%2Fdut.txt%0A%0Aci_dashboard_url%3A+https%3A%2F%2Ftest_dashboard_host%2Fd%2FUcg477Fnz%2Fcase-list%3Fvar-case_id%3Desp32c3.release.test_wear_levelling%0A%0A&components=11909&priority=3&assignee=zhangjianwen&versions=17602">Create</a></td>
<td>Not found</td> <td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.release.test_wear_levelling">link</a></td> <td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.release.test_wear_levelling">link</a></td>
</tr> </tr>

1
tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py
View File

@@ -32,6 +32,7 @@ class TestReportGeneration(unittest.TestCase):
'CI_DASHBOARD_HOST': 'https://test_dashboard_host', 'CI_DASHBOARD_HOST': 'https://test_dashboard_host',
'CI_PAGES_URL': 'https://artifacts_path', 'CI_PAGES_URL': 'https://artifacts_path',
'CI_JOB_ID': '1', 'CI_JOB_ID': '1',
'JIRA_SERVER': 'https://jira.com',
}) })
self.MockGitlab = self.gitlab_patcher.start() self.MockGitlab = self.gitlab_patcher.start()

33
tools/ci/dynamic_pipelines/utils.py
View File

@@ -5,6 +5,8 @@ import os
import re import re
import typing as t import typing as t
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from urllib.parse import quote
from urllib.parse import urlencode
from urllib.parse import urlparse from urllib.parse import urlparse
import requests import requests
@@ -216,3 +218,34 @@ def get_repository_file_url(file_path: str) -> str:
:return: The modified URL pointing to the file's path in the repository. :return: The modified URL pointing to the file's path in the repository.
""" """
return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}' return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}'
def known_failure_issue_jira_fast_link(_item: TestCase) -> str:
"""
Generate a JIRA fast link for known issues with relevant test case details.
"""
jira_url = os.getenv('JIRA_SERVER')
jira_pid = os.getenv('JIRA_KNOWN_FAILURE_PID', '10514')
jira_issuetype = os.getenv('JIRA_KNOWN_FAILURE_ISSUETYPE', '10004')
jira_component = os.getenv('JIRA_KNOWN_FAILURE_COMPONENT', '11909')
jira_assignee = os.getenv('JIRA_KNOWN_FAILURE_ASSIGNEE', 'zhangjianwen')
jira_affected_versions = os.getenv('JIRA_KNOWN_FAILURE_VERSIONS', '17602')
jira_priority = os.getenv('JIRA_KNOWN_FAILURE_PRIORITY', '3')
base_url = f'{jira_url}/secure/CreateIssueDetails!init.jspa?'
params = {
'pid': jira_pid,
'issuetype': jira_issuetype,
'summary': f'[Test Case]{_item.name}',
'description': (
f"job_url: {quote(_item.ci_job_url, safe=':/')}\n\n"
f"dut_log_url: {quote(_item.dut_log_url, safe=':/')}\n\n"
f'ci_dashboard_url: {_item.ci_dashboard_url}\n\n'
),
'components': jira_component,
'priority': jira_priority,
'assignee': jira_assignee,
'versions': jira_affected_versions
}
query_string = urlencode(params)
return f'<a href="{base_url}{query_string}">Create</a>'