diff --git a/conftest.py b/conftest.py index 850a49faa0..06699f2e6a 100644 --- a/conftest.py +++ b/conftest.py @@ -39,7 +39,7 @@ from _pytest.fixtures import FixtureRequest from artifacts_handler import ArtifactType from dynamic_pipelines.constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME from idf_ci.app import import_apps_from_txt -from idf_ci.uploader import AppUploader +from idf_ci.uploader import AppDownloader, AppUploader from idf_ci_utils import IDF_PATH from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS, PytestCase from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, ITEM_PYTEST_CASE_KEY, IdfPytestEmbedded @@ -81,7 +81,7 @@ def config(request: FixtureRequest) -> str: @pytest.fixture @multi_dut_fixture -def target(request: FixtureRequest, dut_total: int, dut_index: int) -> str: +def target(request: FixtureRequest, dut_total: int, dut_index: int) -> str: plugin = request.config.stash[IDF_PYTEST_EMBEDDED_KEY] if dut_total == 1: @@ -116,21 +116,14 @@ def pipeline_id(request: FixtureRequest) -> t.Optional[str]: return request.config.getoption('pipeline_id', None) or os.getenv('PARENT_PIPELINE_ID', None) # type: ignore -class BuildReportDownloader: +class BuildReportDownloader(AppDownloader): def __init__(self, presigned_url_yaml: str) -> None: self.app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = yaml.safe_load(presigned_url_yaml) - def download_app( - self, app_build_path: str, artifact_type: ArtifactType = ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) -> None: - if app_build_path not in self.app_presigned_urls_dict: - raise ValueError(f'No presigned url found for {app_build_path}. ' - f'Usually this should not happen, please re-trigger a pipeline.' - f'If this happens again, please report this bug to the CI channel.') - + def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: url = self.app_presigned_urls_dict[app_build_path][artifact_type.value] - logging.debug('Downloading app from %s', url) + logging.info('Downloading app from %s', url) with io.BytesIO() as f: for chunk in requests.get(url).iter_content(chunk_size=1024 * 1024): if chunk: @@ -141,9 +134,19 @@ class BuildReportDownloader: with zipfile.ZipFile(f) as zip_ref: zip_ref.extractall(IDF_PATH) + def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: + if app_build_path not in self.app_presigned_urls_dict: + raise ValueError( + f'No presigned url found for {app_build_path}. ' + f'Usually this should not happen, please re-trigger a pipeline.' + f'If this happens again, please report this bug to the CI channel.' + ) + + super().download_app(app_build_path, artifact_type) + @pytest.fixture(scope='session') -def app_downloader(pipeline_id: t.Optional[str]) -> t.Union[AppUploader, BuildReportDownloader, None]: +def app_downloader(pipeline_id: t.Optional[str]) -> t.Optional[AppDownloader]: if not pipeline_id: return None @@ -199,7 +202,7 @@ def build_dir( app_path: str, target: t.Optional[str], config: t.Optional[str], - app_downloader: t.Optional[AppUploader], + app_downloader: t.Optional[AppDownloader], ) -> str: """ Check local build dir with the following priority: diff --git a/tools/ci/idf_ci/uploader.py b/tools/ci/idf_ci/uploader.py index ad91fa619e..4d55ac304c 100644 --- a/tools/ci/idf_ci/uploader.py +++ b/tools/ci/idf_ci/uploader.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +import abc import glob import os import typing as t @@ -17,7 +18,28 @@ from idf_ci_utils import IDF_PATH from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME -class AppUploader: +class AppDownloader: + ALL_ARTIFACT_TYPES = [ArtifactType.MAP_AND_ELF_FILES, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES] + + @abc.abstractmethod + def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: + pass + + def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: + """ + Download the app + :param app_build_path: the path to the build directory + :param artifact_type: if not specify, download all types of artifacts + :return: None + """ + if not artifact_type: + for _artifact_type in self.ALL_ARTIFACT_TYPES: + self._download_app(app_build_path, _artifact_type) + else: + self._download_app(app_build_path, artifact_type) + + +class AppUploader(AppDownloader): TYPE_PATTERNS_DICT = { ArtifactType.MAP_AND_ELF_FILES: [ 'bootloader/*.map', @@ -123,13 +145,6 @@ class AppUploader: finally: os.chdir(current_dir) - def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: - if not artifact_type: - for _artifact_type in [ArtifactType.MAP_AND_ELF_FILES, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES]: - self._download_app(app_build_path, _artifact_type) - else: - self._download_app(app_build_path, artifact_type) - def get_app_presigned_url(self, app: App, artifact_type: ArtifactType) -> str: obj_name = self.get_app_object_name(app.app_dir, f'{app.build_dir}.zip', artifact_type) try: diff --git a/tools/ci/idf_pytest/constants.py b/tools/ci/idf_pytest/constants.py index 2eec0efda1..f2fce34573 100644 --- a/tools/ci/idf_pytest/constants.py +++ b/tools/ci/idf_pytest/constants.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - """ Pytest Related Constants. Don't import third-party packages here. """ @@ -246,7 +245,7 @@ class PytestCase: if 'jtag' in self.env_markers or 'usb_serial_jtag' in self.env_markers: return True - if any('panic' in Path(app.path).parts for app in self.apps): + if any('panic' in Path(app.path).resolve().parts for app in self.apps): return True return False