espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-05 21:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ci/move_stdout_print_to_stderr_logging' into 'master'

Browse Source 
ci: move stdout print to stderr logging in gitlab api

See merge request espressif/esp-idf!22387
This commit is contained in:
Fu Hanxi
2023-02-16 13:45:55 +08:00
parent 76989b504a 8d0dda23b0
commit 610102ace1
1 changed files with 18 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tools/ci/python_packages/gitlab_api.py
View File

@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
import argparse import argparse
import logging
import os import os
import re import re
import tarfile import tarfile
@@ -14,12 +15,15 @@ import gitlab
TR = Callable[..., Any] TR = Callable[..., Any]
logging.basicConfig(level=logging.INFO)
def retry(func: TR) -> TR: def retry(func: TR) -> TR:
""" """
This wrapper will only catch several exception types associated with This wrapper will only catch several exception types associated with
"network issues" and retry the whole function. "network issues" and retry the whole function.
""" """
@wraps(func) @wraps(func)
def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any: def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any:
retried = 0 retried = 0
@@ -41,12 +45,15 @@ def retry(func: TR) -> TR:
if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
raise e # get out of the loop raise e # get out of the loop
else: else:
print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried)) logging.warning(
'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
retried))
time.sleep(2 ** retried) # wait a bit more after each retry time.sleep(2 ** retried) # wait a bit more after each retry
continue continue
else: else:
break break
return res return res
return wrapper return wrapper
@@ -138,7 +145,7 @@ class Gitlab(object):
try: try:
data = job.artifact(a_path) # type: bytes data = job.artifact(a_path) # type: bytes
except gitlab.GitlabGetError as e: except gitlab.GitlabGetError as e:
print("Failed to download '{}' from job {}".format(a_path, job_id)) logging.error("Failed to download '{}' from job {}".format(a_path, job_id))
raise e raise e
raw_data_list.append(data) raw_data_list.append(data)
if destination: if destination:
@@ -177,7 +184,8 @@ class Gitlab(object):
return job_id_list return job_id_list
@retry @retry
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None) -> str: def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
cache_dir: Optional[str] = None) -> str:
""" """
Download archive of certain commit of a repository and extract to destination path Download archive of certain commit of a repository and extract to destination path
@@ -195,15 +203,16 @@ class Gitlab(object):
local_archive_file = os.path.join(cache_dir, f'{ref}.tar.gz') local_archive_file = os.path.join(cache_dir, f'{ref}.tar.gz')
os.makedirs(os.path.dirname(local_archive_file), exist_ok=True) os.makedirs(os.path.dirname(local_archive_file), exist_ok=True)
if os.path.isfile(local_archive_file): if os.path.isfile(local_archive_file):
print('Use cached archive file. Skipping download...') logging.info('Use cached archive file. Skipping download...')
else: else:
with open(local_archive_file, 'wb') as fw: with open(local_archive_file, 'wb') as fw:
try: try:
project.repository_archive(sha=ref, streamed=True, action=fw.write) project.repository_archive(sha=ref, streamed=True, action=fw.write)
except gitlab.GitlabGetError as e: except gitlab.GitlabGetError as e:
print('Failed to archive from project {}'.format(project_id)) logging.error('Failed to archive from project {}'.format(project_id))
raise e raise e
print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(local_archive_file)) / (1024 * 1024))) logging.info('Downloaded archive size: {:.03f}MB'.format(
float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
return self.decompress_archive(local_archive_file, destination) return self.decompress_archive(local_archive_file, destination)
@@ -212,10 +221,10 @@ class Gitlab(object):
try: try:
project.repository_archive(sha=ref, streamed=True, action=temp_file.write) project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
except gitlab.GitlabGetError as e: except gitlab.GitlabGetError as e:
print('Failed to archive from project {}'.format(project_id)) logging.error('Failed to archive from project {}'.format(project_id))
raise e raise e
print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))) logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
return self.decompress_archive(temp_file.name, destination) return self.decompress_archive(temp_file.name, destination)