espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-03 20:54:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ci/move_stdout_print_to_stderr_logging' into 'master'

Browse Source 
ci: move stdout print to stderr logging in gitlab api

See merge request espressif/esp-idf!22387
This commit is contained in:
Fu Hanxi
2023-02-16 13:45:55 +08:00
parent 76989b504a 8d0dda23b0
commit 610102ace1
1 changed files with 18 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tools/ci/python_packages/gitlab_api.py
View File

@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import logging
import os
import re
import tarfile
@@ -14,12 +15,15 @@ import gitlab
TR = Callable[..., Any]
logging.basicConfig(level=logging.INFO)
def retry(func: TR) -> TR:
"""
This wrapper will only catch several exception types associated with
"network issues" and retry the whole function.
"""
@wraps(func)
def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any:
retried = 0
@@ -41,12 +45,15 @@ def retry(func: TR) -> TR:
if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
raise e # get out of the loop
else:
print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried))
logging.warning(
'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
retried))
time.sleep(2 ** retried) # wait a bit more after each retry
continue
else:
break
return res
return wrapper
@@ -138,7 +145,7 @@ class Gitlab(object):
try:
data = job.artifact(a_path) # type: bytes
except gitlab.GitlabGetError as e:
print("Failed to download '{}' from job {}".format(a_path, job_id))
logging.error("Failed to download '{}' from job {}".format(a_path, job_id))
raise e
raw_data_list.append(data)
if destination:
@@ -177,7 +184,8 @@ class Gitlab(object):
return job_id_list
@retry
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None) -> str:
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
cache_dir: Optional[str] = None) -> str:
"""
Download archive of certain commit of a repository and extract to destination path
@@ -195,15 +203,16 @@ class Gitlab(object):
local_archive_file = os.path.join(cache_dir, f'{ref}.tar.gz')
os.makedirs(os.path.dirname(local_archive_file), exist_ok=True)
if os.path.isfile(local_archive_file):
print('Use cached archive file. Skipping download...')
logging.info('Use cached archive file. Skipping download...')
else:
with open(local_archive_file, 'wb') as fw:
try:
project.repository_archive(sha=ref, streamed=True, action=fw.write)
except gitlab.GitlabGetError as e:
print('Failed to archive from project {}'.format(project_id))
logging.error('Failed to archive from project {}'.format(project_id))
raise e
print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
logging.info('Downloaded archive size: {:.03f}MB'.format(
float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
return self.decompress_archive(local_archive_file, destination)
@@ -212,10 +221,10 @@ class Gitlab(object):
try:
project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
except gitlab.GitlabGetError as e:
print('Failed to archive from project {}'.format(project_id))
logging.error('Failed to archive from project {}'.format(project_id))
raise e
print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
return self.decompress_archive(temp_file.name, destination)