From 39528b17836ca5dc9165204eb74bed16add7e4f7 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 8 Jun 2021 15:04:06 +0800 Subject: [PATCH 1/3] ci: retry download artifacts --- tools/ci/python_packages/gitlab_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 6df5a83500..c6d6fd7a76 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -98,6 +98,7 @@ class Gitlab(object): return res return wrapper + @retry_download # type: ignore def download_artifact(self, job_id, artifact_path, destination=None): """ download specific path of job artifacts and extract to destination. @@ -152,7 +153,7 @@ class Gitlab(object): job_id_list.append({'id': job.id, 'parallel_num': match.group(3)}) return job_id_list - @retry_download + @retry_download # type: ignore def download_archive(self, ref, destination, project_id=None): """ Download archive of certain commit of a repository and extract to destination path From 81e12775594493fb54fd3d23e8482d8a90a2dc5d Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Thu, 14 Oct 2021 14:19:50 +0200 Subject: [PATCH 2/3] ci: retry Gitlab operations on error 500 --- tools/ci/python_packages/gitlab_api.py | 92 +++++++++++++++----------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index c6d6fd7a76..f380bc1c86 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -3,27 +3,63 @@ import os import re import tarfile import tempfile +import time import zipfile from functools import wraps +from typing import Any, Callable, Dict, List, Optional import gitlab +TR = Callable[..., Any] + + +def retry(func: TR) -> TR: + """ + This wrapper will only catch several exception types associated with + "network issues" and retry the whole function. + """ + @wraps(func) + def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any: + retried = 0 + while True: + try: + res = func(self, *args, **kwargs) + except (IOError, EOFError, gitlab.exceptions.GitlabError) as e: + if isinstance(e, gitlab.exceptions.GitlabError) and e.response_code != 500: + # Only retry on error 500 + raise e + retried += 1 + if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: + raise e # get out of the loop + else: + print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried)) + time.sleep(2 ** retried) # wait a bit more after each retry + continue + else: + break + return res + return wrapper + class Gitlab(object): JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?') DOWNLOAD_ERROR_MAX_RETRIES = 3 - def __init__(self, project_id=None): + def __init__(self, project_id: Optional[int] = None): config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') if config_data_from_env: # prefer to load config from env variable with tempfile.NamedTemporaryFile('w', delete=False) as temp_file: temp_file.write(config_data_from_env) - config_files = [temp_file.name] + config_files = [temp_file.name] # type: Optional[List[str]] else: # otherwise try to use config file at local filesystem config_files = None + self._init_gitlab_inst(project_id, config_files) + + @retry + def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None: gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) self.gitlab_inst.auth() @@ -32,7 +68,8 @@ class Gitlab(object): else: self.project = None - def get_project_id(self, name, namespace=None): + @retry + def get_project_id(self, name: str, namespace: Optional[str] = None) -> int: """ search project ID by name @@ -56,9 +93,10 @@ class Gitlab(object): if not res: raise ValueError("Can't find project") - return res[0] + return int(res[0]) - def download_artifacts(self, job_id, destination): + @retry + def download_artifacts(self, job_id: int, destination: str) -> None: """ download full job artifacts and extract to destination. @@ -73,33 +111,8 @@ class Gitlab(object): with zipfile.ZipFile(temp_file.name, 'r') as archive_file: archive_file.extractall(destination) - def retry_download(func): - """ - This wrapper will only catch IOError and retry the whole function. - - So only use it with download functions, read() inside and atomic - functions - """ - @wraps(func) - def wrapper(self, *args, **kwargs): - retried = 0 - while True: - try: - res = func(self, *args, **kwargs) - except (IOError, EOFError) as e: - retried += 1 - if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: - raise e # get out of the loop - else: - print('Retried for the {} time'.format(retried)) - continue - else: - break - return res - return wrapper - - @retry_download # type: ignore - def download_artifact(self, job_id, artifact_path, destination=None): + @retry + def download_artifact(self, job_id: int, artifact_path: str, destination: Optional[str] = None) -> List[bytes]: """ download specific path of job artifacts and extract to destination. @@ -114,7 +127,7 @@ class Gitlab(object): for a_path in artifact_path: try: - data = job.artifact(a_path) + data = job.artifact(a_path) # type: bytes except gitlab.GitlabGetError as e: print("Failed to download '{}' from job {}".format(a_path, job_id)) raise e @@ -131,7 +144,8 @@ class Gitlab(object): return raw_data_list - def find_job_id(self, job_name, pipeline_id=None, job_status='success'): + @retry + def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]: """ Get Job ID from job name of specific pipeline @@ -153,8 +167,8 @@ class Gitlab(object): job_id_list.append({'id': job.id, 'parallel_num': match.group(3)}) return job_id_list - @retry_download # type: ignore - def download_archive(self, ref, destination, project_id=None): + @retry + def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None) -> str: """ Download archive of certain commit of a repository and extract to destination path @@ -184,7 +198,7 @@ class Gitlab(object): return os.path.join(os.path.realpath(destination), root_name) -if __name__ == '__main__': +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('action') parser.add_argument('project_id', type=int) @@ -210,3 +224,7 @@ if __name__ == '__main__': elif args.action == 'get_project_id': ret = gitlab_inst.get_project_id(args.project_name) print('project id: {}'.format(ret)) + + +if __name__ == '__main__': + main() From f7d023383bf0cdd8e572006759408b2850d5549c Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 15 Oct 2021 10:57:08 +0200 Subject: [PATCH 3/3] ci: revert type annotations in gitlab_api.py to python2 compatible --- tools/ci/python_packages/gitlab_api.py | 27 ++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index f380bc1c86..598e76375e 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -6,20 +6,23 @@ import tempfile import time import zipfile from functools import wraps -from typing import Any, Callable, Dict, List, Optional import gitlab -TR = Callable[..., Any] +try: + from typing import Any, Callable, Dict, List, Optional + TR = Callable[..., Any] +except ImportError: + pass -def retry(func: TR) -> TR: +def retry(func): # type: (TR) -> TR """ This wrapper will only catch several exception types associated with "network issues" and retry the whole function. """ @wraps(func) - def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any: + def wrapper(self, *args, **kwargs): # type: (Gitlab, Any, Any) -> Any retried = 0 while True: try: @@ -46,7 +49,7 @@ class Gitlab(object): DOWNLOAD_ERROR_MAX_RETRIES = 3 - def __init__(self, project_id: Optional[int] = None): + def __init__(self, project_id=None): # type: (Optional[int]) -> None config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') if config_data_from_env: # prefer to load config from env variable @@ -59,7 +62,7 @@ class Gitlab(object): self._init_gitlab_inst(project_id, config_files) @retry - def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None: + def _init_gitlab_inst(self, project_id, config_files): # type: (Optional[int], Optional[List[str]]) -> None gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) self.gitlab_inst.auth() @@ -69,7 +72,7 @@ class Gitlab(object): self.project = None @retry - def get_project_id(self, name: str, namespace: Optional[str] = None) -> int: + def get_project_id(self, name, namespace=None): # type: (str, Optional[str]) -> int """ search project ID by name @@ -96,7 +99,7 @@ class Gitlab(object): return int(res[0]) @retry - def download_artifacts(self, job_id: int, destination: str) -> None: + def download_artifacts(self, job_id, destination): # type (int, str) -> None """ download full job artifacts and extract to destination. @@ -112,7 +115,7 @@ class Gitlab(object): archive_file.extractall(destination) @retry - def download_artifact(self, job_id: int, artifact_path: str, destination: Optional[str] = None) -> List[bytes]: + def download_artifact(self, job_id, artifact_path, destination=None): # type: (int, str, Optional[str]) -> List[bytes] """ download specific path of job artifacts and extract to destination. @@ -145,7 +148,7 @@ class Gitlab(object): return raw_data_list @retry - def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]: + def find_job_id(self, job_name, pipeline_id=None, job_status='success'): # type: (str, Optional[str], str) -> List[Dict] """ Get Job ID from job name of specific pipeline @@ -168,7 +171,7 @@ class Gitlab(object): return job_id_list @retry - def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None) -> str: + def download_archive(self, ref, destination, project_id=None): # type: (str, str, Optional[int]) -> str """ Download archive of certain commit of a repository and extract to destination path @@ -198,7 +201,7 @@ class Gitlab(object): return os.path.join(os.path.realpath(destination), root_name) -def main() -> None: +def main(): # type: () -> None parser = argparse.ArgumentParser() parser.add_argument('action') parser.add_argument('project_id', type=int)