Merge branch 'bugfix/retry_download_submodule_v4.3' into 'release/v4.3'

ci: retry download from Gitlab on error 500 (v4.3)

See merge request espressif/esp-idf!15532

tools/ci/python_packages/gitlab_api.py

@@ -3,27 +3,66 @@ import os
 import re
 import tarfile
 import tempfile
+import time
 import zipfile
 from functools import wraps
 
 import gitlab
 
+try:
+    from typing import Any, Callable, Dict, List, Optional
+    TR = Callable[..., Any]
+except ImportError:
+    pass
+
+
+def retry(func):  # type: (TR) -> TR
+    """
+    This wrapper will only catch several exception types associated with
+    "network issues" and retry the whole function.
+    """
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):  # type: (Gitlab, Any, Any) -> Any
+        retried = 0
+        while True:
+            try:
+                res = func(self, *args, **kwargs)
+            except (IOError, EOFError, gitlab.exceptions.GitlabError) as e:
+                if isinstance(e, gitlab.exceptions.GitlabError) and e.response_code != 500:
+                    # Only retry on error 500
+                    raise e
+                retried += 1
+                if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
+                    raise e  # get out of the loop
+                else:
+                    print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried))
+                    time.sleep(2 ** retried)  # wait a bit more after each retry
+                    continue
+            else:
+                break
+        return res
+    return wrapper
+
 
 class Gitlab(object):
     JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
 
     DOWNLOAD_ERROR_MAX_RETRIES = 3
 
-    def __init__(self, project_id=None):
+    def __init__(self, project_id=None):  # type: (Optional[int]) -> None
         config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
         if config_data_from_env:
             # prefer to load config from env variable
             with tempfile.NamedTemporaryFile('w', delete=False) as temp_file:
                 temp_file.write(config_data_from_env)
-            config_files = [temp_file.name]
+            config_files = [temp_file.name]  # type: Optional[List[str]]
         else:
             # otherwise try to use config file at local filesystem
             config_files = None
+        self._init_gitlab_inst(project_id, config_files)
+
+    @retry
+    def _init_gitlab_inst(self, project_id, config_files):  # type: (Optional[int], Optional[List[str]]) -> None
         gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST')  # if None, will use the default gitlab server
         self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
         self.gitlab_inst.auth()
@@ -32,7 +71,8 @@ class Gitlab(object):
         else:
             self.project = None
 
-    def get_project_id(self, name, namespace=None):
+    @retry
+    def get_project_id(self, name, namespace=None):  # type: (str, Optional[str]) -> int
         """
         search project ID by name
 
@@ -56,9 +96,10 @@ class Gitlab(object):
 
         if not res:
             raise ValueError("Can't find project")
-        return res[0]
+        return int(res[0])
 
-    def download_artifacts(self, job_id, destination):
+    @retry
+    def download_artifacts(self, job_id, destination):  # type (int, str) -> None
         """
         download full job artifacts and extract to destination.
 
@@ -73,32 +114,8 @@ class Gitlab(object):
         with zipfile.ZipFile(temp_file.name, 'r') as archive_file:
             archive_file.extractall(destination)
 
-    def retry_download(func):
+    @retry
-        """
+    def download_artifact(self, job_id, artifact_path, destination=None):  # type: (int, str, Optional[str]) -> List[bytes]
-        This wrapper will only catch IOError and retry the whole function.
-
-        So only use it with download functions, read() inside and atomic
-        functions
-        """
-        @wraps(func)
-        def wrapper(self, *args, **kwargs):
-            retried = 0
-            while True:
-                try:
-                    res = func(self, *args, **kwargs)
-                except (IOError, EOFError) as e:
-                    retried += 1
-                    if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
-                        raise e  # get out of the loop
-                    else:
-                        print('Retried for the {} time'.format(retried))
-                        continue
-                else:
-                    break
-            return res
-        return wrapper
-
-    def download_artifact(self, job_id, artifact_path, destination=None):
         """
         download specific path of job artifacts and extract to destination.
 
@@ -113,7 +130,7 @@ class Gitlab(object):
 
         for a_path in artifact_path:
             try:
-                data = job.artifact(a_path)
+                data = job.artifact(a_path)  # type: bytes
             except gitlab.GitlabGetError as e:
                 print("Failed to download '{}' from job {}".format(a_path, job_id))
                 raise e
@@ -130,7 +147,8 @@ class Gitlab(object):
 
         return raw_data_list
 
-    def find_job_id(self, job_name, pipeline_id=None, job_status='success'):
+    @retry
+    def find_job_id(self, job_name, pipeline_id=None, job_status='success'):  # type: (str, Optional[str], str) -> List[Dict]
         """
         Get Job ID from job name of specific pipeline
 
@@ -152,8 +170,8 @@ class Gitlab(object):
                     job_id_list.append({'id': job.id, 'parallel_num': match.group(3)})
         return job_id_list
 
-    @retry_download
+    @retry
-    def download_archive(self, ref, destination, project_id=None):
+    def download_archive(self, ref, destination, project_id=None):  # type: (str, str, Optional[int]) -> str
         """
         Download archive of certain commit of a repository and extract to destination path
 
@@ -183,7 +201,7 @@ class Gitlab(object):
         return os.path.join(os.path.realpath(destination), root_name)
 
 
-if __name__ == '__main__':
+def main():  # type: () -> None
     parser = argparse.ArgumentParser()
     parser.add_argument('action')
     parser.add_argument('project_id', type=int)
@@ -209,3 +227,7 @@ if __name__ == '__main__':
     elif args.action == 'get_project_id':
         ret = gitlab_inst.get_project_id(args.project_name)
         print('project id: {}'.format(ret))
+
+
+if __name__ == '__main__':
+    main()