Implement mirroring for HTTP client

platformio/__init__.py

@@ -40,7 +40,10 @@ __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO"
 
 __accounts_api__ = "https://api.accounts.platformio.org"
-__registry_api__ = "https://api.registry.platformio.org"
+__registry_api__ = [
+    "https://api.registry.platformio.org",
+    "https://api.registry.ns1.platformio.org",
+]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
 __core_packages__ = {

platformio/clients/account.py

@@ -40,7 +40,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
     SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
 
     def __init__(self):
-        super(AccountClient, self).__init__(base_url=__accounts_api__)
+        super(AccountClient, self).__init__(__accounts_api__)
 
     @staticmethod
     def get_refresh_token():

platformio/clients/http.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import math
 import os
 import socket
 
@@ -23,6 +24,12 @@ from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util
 from platformio.cache import ContentCache
 from platformio.exception import PlatformioException, UserSideException
 
+try:
+    from urllib.parse import urljoin
+except ImportError:
+    from urlparse import urljoin
+
+
 PING_REMOTE_HOSTS = [
     "140.82.118.3",  # Github.com
     "35.231.145.151",  # Gitlab.com
@@ -51,23 +58,54 @@ class InternetIsOffline(UserSideException):
     )
 
 
-class HTTPClient(object):
+class EndpointSession(requests.Session):
-    def __init__(
+    def __init__(self, base_url, *args, **kwargs):
-        self, base_url,
+        super(EndpointSession, self).__init__(*args, **kwargs)
-    ):
-        if base_url.endswith("/"):
-            base_url = base_url[:-1]
         self.base_url = base_url
-        self._session = requests.Session()
+
-        self._session.headers.update({"User-Agent": app.get_user_agent()})
+    def request(  # pylint: disable=signature-differs,arguments-differ
-        retry = Retry(
+        self, method, url, *args, **kwargs
-            total=5,
+    ):
+        print(self.base_url, method, url, args, kwargs)
+        return super(EndpointSession, self).request(
+            method, urljoin(self.base_url, url), *args, **kwargs
+        )
+
+
+class EndpointSessionIterator(object):
+    def __init__(self, endpoints):
+        if not isinstance(endpoints, list):
+            endpoints = [endpoints]
+        self.endpoints = endpoints
+        self.endpoints_iter = iter(endpoints)
+        self.retry = Retry(
+            total=math.ceil(6 / len(self.endpoints)),
             backoff_factor=1,
             # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"],
             status_forcelist=[413, 429, 500, 502, 503, 504],
         )
-        adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+
-        self._session.mount(base_url, adapter)
+    def __iter__(self):  # pylint: disable=non-iterator-returned
+        return self
+
+    def next(self):
+        """ For Python 2 compatibility """
+        return self.__next__()
+
+    def __next__(self):
+        base_url = next(self.endpoints_iter)
+        session = EndpointSession(base_url)
+        session.headers.update({"User-Agent": app.get_user_agent()})
+        adapter = requests.adapters.HTTPAdapter(max_retries=self.retry)
+        session.mount(base_url, adapter)
+        return session
+
+
+class HTTPClient(object):
+    def __init__(self, endpoints):
+        self._session_iter = EndpointSessionIterator(endpoints)
+        self._session = None
+        self._next_session()
 
     def __del__(self):
         if not self._session:
@@ -75,20 +113,31 @@ class HTTPClient(object):
         self._session.close()
         self._session = None
 
+    def _next_session(self):
+        if self._session:
+            self._session.close()
+        self._session = next(self._session_iter)
+
     @util.throttle(500)
     def send_request(self, method, path, **kwargs):
         # check Internet before and resolve issue with 60 seconds timeout
-        # print(self, method, path, kwargs)
         ensure_internet_on(raise_exception=True)
 
         # set default timeout
         if "timeout" not in kwargs:
             kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT
 
-        try:
+        while True:
-            return getattr(self._session, method)(self.base_url + path, **kwargs)
+            try:
-        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
+                return getattr(self._session, method)(path, **kwargs)
-            raise HTTPClientError(str(e))
+            except (
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout,
+            ) as e:
+                try:
+                    self._next_session()
+                except:  # pylint: disable=bare-except
+                    raise HTTPClientError(str(e))
 
     def fetch_json_data(self, method, path, **kwargs):
         cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None

platformio/clients/registry.py

@@ -22,7 +22,7 @@ from platformio.package.meta import PackageType
 
 class RegistryClient(HTTPClient):
     def __init__(self):
-        super(RegistryClient, self).__init__(base_url=__registry_api__)
+        super(RegistryClient, self).__init__(__registry_api__)
 
     def send_auth_request(self, *args, **kwargs):
         headers = kwargs.get("headers", {})

platformio/package/manager/_registry.py

@@ -27,19 +27,23 @@ except ImportError:
     from urlparse import urlparse
 
 
-class RegistryFileMirrorsIterator(object):
+class RegistryFileMirrorIterator(object):
 
     HTTP_CLIENT_INSTANCES = {}
 
     def __init__(self, download_url):
         self.download_url = download_url
         self._url_parts = urlparse(download_url)
-        self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc)
+        self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc)
         self._visited_mirrors = []
 
     def __iter__(self):  # pylint: disable=non-iterator-returned
         return self
 
+    def next(self):
+        """ For Python 2 compatibility """
+        return self.__next__()
+
     def __next__(self):
         http = self.get_http_client()
         response = http.send_request(
@@ -64,16 +68,12 @@ class RegistryFileMirrorsIterator(object):
             response.headers.get("X-PIO-Content-SHA256"),
         )
 
-    def next(self):
-        """ For Python 2 compatibility """
-        return self.__next__()
-
     def get_http_client(self):
-        if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES:
+        if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES:
-            RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[
+            RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient(
-                self._base_url
+                self._mirror
-            ] = HTTPClient(self._base_url)
+            )
-        return RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[self._base_url]
+        return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror]
 
 
 class PackageManageRegistryMixin(object):
@@ -98,7 +98,7 @@ class PackageManageRegistryMixin(object):
         if not pkgfile:
             raise UnknownPackageError(spec.humanize())
 
-        for url, checksum in RegistryFileMirrorsIterator(pkgfile["download_url"]):
+        for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]):
             try:
                 return self.install_from_url(
                     url,