diff --git a/platformio/__init__.py b/platformio/__init__.py index 0d25faf8..b68f3371 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -40,7 +40,10 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" __accounts_api__ = "https://api.accounts.platformio.org" -__registry_api__ = "https://api.registry.platformio.org" +__registry_api__ = [ + "https://api.registry.platformio.org", + "https://api.registry.ns1.platformio.org", +] __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __core_packages__ = { diff --git a/platformio/clients/account.py b/platformio/clients/account.py index ba2c3451..e2abde17 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -40,7 +40,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__(self): - super(AccountClient, self).__init__(base_url=__accounts_api__) + super(AccountClient, self).__init__(__accounts_api__) @staticmethod def get_refresh_token(): diff --git a/platformio/clients/http.py b/platformio/clients/http.py index b1330f2e..8ee15b35 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import math import os import socket @@ -23,6 +24,12 @@ from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util from platformio.cache import ContentCache from platformio.exception import PlatformioException, UserSideException +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin + + PING_REMOTE_HOSTS = [ "140.82.118.3", # Github.com "35.231.145.151", # Gitlab.com @@ -51,23 +58,54 @@ class InternetIsOffline(UserSideException): ) -class HTTPClient(object): - def __init__( - self, base_url, - ): - if base_url.endswith("/"): - base_url = base_url[:-1] +class EndpointSession(requests.Session): + def __init__(self, base_url, *args, **kwargs): + super(EndpointSession, self).__init__(*args, **kwargs) self.base_url = base_url - self._session = requests.Session() - self._session.headers.update({"User-Agent": app.get_user_agent()}) - retry = Retry( - total=5, + + def request( # pylint: disable=signature-differs,arguments-differ + self, method, url, *args, **kwargs + ): + print(self.base_url, method, url, args, kwargs) + return super(EndpointSession, self).request( + method, urljoin(self.base_url, url), *args, **kwargs + ) + + +class EndpointSessionIterator(object): + def __init__(self, endpoints): + if not isinstance(endpoints, list): + endpoints = [endpoints] + self.endpoints = endpoints + self.endpoints_iter = iter(endpoints) + self.retry = Retry( + total=math.ceil(6 / len(self.endpoints)), backoff_factor=1, # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], status_forcelist=[413, 429, 500, 502, 503, 504], ) - adapter = requests.adapters.HTTPAdapter(max_retries=retry) - self._session.mount(base_url, adapter) + + def __iter__(self): # pylint: disable=non-iterator-returned + return self + + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + + def __next__(self): + base_url = next(self.endpoints_iter) + session = EndpointSession(base_url) + session.headers.update({"User-Agent": app.get_user_agent()}) + adapter = requests.adapters.HTTPAdapter(max_retries=self.retry) + session.mount(base_url, adapter) + return session + + +class HTTPClient(object): + def __init__(self, endpoints): + self._session_iter = EndpointSessionIterator(endpoints) + self._session = None + self._next_session() def __del__(self): if not self._session: @@ -75,20 +113,31 @@ class HTTPClient(object): self._session.close() self._session = None + def _next_session(self): + if self._session: + self._session.close() + self._session = next(self._session_iter) + @util.throttle(500) def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout - # print(self, method, path, kwargs) ensure_internet_on(raise_exception=True) # set default timeout if "timeout" not in kwargs: kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT - try: - return getattr(self._session, method)(self.base_url + path, **kwargs) - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - raise HTTPClientError(str(e)) + while True: + try: + return getattr(self._session, method)(path, **kwargs) + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + ) as e: + try: + self._next_session() + except: # pylint: disable=bare-except + raise HTTPClientError(str(e)) def fetch_json_data(self, method, path, **kwargs): cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index e990a65f..c8fbeeea 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -22,7 +22,7 @@ from platformio.package.meta import PackageType class RegistryClient(HTTPClient): def __init__(self): - super(RegistryClient, self).__init__(base_url=__registry_api__) + super(RegistryClient, self).__init__(__registry_api__) def send_auth_request(self, *args, **kwargs): headers = kwargs.get("headers", {}) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 72f189fb..415a9977 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -27,19 +27,23 @@ except ImportError: from urlparse import urlparse -class RegistryFileMirrorsIterator(object): +class RegistryFileMirrorIterator(object): HTTP_CLIENT_INSTANCES = {} def __init__(self, download_url): self.download_url = download_url self._url_parts = urlparse(download_url) - self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) + self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) self._visited_mirrors = [] def __iter__(self): # pylint: disable=non-iterator-returned return self + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + def __next__(self): http = self.get_http_client() response = http.send_request( @@ -64,16 +68,12 @@ class RegistryFileMirrorsIterator(object): response.headers.get("X-PIO-Content-SHA256"), ) - def next(self): - """ For Python 2 compatibility """ - return self.__next__() - def get_http_client(self): - if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES: - RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[ - self._base_url - ] = HTTPClient(self._base_url) - return RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[self._base_url] + if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: + RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( + self._mirror + ) + return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] class PackageManageRegistryMixin(object): @@ -98,7 +98,7 @@ class PackageManageRegistryMixin(object): if not pkgfile: raise UnknownPackageError(spec.humanize()) - for url, checksum in RegistryFileMirrorsIterator(pkgfile["download_url"]): + for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]): try: return self.install_from_url( url,