Add support for DL mirrors

platformio/__init__.py

@@ -38,9 +38,9 @@ __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO Labs"
 
 __accounts_api__ = "https://api.accounts.platformio.org"
-__registry_api__ = [
+__registry_mirror_hosts__ = [
-    "https://api.registry.platformio.org",
+    "registry.platformio.org",
-    "https://api.registry.ns1.platformio.org",
+    "registry.nm1.platformio.org",
 ]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
@@ -60,5 +60,4 @@ __check_internet_hosts__ = [
     "185.199.110.153",  # Github.com
     "88.198.170.159",  # platformio.org
     "github.com",
-    "platformio.org",
+] + __registry_mirror_hosts__
-]

platformio/clients/http.py

@@ -73,10 +73,6 @@ class EndpointSessionIterator(object):
     def __iter__(self):  # pylint: disable=non-iterator-returned
         return self
 
-    def next(self):
-        """For Python 2 compatibility"""
-        return self.__next__()
-
     def __next__(self):
         base_url = next(self.endpoints_iter)
         session = EndpointSession(base_url)
@@ -143,7 +139,7 @@ class HTTPClient(object):
                     raise HTTPClientError(str(e))
 
     def fetch_json_data(self, method, path, **kwargs):
-        if method != "get":
+        if method not in ("get", "head", "options"):
             cleanup_content_cache("http")
         cache_valid = kwargs.pop("x_cache_valid") if "x_cache_valid" in kwargs else None
         if not cache_valid:

platformio/clients/registry.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio import __registry_api__, fs
+from platformio import __registry_mirror_hosts__, fs
 from platformio.clients.account import AccountClient, AccountError
 from platformio.clients.http import HTTPClient, HTTPClientError
 
@@ -21,7 +21,8 @@ from platformio.clients.http import HTTPClient, HTTPClientError
 
 class RegistryClient(HTTPClient):
     def __init__(self):
-        super().__init__(__registry_api__)
+        endpoints = [f"https://api.{host}" for host in __registry_mirror_hosts__]
+        super().__init__(endpoints)
 
     @staticmethod
     def allowed_private_packages():
@@ -102,6 +103,7 @@ class RegistryClient(HTTPClient):
             "get",
             "/v3/resources",
             params={"owner": owner} if owner else None,
+            x_cache_valid="1h",
             x_with_authorization=True,
         )
 

platformio/commands/platform.py

@@ -18,7 +18,6 @@ import os
 
 import click
 
-from platformio.cache import cleanup_content_cache
 from platformio.commands.boards import print_boards
 from platformio.exception import UserSideException
 from platformio.package.exception import UnknownPackageError
@@ -313,9 +312,6 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
             result.append(data)
         return click.echo(json.dumps(result))
 
-    # cleanup cached board and platform lists
-    cleanup_content_cache("http")
-
     for platform in platforms:
         click.echo(
             "Platform %s"

platformio/commands/update.py

@@ -14,7 +14,6 @@
 
 import click
 
-from platformio.cache import cleanup_content_cache
 from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
@@ -39,9 +38,6 @@ from platformio.package.manager.library import LibraryPackageManager
 )
 @click.pass_context
 def cli(ctx, core_packages, only_check, dry_run):
-    # cleanup lib search results, cached board and platform lists
-    cleanup_content_cache("http")
-
     only_check = dry_run or only_check
 
     if not only_check:

platformio/package/manager/_registry.py

@@ -17,6 +17,7 @@ from urllib.parse import urlparse
 
 import click
 
+from platformio import __registry_mirror_hosts__
 from platformio.clients.http import HTTPClient
 from platformio.clients.registry import RegistryClient
 from platformio.package.exception import UnknownPackageError
@@ -37,10 +38,6 @@ class RegistryFileMirrorIterator(object):
     def __iter__(self):  # pylint: disable=non-iterator-returned
         return self
 
-    def next(self):
-        """For Python 2 compatibility"""
-        return self.__next__()
-
     def __next__(self):
         http = self.get_http_client()
         response = http.send_request(
@@ -68,8 +65,13 @@ class RegistryFileMirrorIterator(object):
 
     def get_http_client(self):
         if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES:
+            endpoints = [self._mirror]
+            for host in __registry_mirror_hosts__:
+                endpoint = f"https://dl.{host}"
+                if endpoint not in endpoints:
+                    endpoints.append(endpoint)
             RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient(
-                self._mirror
+                endpoints
             )
         return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror]