Cache DL requests

platformio/package/manager/_registry.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import time
 from urllib.parse import urlparse
 
 import click
 
 from platformio import __registry_mirror_hosts__
+from platformio.cache import ContentCache
 from platformio.clients.http import HTTPClient
 from platformio.clients.registry import RegistryClient
 from platformio.package.exception import UnknownPackageError
@@ -39,29 +41,56 @@ class RegistryFileMirrorIterator(object):
         return self
 
     def __next__(self):
-        http = self.get_http_client()
-        response = http.send_request(
-            "head",
-            self._url_parts.path,
-            allow_redirects=False,
-            params=dict(bypass=",".join(self._visited_mirrors))
-            if self._visited_mirrors
-            else None,
-            x_with_authorization=RegistryClient.allowed_private_packages(),
-        )
-        stop_conditions = [
-            response.status_code not in (302, 307),
-            not response.headers.get("Location"),
-            not response.headers.get("X-PIO-Mirror"),
-            response.headers.get("X-PIO-Mirror") in self._visited_mirrors,
-        ]
-        if any(stop_conditions):
-            raise StopIteration
-        self._visited_mirrors.append(response.headers.get("X-PIO-Mirror"))
-        return (
-            response.headers.get("Location"),
-            response.headers.get("X-PIO-Content-SHA256"),
+        cache_key = ContentCache.key_from_args(
+            "head", self.download_url, self._visited_mirrors
         )
+        with ContentCache("http") as cc:
+            result = cc.get(cache_key)
+            if result is not None:
+                try:
+                    headers = json.loads(result)
+                    return (
+                        headers["Location"],
+                        headers["X-PIO-Content-SHA256"],
+                    )
+                except (ValueError, KeyError):
+                    pass
+
+            http = self.get_http_client()
+            response = http.send_request(
+                "head",
+                self._url_parts.path,
+                allow_redirects=False,
+                params=dict(bypass=",".join(self._visited_mirrors))
+                if self._visited_mirrors
+                else None,
+                x_with_authorization=RegistryClient.allowed_private_packages(),
+            )
+            stop_conditions = [
+                response.status_code not in (302, 307),
+                not response.headers.get("Location"),
+                not response.headers.get("X-PIO-Mirror"),
+                response.headers.get("X-PIO-Mirror") in self._visited_mirrors,
+            ]
+            if any(stop_conditions):
+                raise StopIteration
+            self._visited_mirrors.append(response.headers.get("X-PIO-Mirror"))
+            cc.set(
+                cache_key,
+                json.dumps(
+                    {
+                        "Location": response.headers.get("Location"),
+                        "X-PIO-Content-SHA256": response.headers.get(
+                            "X-PIO-Content-SHA256"
+                        ),
+                    }
+                ),
+                "1h",
+            )
+            return (
+                response.headers.get("Location"),
+                response.headers.get("X-PIO-Content-SHA256"),
+            )
 
     def get_http_client(self):
         if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: