From 7e7856e44cf7febe66cb6263bc02c8d5fbe8030e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 1 Jun 2022 12:35:55 +0300 Subject: [PATCH] Restructure "registry" modules --- platformio/commands/access.py | 2 +- platformio/package/commands/publish.py | 2 +- platformio/package/commands/search.py | 2 +- platformio/package/commands/show.py | 2 +- platformio/package/commands/unpublish.py | 2 +- platformio/package/manager/_registry.py | 86 +------------------ platformio/registry/__init__.py | 13 +++ platformio/{package => registry}/client.py | 0 platformio/registry/mirror.py | 99 ++++++++++++++++++++++ tests/commands/test_lib.py | 2 +- tests/misc/test_misc.py | 2 +- 11 files changed, 121 insertions(+), 91 deletions(-) create mode 100644 platformio/registry/__init__.py rename platformio/{package => registry}/client.py (100%) create mode 100644 platformio/registry/mirror.py diff --git a/platformio/commands/access.py b/platformio/commands/access.py index d35d0830..fa726927 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -22,7 +22,7 @@ from tabulate import tabulate from platformio.account.helpers import validate_username from platformio.commands.team import validate_orgname_teamname -from platformio.package.client import RegistryClient +from platformio.registry.client import RegistryClient def validate_client(value): diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index ee7e4a25..c1c12a5a 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -23,12 +23,12 @@ from tabulate import tabulate from platformio import fs from platformio.account.client import AccountClient from platformio.exception import UserSideException -from platformio.package.client import RegistryClient from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.package.meta import PackageType from platformio.package.pack import PackagePacker from platformio.package.unpack import FileUnpacker, TARArchiver +from platformio.registry.client import RegistryClient def validate_datetime(ctx, param, value): # pylint: disable=unused-argument diff --git a/platformio/package/commands/search.py b/platformio/package/commands/search.py index d93b9f81..ac71ef4c 100644 --- a/platformio/package/commands/search.py +++ b/platformio/package/commands/search.py @@ -17,7 +17,7 @@ import math import click from platformio import util -from platformio.package.client import RegistryClient +from platformio.registry.client import RegistryClient @click.command("search", short_help="Search for packages") diff --git a/platformio/package/commands/show.py b/platformio/package/commands/show.py index c1ee28bb..82b6e47c 100644 --- a/platformio/package/commands/show.py +++ b/platformio/package/commands/show.py @@ -19,9 +19,9 @@ from tabulate import tabulate from platformio import fs, util from platformio.exception import UserSideException -from platformio.package.client import RegistryClient from platformio.package.manager._registry import PackageManagerRegistryMixin from platformio.package.meta import PackageSpec, PackageType +from platformio.registry.client import RegistryClient @click.command("show", short_help="Show package information") diff --git a/platformio/package/commands/unpublish.py b/platformio/package/commands/unpublish.py index 9c1d067e..7d0e633e 100644 --- a/platformio/package/commands/unpublish.py +++ b/platformio/package/commands/unpublish.py @@ -15,8 +15,8 @@ import click from platformio.account.client import AccountClient -from platformio.package.client import RegistryClient from platformio.package.meta import PackageSpec, PackageType +from platformio.registry.client import RegistryClient @click.command("unpublish", short_help="Remove a pushed package from the registry") diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 2fc11e1d..0a9f39c5 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -12,97 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import time -from urllib.parse import urlparse import click -from platformio import __registry_mirror_hosts__ -from platformio.cache import ContentCache -from platformio.http import HTTPClient -from platformio.package.client import RegistryClient from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageSpec from platformio.package.version import cast_version_to_semver - - -class RegistryFileMirrorIterator(object): - - HTTP_CLIENT_INSTANCES = {} - - def __init__(self, download_url): - self.download_url = download_url - self._url_parts = urlparse(download_url) - self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) - self._visited_mirrors = [] - - def __iter__(self): # pylint: disable=non-iterator-returned - return self - - def __next__(self): - cache_key = ContentCache.key_from_args( - "head", self.download_url, self._visited_mirrors - ) - with ContentCache("http") as cc: - result = cc.get(cache_key) - if result is not None: - try: - headers = json.loads(result) - return ( - headers["Location"], - headers["X-PIO-Content-SHA256"], - ) - except (ValueError, KeyError): - pass - - http = self.get_http_client() - response = http.send_request( - "head", - self._url_parts.path, - allow_redirects=False, - params=dict(bypass=",".join(self._visited_mirrors)) - if self._visited_mirrors - else None, - x_with_authorization=RegistryClient.allowed_private_packages(), - ) - stop_conditions = [ - response.status_code not in (302, 307), - not response.headers.get("Location"), - not response.headers.get("X-PIO-Mirror"), - response.headers.get("X-PIO-Mirror") in self._visited_mirrors, - ] - if any(stop_conditions): - raise StopIteration - self._visited_mirrors.append(response.headers.get("X-PIO-Mirror")) - cc.set( - cache_key, - json.dumps( - { - "Location": response.headers.get("Location"), - "X-PIO-Content-SHA256": response.headers.get( - "X-PIO-Content-SHA256" - ), - } - ), - "1h", - ) - return ( - response.headers.get("Location"), - response.headers.get("X-PIO-Content-SHA256"), - ) - - def get_http_client(self): - if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: - endpoints = [self._mirror] - for host in __registry_mirror_hosts__: - endpoint = f"https://dl.{host}" - if endpoint not in endpoints: - endpoints.append(endpoint) - RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( - endpoints - ) - return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] +from platformio.registry.client import RegistryClient +from platformio.registry.mirror import RegistryFileMirrorIterator class PackageManagerRegistryMixin(object): diff --git a/platformio/registry/__init__.py b/platformio/registry/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/registry/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/client.py b/platformio/registry/client.py similarity index 100% rename from platformio/package/client.py rename to platformio/registry/client.py diff --git a/platformio/registry/mirror.py b/platformio/registry/mirror.py new file mode 100644 index 00000000..d967838e --- /dev/null +++ b/platformio/registry/mirror.py @@ -0,0 +1,99 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from urllib.parse import urlparse + +from platformio import __registry_mirror_hosts__ +from platformio.cache import ContentCache +from platformio.http import HTTPClient +from platformio.registry.client import RegistryClient + + +class RegistryFileMirrorIterator(object): + + HTTP_CLIENT_INSTANCES = {} + + def __init__(self, download_url): + self.download_url = download_url + self._url_parts = urlparse(download_url) + self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) + self._visited_mirrors = [] + + def __iter__(self): # pylint: disable=non-iterator-returned + return self + + def __next__(self): + cache_key = ContentCache.key_from_args( + "head", self.download_url, self._visited_mirrors + ) + with ContentCache("http") as cc: + result = cc.get(cache_key) + if result is not None: + try: + headers = json.loads(result) + return ( + headers["Location"], + headers["X-PIO-Content-SHA256"], + ) + except (ValueError, KeyError): + pass + + http = self.get_http_client() + response = http.send_request( + "head", + self._url_parts.path, + allow_redirects=False, + params=dict(bypass=",".join(self._visited_mirrors)) + if self._visited_mirrors + else None, + x_with_authorization=RegistryClient.allowed_private_packages(), + ) + stop_conditions = [ + response.status_code not in (302, 307), + not response.headers.get("Location"), + not response.headers.get("X-PIO-Mirror"), + response.headers.get("X-PIO-Mirror") in self._visited_mirrors, + ] + if any(stop_conditions): + raise StopIteration + self._visited_mirrors.append(response.headers.get("X-PIO-Mirror")) + cc.set( + cache_key, + json.dumps( + { + "Location": response.headers.get("Location"), + "X-PIO-Content-SHA256": response.headers.get( + "X-PIO-Content-SHA256" + ), + } + ), + "1h", + ) + return ( + response.headers.get("Location"), + response.headers.get("X-PIO-Content-SHA256"), + ) + + def get_http_client(self): + if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: + endpoints = [self._mirror] + for host in __registry_mirror_hosts__: + endpoint = f"https://dl.{host}" + if endpoint not in endpoints: + endpoints.append(endpoint) + RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( + endpoints + ) + return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index dcfcab20..cf1e53b6 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -21,10 +21,10 @@ import pytest import semantic_version from platformio.commands.lib.command import cli as cmd_lib -from platformio.package.client import RegistryClient from platformio.package.meta import PackageType from platformio.package.vcsclient import VCSClientFactory from platformio.project.config import ProjectConfig +from platformio.registry.client import RegistryClient def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): diff --git a/tests/misc/test_misc.py b/tests/misc/test_misc.py index a7d776e1..52a6cac4 100644 --- a/tests/misc/test_misc.py +++ b/tests/misc/test_misc.py @@ -18,7 +18,7 @@ import pytest import requests from platformio import __check_internet_hosts__, http, proc -from platformio.package.client import RegistryClient +from platformio.registry.client import RegistryClient def test_platformio_cli():