platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-29 17:47:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Restructure "registry" modules

Browse Source
This commit is contained in:
Ivan Kravets
2022-06-01 12:35:55 +03:00
parent b104b840c4
commit 7e7856e44c
11 changed files with 121 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
platformio/commands/access.py
View File

@ -22,7 +22,7 @@ from tabulate import tabulate
from platformio.account.helpers import validate_username
from platformio.commands.team import validate_orgname_teamname
from platformio.package.client import RegistryClient
from platformio.registry.client import RegistryClient
def validate_client(value):

2
platformio/package/commands/publish.py
View File

@ -23,12 +23,12 @@ from tabulate import tabulate
from platformio import fs
from platformio.account.client import AccountClient
from platformio.exception import UserSideException
from platformio.package.client import RegistryClient
from platformio.package.manifest.parser import ManifestParserFactory
from platformio.package.manifest.schema import ManifestSchema
from platformio.package.meta import PackageType
from platformio.package.pack import PackagePacker
from platformio.package.unpack import FileUnpacker, TARArchiver
from platformio.registry.client import RegistryClient
def validate_datetime(ctx, param, value): # pylint: disable=unused-argument

2
platformio/package/commands/search.py
View File

@ -17,7 +17,7 @@ import math
import click
from platformio import util
from platformio.package.client import RegistryClient
from platformio.registry.client import RegistryClient
@click.command("search", short_help="Search for packages")

2
platformio/package/commands/show.py
View File

@ -19,9 +19,9 @@ from tabulate import tabulate
from platformio import fs, util
from platformio.exception import UserSideException
from platformio.package.client import RegistryClient
from platformio.package.manager._registry import PackageManagerRegistryMixin
from platformio.package.meta import PackageSpec, PackageType
from platformio.registry.client import RegistryClient
@click.command("show", short_help="Show package information")

2
platformio/package/commands/unpublish.py
View File

@ -15,8 +15,8 @@
import click
from platformio.account.client import AccountClient
from platformio.package.client import RegistryClient
from platformio.package.meta import PackageSpec, PackageType
from platformio.registry.client import RegistryClient
@click.command("unpublish", short_help="Remove a pushed package from the registry")

86
platformio/package/manager/_registry.py
View File

@ -12,97 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import time
from urllib.parse import urlparse
import click
from platformio import __registry_mirror_hosts__
from platformio.cache import ContentCache
from platformio.http import HTTPClient
from platformio.package.client import RegistryClient
from platformio.package.exception import UnknownPackageError
from platformio.package.meta import PackageSpec
from platformio.package.version import cast_version_to_semver
class RegistryFileMirrorIterator(object):
HTTP_CLIENT_INSTANCES = {}
def __init__(self, download_url):
self.download_url = download_url
self._url_parts = urlparse(download_url)
self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc)
self._visited_mirrors = []
def __iter__(self): # pylint: disable=non-iterator-returned
return self
def __next__(self):
cache_key = ContentCache.key_from_args(
"head", self.download_url, self._visited_mirrors
)
with ContentCache("http") as cc:
result = cc.get(cache_key)
if result is not None:
try:
headers = json.loads(result)
return (
headers["Location"],
headers["X-PIO-Content-SHA256"],
)
except (ValueError, KeyError):
pass
http = self.get_http_client()
response = http.send_request(
"head",
self._url_parts.path,
allow_redirects=False,
params=dict(bypass=",".join(self._visited_mirrors))
if self._visited_mirrors
else None,
x_with_authorization=RegistryClient.allowed_private_packages(),
)
stop_conditions = [
response.status_code not in (302, 307),
not response.headers.get("Location"),
not response.headers.get("X-PIO-Mirror"),
response.headers.get("X-PIO-Mirror") in self._visited_mirrors,
]
if any(stop_conditions):
raise StopIteration
self._visited_mirrors.append(response.headers.get("X-PIO-Mirror"))
cc.set(
cache_key,
json.dumps(
{
"Location": response.headers.get("Location"),
"X-PIO-Content-SHA256": response.headers.get(
"X-PIO-Content-SHA256"
),
}
),
"1h",
)
return (
response.headers.get("Location"),
response.headers.get("X-PIO-Content-SHA256"),
)
def get_http_client(self):
if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES:
endpoints = [self._mirror]
for host in __registry_mirror_hosts__:
endpoint = f"https://dl.{host}"
if endpoint not in endpoints:
endpoints.append(endpoint)
RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient(
endpoints
)
return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror]
from platformio.registry.client import RegistryClient
from platformio.registry.mirror import RegistryFileMirrorIterator
class PackageManagerRegistryMixin(object):

13
platformio/registry/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

0
platformio/package/client.py → platformio/registry/client.py
View File

99
platformio/registry/mirror.py Normal file
View File

@ -0,0 +1,99 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from urllib.parse import urlparse
from platformio import __registry_mirror_hosts__
from platformio.cache import ContentCache
from platformio.http import HTTPClient
from platformio.registry.client import RegistryClient
class RegistryFileMirrorIterator(object):
HTTP_CLIENT_INSTANCES = {}
def __init__(self, download_url):
self.download_url = download_url
self._url_parts = urlparse(download_url)
self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc)
self._visited_mirrors = []
def __iter__(self): # pylint: disable=non-iterator-returned
return self
def __next__(self):
cache_key = ContentCache.key_from_args(
"head", self.download_url, self._visited_mirrors
)
with ContentCache("http") as cc:
result = cc.get(cache_key)
if result is not None:
try:
headers = json.loads(result)
return (
headers["Location"],
headers["X-PIO-Content-SHA256"],
)
except (ValueError, KeyError):
pass
http = self.get_http_client()
response = http.send_request(
"head",
self._url_parts.path,
allow_redirects=False,
params=dict(bypass=",".join(self._visited_mirrors))
if self._visited_mirrors
else None,
x_with_authorization=RegistryClient.allowed_private_packages(),
)
stop_conditions = [
response.status_code not in (302, 307),
not response.headers.get("Location"),
not response.headers.get("X-PIO-Mirror"),
response.headers.get("X-PIO-Mirror") in self._visited_mirrors,
]
if any(stop_conditions):
raise StopIteration
self._visited_mirrors.append(response.headers.get("X-PIO-Mirror"))
cc.set(
cache_key,
json.dumps(
{
"Location": response.headers.get("Location"),
"X-PIO-Content-SHA256": response.headers.get(
"X-PIO-Content-SHA256"
),
}
),
"1h",
)
return (
response.headers.get("Location"),
response.headers.get("X-PIO-Content-SHA256"),
)
def get_http_client(self):
if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES:
endpoints = [self._mirror]
for host in __registry_mirror_hosts__:
endpoint = f"https://dl.{host}"
if endpoint not in endpoints:
endpoints.append(endpoint)
RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient(
endpoints
)
return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror]

2
tests/commands/test_lib.py
View File

@ -21,10 +21,10 @@ import pytest
import semantic_version
from platformio.commands.lib.command import cli as cmd_lib
from platformio.package.client import RegistryClient
from platformio.package.meta import PackageType
from platformio.package.vcsclient import VCSClientFactory
from platformio.project.config import ProjectConfig
from platformio.registry.client import RegistryClient
def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory):

2
tests/misc/test_misc.py
View File

@ -18,7 +18,7 @@ import pytest
import requests
from platformio import __check_internet_hosts__, http, proc
from platformio.package.client import RegistryClient
from platformio.registry.client import RegistryClient
def test_platformio_cli():