From 19cdc7d34ad2be76b46790ebb5b9c206e7045c80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 22:01:32 +0300 Subject: [PATCH] Initial support for package publishing in to the registry --- platformio/__init__.py | 4 +- platformio/clients/__init__.py | 13 ++++++ platformio/clients/registry.py | 41 ++++++++++++++++++ platformio/clients/rest.py | 62 +++++++++++++++++++++++++++ platformio/commands/account/client.py | 6 +-- platformio/commands/package.py | 59 +++++++++++++++++++++++++ platformio/package/manifest/parser.py | 2 +- platformio/package/pack.py | 37 ++++++++++++++++ 8 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 platformio/clients/__init__.py create mode 100644 platformio/clients/registry.py create mode 100644 platformio/clients/rest.py create mode 100644 platformio/commands/package.py diff --git a/platformio/__init__.py b/platformio/__init__.py index 92359835..60621751 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -34,5 +34,7 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" -__pioaccount_api__ = "https://api.accounts.platformio.org" + +__accounts_api__ = "https://api.accounts.platformio.org" +__registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" diff --git a/platformio/clients/__init__.py b/platformio/clients/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/clients/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py new file mode 100644 index 00000000..7ab3a3c4 --- /dev/null +++ b/platformio/clients/registry.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import __registry_api__ +from platformio.clients.rest import RESTClient +from platformio.commands.account.client import AccountClient +from platformio.package.pack import PackageType + + +class RegistryClient(RESTClient): + def __init__(self): + super(RegistryClient, self).__init__(base_url=__registry_api__) + + def publish_package( + self, archive_path, owner=None, released_at=None, private=False + ): + client = AccountClient() + if not owner: + owner = client.get_account_info(offline=True).get("profile").get("username") + with open(archive_path, "rb") as fp: + response = self.send_request( + "post", + "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), + params={"private": 1 if private else 0, "released_at": released_at}, + headers={ + "Authorization": "Bearer %s" % client.fetch_authentication_token() + }, + data=fp, + ) + return response diff --git a/platformio/clients/rest.py b/platformio/clients/rest.py new file mode 100644 index 00000000..4921e2cc --- /dev/null +++ b/platformio/clients/rest.py @@ -0,0 +1,62 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests.adapters +from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error + +from platformio import app, util +from platformio.exception import PlatformioException + + +class RESTClientError(PlatformioException): + pass + + +class RESTClient(object): + def __init__(self, base_url): + if base_url.endswith("/"): + base_url = base_url[:-1] + self.base_url = base_url + self._session = requests.Session() + self._session.headers.update({"User-Agent": app.get_user_agent()}) + retry = Retry( + total=5, + backoff_factor=1, + method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], + status_forcelist=[500, 502, 503, 504], + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retry) + self._session.mount(base_url, adapter) + + def send_request(self, method, path, **kwargs): + # check internet before and resolve issue with 60 seconds timeout + util.internet_on(raise_exception=True) + try: + response = getattr(self._session, method)(self.base_url + path, **kwargs) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: + raise RESTClientError(e) + return self.raise_error_from_response(response) + + @staticmethod + def raise_error_from_response(response, expected_codes=(200, 201, 202)): + if response.status_code in expected_codes: + try: + return response.json() + except ValueError: + pass + try: + message = response.json()["message"] + except (KeyError, ValueError): + message = response.text + raise RESTClientError(message) diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py index fb679dc0..e49f30bb 100644 --- a/platformio/commands/account/client.py +++ b/platformio/commands/account/client.py @@ -20,7 +20,7 @@ import time import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import __pioaccount_api__, app +from platformio import __accounts_api__, app from platformio.commands.account import exception from platformio.exception import InternetIsOffline @@ -30,7 +30,7 @@ class AccountClient(object): SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__( - self, api_base_url=__pioaccount_api__, retries=3, + self, api_base_url=__accounts_api__, retries=3, ): if api_base_url.endswith("/"): api_base_url = api_base_url[:-1] @@ -184,7 +184,7 @@ class AccountClient(object): ) return response - def get_account_info(self, offline): + def get_account_info(self, offline=False): account = app.get_state_item("account") if not account: raise exception.AccountNotAuthorized() diff --git a/platformio/commands/package.py b/platformio/commands/package.py new file mode 100644 index 00000000..a2b6c383 --- /dev/null +++ b/platformio/commands/package.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +import click + +from platformio.clients.registry import RegistryClient +from platformio.package.pack import PackagePacker + + +def validate_datetime(ctx, param, value): # pylint: disable=unused-argument + try: + datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError as e: + raise click.BadParameter(e) + return value + + +@click.group("package", short_help="Package Manager") +def cli(): + pass + + +@cli.command( + "publish", short_help="Publish a package to the PlatformIO Universal Registry" +) +@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +@click.option( + "--owner", + help="PIO Account username (could be organization username). " + "Default is set to a username of the authorized PIO Account", +) +@click.option( + "--released-at", + callback=validate_datetime, + help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52", +) +@click.option("--private", is_flag=True, help="Restricted access (not a public)") +def package_publish(package, owner, released_at, private): + p = PackagePacker(package) + archive_path = p.pack() + response = RegistryClient().publish_package( + archive_path, owner, released_at, private + ) + os.remove(archive_path) + click.secho(response.get("message"), fg="green") diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index e8ec5929..bf017721 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -40,7 +40,7 @@ class ManifestFileType(object): @classmethod def items(cls): - return get_object_members(ManifestFileType) + return get_object_members(cls) @classmethod def from_uri(cls, uri): diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 1e18c55a..ecf14a4f 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -19,12 +19,49 @@ import tarfile import tempfile from platformio import fs +from platformio.compat import get_object_members from platformio.package.exception import PackageException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.unpacker import FileUnpacker +class PackageType(object): + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r|gz") as tf: + for t in sorted(cls.items().values()): + try: + for manifest in manifest_map[t]: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + + class PackagePacker(object): EXCLUDE_DEFAULT = [ "._*",