From 102aa5f22bec5c7c79de71f3db3b8b252e7123f8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 17:49:29 +0300 Subject: [PATCH] Port legacy API requests to the new registry client --- platformio/__init__.py | 2 - platformio/clients/registry.py | 15 ++-- platformio/commands/lib/command.py | 24 ++++--- platformio/commands/platform.py | 6 +- platformio/package/manager/platform.py | 7 +- platformio/util.py | 98 -------------------------- 6 files changed, 28 insertions(+), 124 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b51a0be0..0d25faf8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -39,8 +39,6 @@ __email__ = "contact@platformio.org" __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" -__apiurl__ = "https://api.platformio.org" - __accounts_api__ = "https://api.accounts.platformio.org" __registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f8130c60..6111846d 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -17,11 +17,6 @@ from platformio.clients.account import AccountClient from platformio.clients.http import HTTPClient, HTTPClientError from platformio.package.meta import PackageType -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - # pylint: disable=too-many-arguments @@ -35,7 +30,7 @@ class RegistryClient(HTTPClient): token = AccountClient().fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.request_json_data(*args, **kwargs) + return self.fetch_json_data(*args, **kwargs) def publish_package( self, archive_path, owner=None, released_at=None, private=False, notify=True @@ -123,17 +118,17 @@ class RegistryClient(HTTPClient): search_query.append('%s:"%s"' % (name[:-1], value)) if query: search_query.append(query) - params = dict(query=quote(" ".join(search_query))) + params = dict(query=" ".join(search_query)) if page: params["page"] = int(page) - return self.request_json_data("get", "/v3/packages", params=params) + return self.fetch_json_data("get", "/v3/packages", params=params) def get_package(self, type_, owner, name, version=None): try: - return self.request_json_data( + return self.fetch_json_data( "get", "/v3/packages/{owner}/{type}/{name}".format( - type=type_, owner=owner.lower(), name=quote(name.lower()) + type=type_, owner=owner.lower(), name=name.lower() ), params=dict(version=version) if version else None, ) diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 6ca0ee77..96d39814 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -347,6 +347,7 @@ def lib_list(ctx, json_output): help="Do not prompt, automatically paginate with delay", ) def lib_search(query, json_output, page, noninteractive, **filters): + regclient = LibraryPackageManager().get_registry_client_instance() if not query: query = [] if not isinstance(query, list): @@ -356,8 +357,11 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d" + result = regclient.fetch_json_data( + "get", + "/v2/lib/search", + params=dict(query=" ".join(query), page=page), + cache_valid="1d", ) if json_output: @@ -406,9 +410,10 @@ def lib_search(query, json_output, page, noninteractive, **filters): time.sleep(5) elif not click.confirm("Show next libraries?"): break - result = util.get_api_result( + result = regclient.fetch_json_data( + "get", "/v2/lib/search", - {"query": " ".join(query), "page": int(result["page"]) + 1}, + params=dict(query=" ".join(query), page=int(result["page"]) + 1), cache_valid="1d", ) @@ -438,10 +443,10 @@ def lib_builtin(storage, json_output): @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) def lib_show(library, json_output): - lib_id = LibraryPackageManager().reveal_registry_package_id( - library, silent=json_output - ) - lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") + lm = LibraryPackageManager() + lib_id = lm.reveal_registry_package_id(library, silent=json_output) + regclient = lm.get_registry_client_instance() + lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h") if json_output: return click.echo(dump_json_to_unicode(lib)) @@ -534,7 +539,8 @@ def lib_register(config_url): # pylint: disable=unused-argument @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): - result = util.get_api_result("/lib/stats", cache_valid="1h") + regclient = LibraryPackageManager().get_registry_client_instance() + result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h") if json_output: return click.echo(dump_json_to_unicode(result)) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index b996a16a..803149b1 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -57,7 +57,8 @@ def _print_platforms(platforms): def _get_registry_platforms(): - return util.get_api_result("/platforms", cache_valid="7d") + regclient = PlatformPackageManager().get_registry_client_instance() + return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d") def _get_platform_data(*args, **kwargs): @@ -188,8 +189,9 @@ def platform_search(query, json_output): @click.argument("query", required=False) @click.option("--json-output", is_flag=True) def platform_frameworks(query, json_output): + regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] - for framework in util.get_api_result("/frameworks", cache_valid="7d"): + for framework in regclient.fetch_json_data("get", "/v2/frameworks", cache_valid="1d"): if query == "all": query = "" search_data = dump_json_to_unicode(framework) diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 2172e672..71e8c5fb 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -164,9 +164,10 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an boards.append(board) return boards - @staticmethod - def get_registered_boards(): - return util.get_api_result("/boards", cache_valid="7d") + def get_registered_boards(self): + return self.get_registry_client_instance().fetch_json_data( + "get", "/v2/boards", cache_valid="1d" + ) def get_all_boards(self): boards = self.get_installed_boards() diff --git a/platformio/util.py b/platformio/util.py index a0686377..04576b24 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -242,104 +242,6 @@ def get_mdns_services(): return items -@memoized(expire="60s") -def _api_request_session(): - return requests.Session() - - -@throttle(500) -def _get_api_result( - url, params=None, data=None, auth=None # pylint: disable=too-many-branches -): - # pylint: disable=import-outside-toplevel - from platformio.app import get_user_agent, get_setting - - result = {} - r = None - verify_ssl = sys.version_info >= (2, 7, 9) - - if not url.startswith("http"): - url = __apiurl__ + url - if not get_setting("strict_ssl"): - url = url.replace("https://", "http://") - - headers = {"User-Agent": get_user_agent()} - try: - if data: - r = _api_request_session().post( - url, - params=params, - data=data, - headers=headers, - auth=auth, - verify=verify_ssl, - timeout=DEFAULT_REQUESTS_TIMEOUT, - ) - else: - r = _api_request_session().get( - url, - params=params, - headers=headers, - auth=auth, - verify=verify_ssl, - timeout=DEFAULT_REQUESTS_TIMEOUT, - ) - result = r.json() - r.raise_for_status() - return r.text - except requests.exceptions.HTTPError as e: - if result and "message" in result: - raise exception.APIRequestError(result["message"]) - if result and "errors" in result: - raise exception.APIRequestError(result["errors"][0]["title"]) - raise exception.APIRequestError(e) - except ValueError: - raise exception.APIRequestError("Invalid response: %s" % r.text.encode("utf-8")) - finally: - if r: - r.close() - return None - - -def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): - from platformio.app import ContentCache # pylint: disable=import-outside-toplevel - - total = 0 - max_retries = 5 - cache_key = ( - ContentCache.key_from_args(url, params, data, auth) if cache_valid else None - ) - while total < max_retries: - try: - with ContentCache() as cc: - if cache_key: - result = cc.get(cache_key) - if result is not None: - return json.loads(result) - - # check internet before and resolve issue with 60 seconds timeout - internet_on(raise_exception=True) - - result = _get_api_result(url, params, data) - if cache_valid: - with ContentCache() as cc: - cc.set(cache_key, result, cache_valid) - return json.loads(result) - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - total += 1 - if not PlatformioCLI.in_silence(): - click.secho( - "[API] ConnectionError: {0} (incremented retry: max={1}, " - "total={2})".format(e, max_retries, total), - fg="yellow", - ) - time.sleep(2 * total) - - raise exception.APIRequestError( - "Could not connect to PlatformIO API Service. Please try later." - ) - - def pioversion_to_intstr(): vermatch = re.match(r"^([\d\.]+)", __version__) assert vermatch