From 0eb8895959c90cda07f014294d854cf76913c0d7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 14:25:28 +0300 Subject: [PATCH 001/223] =?UTF-8?q?Add=20support=20for=20=E2=80=9Cglobstar?= =?UTF-8?q?/**=E2=80=9D=20(recursive)=20pattern?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 5 +++++ platformio/commands/check/tools/base.py | 5 ++--- platformio/commands/ci.py | 10 ++++------ platformio/commands/home/rpc/handlers/os.py | 7 ++++--- platformio/compat.py | 7 +++++++ platformio/fs.py | 5 ++--- platformio/project/config.py | 11 ++++++++--- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ab9119fe..250e680a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,11 @@ Release Notes PlatformIO Core 4 ----------------- +4.3.5 (2020-??-??) +~~~~~~~~~~~~~~~~~~ + +* Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. + 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index d6f5d4f1..d873810d 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import os from tempfile import NamedTemporaryFile import click -from platformio import fs, proc +from platformio import compat, fs, proc from platformio.commands.check.defect import DefectItem from platformio.project.helpers import load_project_ide_data @@ -183,7 +182,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes result["c++"].append(os.path.realpath(path)) for pattern in patterns: - for item in glob.glob(pattern): + for item in compat.glob_recursive(pattern): if not os.path.isdir(item): _add_file(item) for root, _, files in os.walk(item, followlinks=True): diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 9a48f262..f68b2bb7 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from glob import glob from os import getenv, makedirs, remove from os.path import basename, isdir, isfile, join, realpath from shutil import copyfile, copytree @@ -20,11 +19,10 @@ from tempfile import mkdtemp import click -from platformio import app, fs +from platformio import app, compat, fs from platformio.commands.project import project_init as cmd_project_init from platformio.commands.project import validate_boards from platformio.commands.run.command import cli as cmd_run -from platformio.compat import glob_escape from platformio.exception import CIBuildEnvsEmpty from platformio.project.config import ProjectConfig @@ -36,7 +34,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument if p.startswith("~"): value[i] = fs.expanduser(p) value[i] = realpath(value[i]) - if not glob(value[i]): + if not compat.glob_recursive(value[i]): invalid_path = p break try: @@ -98,7 +96,7 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches continue contents = [] for p in patterns: - contents += glob(p) + contents += compat.glob_recursive(p) _copy_contents(join(build_dir, dir_name), contents) if project_conf and isfile(project_conf): @@ -159,7 +157,7 @@ def _copy_contents(dst_dir, contents): def _exclude_contents(dst_dir, patterns): contents = [] for p in patterns: - contents += glob(join(glob_escape(dst_dir), p)) + contents += compat.glob_recursive(join(compat.glob_escape(dst_dir), p)) for path in contents: path = realpath(path) if isdir(path): diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 2997e8aa..3b4bd4e1 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import glob import io import os import shutil @@ -25,7 +24,7 @@ from twisted.internet import defer # pylint: disable=import-error from platformio import app, fs, util from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding +from platformio.compat import PY2, get_filesystem_encoding, glob_recursive class OSRPC(object): @@ -115,7 +114,9 @@ class OSRPC(object): pathnames = [pathnames] result = set() for pathname in pathnames: - result |= set(glob.glob(os.path.join(root, pathname) if root else pathname)) + result |= set( + glob_recursive(os.path.join(root, pathname) if root else pathname) + ) return list(result) @staticmethod diff --git a/platformio/compat.py b/platformio/compat.py index c812e98d..7cfe47f1 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -15,6 +15,7 @@ # pylint: disable=unused-import, no-name-in-module, import-error, # pylint: disable=no-member, undefined-variable +import glob import inspect import json import locale @@ -81,6 +82,9 @@ if PY2: _magic_check = re.compile("([*?[])") _magic_check_bytes = re.compile(b"([*?[])") + def glob_recursive(pathname): + return glob.glob(pathname) + def glob_escape(pathname): """Escape all special characters.""" # https://github.com/python/cpython/blob/master/Lib/glob.py#L161 @@ -122,6 +126,9 @@ else: return obj return json.dumps(obj, ensure_ascii=False, sort_keys=True) + def glob_recursive(pathname): + return glob.glob(pathname, recursive=True) + def load_python_module(name, pathname): spec = importlib.util.spec_from_file_location(name, pathname) module = importlib.util.module_from_spec(spec) diff --git a/platformio/fs.py b/platformio/fs.py index 575a14e5..5122c882 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -18,12 +18,11 @@ import re import shutil import stat import sys -from glob import glob import click from platformio import exception -from platformio.compat import WINDOWS, glob_escape +from platformio.compat import WINDOWS, glob_escape, glob_recursive class cd(object): @@ -135,7 +134,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True): src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): items = set() - for item in glob(os.path.join(glob_escape(src_dir), pattern)): + for item in glob_recursive(os.path.join(glob_escape(src_dir), pattern)): if os.path.isdir(item): for root, _, files in os.walk(item, followlinks=followlinks): for f in files: diff --git a/platformio/project/config.py b/platformio/project/config.py index 23d089bf..786f080a 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import json import os import re @@ -21,7 +20,13 @@ from hashlib import sha1 import click from platformio import fs -from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types +from platformio.compat import ( + PY2, + WINDOWS, + glob_recursive, + hashlib_encode_data, + string_types, +) from platformio.project import exception from platformio.project.options import ProjectOptions @@ -117,7 +122,7 @@ class ProjectConfigBase(object): for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): pattern = fs.expanduser(pattern) - for item in glob.glob(pattern): + for item in glob_recursive(pattern): self.read(item) def _maintain_renaimed_options(self): From 38699cca8fe97a47a5b36d248e43586075432caa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 14:26:42 +0300 Subject: [PATCH 002/223] Bump version to 4.3.5a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 1e2e3fd1..92359835 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, 4) +VERSION = (4, 3, "5a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 58470e89114b529030e4a531ec4d98d5c5888678 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 14:30:43 +0300 Subject: [PATCH 003/223] PY2 lint fix --- platformio/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/compat.py b/platformio/compat.py index 7cfe47f1..7f749fc9 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -13,7 +13,7 @@ # limitations under the License. # pylint: disable=unused-import, no-name-in-module, import-error, -# pylint: disable=no-member, undefined-variable +# pylint: disable=no-member, undefined-variable, unexpected-keyword-arg import glob import inspect From 49cc5d606b8dc9ba2c8c3941830672eaee703039 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 21:58:58 +0300 Subject: [PATCH 004/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 68341524..b26f9eb4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 683415246be491a91c2f8e63fa46b0e6ab55f91b +Subproject commit b26f9eb4834bd5de1cc28c9d5a6cd99b4332ebef From 19cdc7d34ad2be76b46790ebb5b9c206e7045c80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 22:01:32 +0300 Subject: [PATCH 005/223] Initial support for package publishing in to the registry --- platformio/__init__.py | 4 +- platformio/clients/__init__.py | 13 ++++++ platformio/clients/registry.py | 41 ++++++++++++++++++ platformio/clients/rest.py | 62 +++++++++++++++++++++++++++ platformio/commands/account/client.py | 6 +-- platformio/commands/package.py | 59 +++++++++++++++++++++++++ platformio/package/manifest/parser.py | 2 +- platformio/package/pack.py | 37 ++++++++++++++++ 8 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 platformio/clients/__init__.py create mode 100644 platformio/clients/registry.py create mode 100644 platformio/clients/rest.py create mode 100644 platformio/commands/package.py diff --git a/platformio/__init__.py b/platformio/__init__.py index 92359835..60621751 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -34,5 +34,7 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" -__pioaccount_api__ = "https://api.accounts.platformio.org" + +__accounts_api__ = "https://api.accounts.platformio.org" +__registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" diff --git a/platformio/clients/__init__.py b/platformio/clients/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/clients/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py new file mode 100644 index 00000000..7ab3a3c4 --- /dev/null +++ b/platformio/clients/registry.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import __registry_api__ +from platformio.clients.rest import RESTClient +from platformio.commands.account.client import AccountClient +from platformio.package.pack import PackageType + + +class RegistryClient(RESTClient): + def __init__(self): + super(RegistryClient, self).__init__(base_url=__registry_api__) + + def publish_package( + self, archive_path, owner=None, released_at=None, private=False + ): + client = AccountClient() + if not owner: + owner = client.get_account_info(offline=True).get("profile").get("username") + with open(archive_path, "rb") as fp: + response = self.send_request( + "post", + "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), + params={"private": 1 if private else 0, "released_at": released_at}, + headers={ + "Authorization": "Bearer %s" % client.fetch_authentication_token() + }, + data=fp, + ) + return response diff --git a/platformio/clients/rest.py b/platformio/clients/rest.py new file mode 100644 index 00000000..4921e2cc --- /dev/null +++ b/platformio/clients/rest.py @@ -0,0 +1,62 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests.adapters +from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error + +from platformio import app, util +from platformio.exception import PlatformioException + + +class RESTClientError(PlatformioException): + pass + + +class RESTClient(object): + def __init__(self, base_url): + if base_url.endswith("/"): + base_url = base_url[:-1] + self.base_url = base_url + self._session = requests.Session() + self._session.headers.update({"User-Agent": app.get_user_agent()}) + retry = Retry( + total=5, + backoff_factor=1, + method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], + status_forcelist=[500, 502, 503, 504], + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retry) + self._session.mount(base_url, adapter) + + def send_request(self, method, path, **kwargs): + # check internet before and resolve issue with 60 seconds timeout + util.internet_on(raise_exception=True) + try: + response = getattr(self._session, method)(self.base_url + path, **kwargs) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: + raise RESTClientError(e) + return self.raise_error_from_response(response) + + @staticmethod + def raise_error_from_response(response, expected_codes=(200, 201, 202)): + if response.status_code in expected_codes: + try: + return response.json() + except ValueError: + pass + try: + message = response.json()["message"] + except (KeyError, ValueError): + message = response.text + raise RESTClientError(message) diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py index fb679dc0..e49f30bb 100644 --- a/platformio/commands/account/client.py +++ b/platformio/commands/account/client.py @@ -20,7 +20,7 @@ import time import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import __pioaccount_api__, app +from platformio import __accounts_api__, app from platformio.commands.account import exception from platformio.exception import InternetIsOffline @@ -30,7 +30,7 @@ class AccountClient(object): SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__( - self, api_base_url=__pioaccount_api__, retries=3, + self, api_base_url=__accounts_api__, retries=3, ): if api_base_url.endswith("/"): api_base_url = api_base_url[:-1] @@ -184,7 +184,7 @@ class AccountClient(object): ) return response - def get_account_info(self, offline): + def get_account_info(self, offline=False): account = app.get_state_item("account") if not account: raise exception.AccountNotAuthorized() diff --git a/platformio/commands/package.py b/platformio/commands/package.py new file mode 100644 index 00000000..a2b6c383 --- /dev/null +++ b/platformio/commands/package.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +import click + +from platformio.clients.registry import RegistryClient +from platformio.package.pack import PackagePacker + + +def validate_datetime(ctx, param, value): # pylint: disable=unused-argument + try: + datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError as e: + raise click.BadParameter(e) + return value + + +@click.group("package", short_help="Package Manager") +def cli(): + pass + + +@cli.command( + "publish", short_help="Publish a package to the PlatformIO Universal Registry" +) +@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +@click.option( + "--owner", + help="PIO Account username (could be organization username). " + "Default is set to a username of the authorized PIO Account", +) +@click.option( + "--released-at", + callback=validate_datetime, + help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52", +) +@click.option("--private", is_flag=True, help="Restricted access (not a public)") +def package_publish(package, owner, released_at, private): + p = PackagePacker(package) + archive_path = p.pack() + response = RegistryClient().publish_package( + archive_path, owner, released_at, private + ) + os.remove(archive_path) + click.secho(response.get("message"), fg="green") diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index e8ec5929..bf017721 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -40,7 +40,7 @@ class ManifestFileType(object): @classmethod def items(cls): - return get_object_members(ManifestFileType) + return get_object_members(cls) @classmethod def from_uri(cls, uri): diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 1e18c55a..ecf14a4f 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -19,12 +19,49 @@ import tarfile import tempfile from platformio import fs +from platformio.compat import get_object_members from platformio.package.exception import PackageException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.unpacker import FileUnpacker +class PackageType(object): + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r|gz") as tf: + for t in sorted(cls.items().values()): + try: + for manifest in manifest_map[t]: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + + class PackagePacker(object): EXCLUDE_DEFAULT = [ "._*", From 8346b9822d236dd1a7614bdd6f144d83c1dfd30f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 22:17:55 +0300 Subject: [PATCH 006/223] Implement "package pack" command --- platformio/clients/registry.py | 8 +++++--- platformio/commands/package.py | 8 ++++++++ platformio/commands/platform.py | 11 ----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 7ab3a3c4..03e5b178 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -25,16 +25,18 @@ class RegistryClient(RESTClient): def publish_package( self, archive_path, owner=None, released_at=None, private=False ): - client = AccountClient() + account = AccountClient() if not owner: - owner = client.get_account_info(offline=True).get("profile").get("username") + owner = ( + account.get_account_info(offline=True).get("profile").get("username") + ) with open(archive_path, "rb") as fp: response = self.send_request( "post", "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={"private": 1 if private else 0, "released_at": released_at}, headers={ - "Authorization": "Bearer %s" % client.fetch_authentication_token() + "Authorization": "Bearer %s" % account.fetch_authentication_token() }, data=fp, ) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index a2b6c383..261523e2 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -34,6 +34,14 @@ def cli(): pass +@cli.command("pack", short_help="Create a tarball from a package") +@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +def package_pack(package): + p = PackagePacker(package) + tarball_path = p.pack() + click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") + + @cli.command( "publish", short_help="Publish a package to the PlatformIO Universal Registry" ) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index d4ff4930..c4a9ca5d 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -20,7 +20,6 @@ from platformio import app, exception, util from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.package.pack import PackagePacker @click.group(short_help="Platform Manager") @@ -411,13 +410,3 @@ def platform_update( # pylint: disable=too-many-locals click.echo() return True - - -@cli.command( - "pack", short_help="Create a tarball from development platform/tool package" -) -@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") -def platform_pack(package): - p = PackagePacker(package) - tarball_path = p.pack() - click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") From deb12972fb900d41e42eb1a3e6978c09a034ca11 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 01:10:35 +0300 Subject: [PATCH 007/223] Implement "package unpublish" CLI --- platformio/clients/registry.py | 19 +++++++++++++++ platformio/commands/package.py | 26 ++++++++++++++++----- platformio/package/spec.py | 42 ++++++++++++++++++++++++++++++++++ tests/package/test_spec.py | 27 ++++++++++++++++++++++ 4 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 platformio/package/spec.py create mode 100644 tests/package/test_spec.py diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 03e5b178..6e9ea3fa 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -41,3 +41,22 @@ class RegistryClient(RESTClient): data=fp, ) return response + + def unpublish_package(self, name, owner=None, version=None, undo=False): + account = AccountClient() + if not owner: + owner = ( + account.get_account_info(offline=True).get("profile").get("username") + ) + path = "/v3/package/%s/%s" % (owner, name) + if version: + path = path + "/version/" + version + response = self.send_request( + "delete", + path, + params={"undo": 1 if undo else 0}, + headers={ + "Authorization": "Bearer %s" % account.fetch_authentication_token() + }, + ) + return response diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 261523e2..bf42b350 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -19,6 +19,7 @@ import click from platformio.clients.registry import RegistryClient from platformio.package.pack import PackagePacker +from platformio.package.spec import PackageSpec def validate_datetime(ctx, param, value): # pylint: disable=unused-argument @@ -38,17 +39,15 @@ def cli(): @click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") def package_pack(package): p = PackagePacker(package) - tarball_path = p.pack() - click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") + archive_path = p.pack() + click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") -@cli.command( - "publish", short_help="Publish a package to the PlatformIO Universal Registry" -) +@cli.command("publish", short_help="Publish a package to the registry") @click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") @click.option( "--owner", - help="PIO Account username (could be organization username). " + help="PIO Account username (can be organization username). " "Default is set to a username of the authorized PIO Account", ) @click.option( @@ -65,3 +64,18 @@ def package_publish(package, owner, released_at, private): ) os.remove(archive_path) click.secho(response.get("message"), fg="green") + + +@cli.command("unpublish", short_help="Remove a pushed package from the registry") +@click.argument("package", required=True, metavar="[<@organization>/][@]") +@click.option( + "--undo", + is_flag=True, + help="Undo a remove, putting a version back into the registry", +) +def package_unpublish(package, undo): + spec = PackageSpec(package) + response = RegistryClient().unpublish_package( + owner=spec.organization, name=spec.name, version=spec.version, undo=undo + ) + click.secho(response.get("message"), fg="green") diff --git a/platformio/package/spec.py b/platformio/package/spec.py new file mode 100644 index 00000000..0de8227a --- /dev/null +++ b/platformio/package/spec.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class PackageSpec(object): + def __init__(self, raw=None, organization=None, name=None, version=None): + if raw is not None: + organization, name, version = self.parse(raw) + + self.organization = organization + self.name = name + self.version = version + + @staticmethod + def parse(raw): + organization = None + name = None + version = None + raw = raw.strip() + if raw.startswith("@") and "/" in raw: + tokens = raw[1:].split("/", 1) + organization = tokens[0].strip() + raw = tokens[1] + if "@" in raw: + name, version = raw.split("@", 1) + name = name.strip() + version = version.strip() + else: + name = raw.strip() + + return organization, name, version diff --git a/tests/package/test_spec.py b/tests/package/test_spec.py new file mode 100644 index 00000000..1886a836 --- /dev/null +++ b/tests/package/test_spec.py @@ -0,0 +1,27 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.spec import PackageSpec + + +def test_parser(): + inputs = [ + ("foo", (None, "foo", None)), + ("@org/foo", ("org", "foo", None)), + ("@org/foo @ 1.2.3", ("org", "foo", "1.2.3")), + ("bar @ 1.2.3", (None, "bar", "1.2.3")), + ("cat@^1.2", (None, "cat", "^1.2")), + ] + for raw, result in inputs: + assert PackageSpec.parse(raw) == result From 0c301b2f5dcadaea4f53c4114895d91ef40bd287 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 01:14:07 +0300 Subject: [PATCH 008/223] Fix order of arguments --- platformio/commands/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index bf42b350..93423cc7 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -76,6 +76,6 @@ def package_publish(package, owner, released_at, private): def package_unpublish(package, undo): spec = PackageSpec(package) response = RegistryClient().unpublish_package( - owner=spec.organization, name=spec.name, version=spec.version, undo=undo + name=spec.name, owner=spec.organization, version=spec.version, undo=undo ) click.secho(response.get("message"), fg="green") From e706a2cfe23d2fb66f00f5937cf97ff9ff63970a Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 27 May 2020 13:39:58 +0300 Subject: [PATCH 009/223] Refactor pio account client. Resolve #3525 (#3529) --- .../account/client.py => clients/account.py} | 116 ++++++------------ platformio/clients/registry.py | 2 +- .../{account/command.py => account.py} | 5 +- platformio/commands/account/__init__.py | 13 -- platformio/commands/account/exception.py | 30 ----- .../commands/home/rpc/handlers/account.py | 2 +- platformio/commands/remote/factory/client.py | 2 +- tests/commands/test_account.py | 2 +- 8 files changed, 45 insertions(+), 127 deletions(-) rename platformio/{commands/account/client.py => clients/account.py} (65%) rename platformio/commands/{account/command.py => account.py} (98%) delete mode 100644 platformio/commands/account/__init__.py delete mode 100644 platformio/commands/account/exception.py diff --git a/platformio/commands/account/client.py b/platformio/clients/account.py similarity index 65% rename from platformio/commands/account/client.py rename to platformio/clients/account.py index e49f30bb..078e2b8a 100644 --- a/platformio/commands/account/client.py +++ b/platformio/clients/account.py @@ -12,47 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=unused-argument - import os import time -import requests.adapters -from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error - from platformio import __accounts_api__, app -from platformio.commands.account import exception -from platformio.exception import InternetIsOffline +from platformio.clients.rest import RESTClient +from platformio.exception import PlatformioException -class AccountClient(object): +class AccountError(PlatformioException): + + MESSAGE = "{0}" + + +class AccountNotAuthorized(AccountError): + + MESSAGE = "You are not authorized! Please log in to PIO Account." + + +class AccountAlreadyAuthorized(AccountError): + + MESSAGE = "You are already authorized with {0} account." + + +class AccountClient(RESTClient): SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 - def __init__( - self, api_base_url=__accounts_api__, retries=3, - ): - if api_base_url.endswith("/"): - api_base_url = api_base_url[:-1] - self.api_base_url = api_base_url - self._session = requests.Session() - self._session.headers.update({"User-Agent": app.get_user_agent()}) - retry = Retry( - total=retries, - read=retries, - connect=retries, - backoff_factor=2, - method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], - ) - adapter = requests.adapters.HTTPAdapter(max_retries=retry) - self._session.mount(api_base_url, adapter) + def __init__(self): + super(AccountClient, self).__init__(base_url=__accounts_api__) @staticmethod def get_refresh_token(): try: return app.get_state_item("account").get("auth").get("refresh_token") except: # pylint:disable=bare-except - raise exception.AccountNotAuthorized() + raise AccountNotAuthorized() @staticmethod def delete_local_session(): @@ -72,14 +67,12 @@ class AccountClient(object): except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthorized( + raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) result = self.send_request( - "post", - self.api_base_url + "/v1/login", - data={"username": username, "password": password}, + "post", "/v1/login", data={"username": username, "password": password}, ) app.set_state_item("account", result) return result @@ -90,13 +83,13 @@ class AccountClient(object): except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthorized( + raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) result = self.send_request( "post", - self.api_base_url + "/v1/login/code", + "/v1/login/code", data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, ) app.set_state_item("account", result) @@ -107,11 +100,9 @@ class AccountClient(object): self.delete_local_session() try: self.send_request( - "post", - self.api_base_url + "/v1/logout", - data={"refresh_token": refresh_token}, + "post", "/v1/logout", data={"refresh_token": refresh_token}, ) - except exception.AccountError: + except AccountError: pass return True @@ -119,7 +110,7 @@ class AccountClient(object): token = self.fetch_authentication_token() self.send_request( "post", - self.api_base_url + "/v1/password", + "/v1/password", headers={"Authorization": "Bearer %s" % token}, data={"old_password": old_password, "new_password": new_password}, ) @@ -133,13 +124,13 @@ class AccountClient(object): except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthorized( + raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) return self.send_request( "post", - self.api_base_url + "/v1/registration", + "/v1/registration", data={ "username": username, "email": email, @@ -153,23 +144,19 @@ class AccountClient(object): token = self.fetch_authentication_token() result = self.send_request( "post", - self.api_base_url + "/v1/token", + "/v1/token", headers={"Authorization": "Bearer %s" % token}, data={"password": password, "regenerate": 1 if regenerate else 0}, ) return result.get("auth_token") def forgot_password(self, username): - return self.send_request( - "post", self.api_base_url + "/v1/forgot", data={"username": username}, - ) + return self.send_request("post", "/v1/forgot", data={"username": username},) def get_profile(self): token = self.fetch_authentication_token() return self.send_request( - "get", - self.api_base_url + "/v1/profile", - headers={"Authorization": "Bearer %s" % token}, + "get", "/v1/profile", headers={"Authorization": "Bearer %s" % token}, ) def update_profile(self, profile, current_password): @@ -178,7 +165,7 @@ class AccountClient(object): self.delete_local_state("summary") response = self.send_request( "put", - self.api_base_url + "/v1/profile", + "/v1/profile", headers={"Authorization": "Bearer %s" % token}, data=profile, ) @@ -187,7 +174,7 @@ class AccountClient(object): def get_account_info(self, offline=False): account = app.get_state_item("account") if not account: - raise exception.AccountNotAuthorized() + raise AccountNotAuthorized() if ( account.get("summary") and account["summary"].get("expire_at", 0) > time.time() @@ -202,9 +189,7 @@ class AccountClient(object): } token = self.fetch_authentication_token() result = self.send_request( - "get", - self.api_base_url + "/v1/summary", - headers={"Authorization": "Bearer %s" % token}, + "get", "/v1/summary", headers={"Authorization": "Bearer %s" % token}, ) account["summary"] = dict( profile=result.get("profile"), @@ -227,36 +212,13 @@ class AccountClient(object): try: result = self.send_request( "post", - self.api_base_url + "/v1/login", + "/v1/login", headers={ "Authorization": "Bearer %s" % auth.get("refresh_token") }, ) app.set_state_item("account", result) return result.get("auth").get("access_token") - except exception.AccountError: + except AccountError: self.delete_local_session() - raise exception.AccountNotAuthorized() - - def send_request(self, method, url, headers=None, data=None): - try: - response = getattr(self._session, method)( - url, headers=headers or {}, data=data or {} - ) - except requests.exceptions.ConnectionError: - raise InternetIsOffline() - return self.raise_error_from_response(response) - - def raise_error_from_response(self, response, expected_codes=(200, 201, 202)): - if response.status_code in expected_codes: - try: - return response.json() - except ValueError: - pass - try: - message = response.json()["message"] - except (KeyError, ValueError): - message = response.text - if "Authorization session has been expired" in message: - self.delete_local_session() - raise exception.AccountError(message) + raise AccountNotAuthorized() diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 6e9ea3fa..8c4d41c6 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -13,8 +13,8 @@ # limitations under the License. from platformio import __registry_api__ +from platformio.clients.account import AccountClient from platformio.clients.rest import RESTClient -from platformio.commands.account.client import AccountClient from platformio.package.pack import PackageType diff --git a/platformio/commands/account/command.py b/platformio/commands/account.py similarity index 98% rename from platformio/commands/account/command.py rename to platformio/commands/account.py index 0177d00a..78c5aa9e 100644 --- a/platformio/commands/account/command.py +++ b/platformio/commands/account.py @@ -21,8 +21,7 @@ import re import click from tabulate import tabulate -from platformio.commands.account import exception -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient, AccountNotAuthorized @click.group("account", short_help="Manage PIO Account") @@ -167,7 +166,7 @@ def account_update(current_password, **kwargs): return None try: client.logout() - except exception.AccountNotAuthorized: + except AccountNotAuthorized: pass if email_changed: return click.secho( diff --git a/platformio/commands/account/__init__.py b/platformio/commands/account/__init__.py deleted file mode 100644 index b0514903..00000000 --- a/platformio/commands/account/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/platformio/commands/account/exception.py b/platformio/commands/account/exception.py deleted file mode 100644 index a1a0059e..00000000 --- a/platformio/commands/account/exception.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from platformio.exception import PlatformioException - - -class AccountError(PlatformioException): - - MESSAGE = "{0}" - - -class AccountNotAuthorized(AccountError): - - MESSAGE = "You are not authorized! Please log in to PIO Account." - - -class AccountAlreadyAuthorized(AccountError): - - MESSAGE = "You are already authorized with {0} account." diff --git a/platformio/commands/home/rpc/handlers/account.py b/platformio/commands/home/rpc/handlers/account.py index 911006bc..d28379f8 100644 --- a/platformio/commands/home/rpc/handlers/account.py +++ b/platformio/commands/home/rpc/handlers/account.py @@ -14,7 +14,7 @@ import jsonrpc # pylint: disable=import-error -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient class AccountRPC(object): diff --git a/platformio/commands/remote/factory/client.py b/platformio/commands/remote/factory/client.py index 26abe080..2b47ab01 100644 --- a/platformio/commands/remote/factory/client.py +++ b/platformio/commands/remote/factory/client.py @@ -17,7 +17,7 @@ from twisted.internet import defer, protocol, reactor # pylint: disable=import- from twisted.spread import pb # pylint: disable=import-error from platformio.app import get_host_id -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory): diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index ef7ffbad..5b160f0c 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -18,7 +18,7 @@ import time import pytest -from platformio.commands.account.command import cli as cmd_account +from platformio.commands.account import cli as cmd_account pytestmark = pytest.mark.skipif( not ( From c06859aa9fc7dfe900dcbb56a87445cb372b96ff Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 14:30:27 +0300 Subject: [PATCH 010/223] Add package type to unpublish command --- platformio/clients/registry.py | 8 ++++--- platformio/commands/package.py | 20 +++++++++++++---- platformio/package/pack.py | 37 ------------------------------ platformio/package/spec.py | 41 ++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 44 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 8c4d41c6..f68c4fb4 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -15,7 +15,7 @@ from platformio import __registry_api__ from platformio.clients.account import AccountClient from platformio.clients.rest import RESTClient -from platformio.package.pack import PackageType +from platformio.package.spec import PackageType class RegistryClient(RESTClient): @@ -42,13 +42,15 @@ class RegistryClient(RESTClient): ) return response - def unpublish_package(self, name, owner=None, version=None, undo=False): + def unpublish_package( # pylint: disable=redefined-builtin,too-many-arguments + self, type, name, owner=None, version=None, undo=False + ): account = AccountClient() if not owner: owner = ( account.get_account_info(offline=True).get("profile").get("username") ) - path = "/v3/package/%s/%s" % (owner, name) + path = "/v3/package/%s/%s/%s" % (owner, type, name) if version: path = path + "/version/" + version response = self.send_request( diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 93423cc7..04e78e30 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -19,7 +19,7 @@ import click from platformio.clients.registry import RegistryClient from platformio.package.pack import PackagePacker -from platformio.package.spec import PackageSpec +from platformio.package.spec import PackageSpec, PackageType def validate_datetime(ctx, param, value): # pylint: disable=unused-argument @@ -67,15 +67,27 @@ def package_publish(package, owner, released_at, private): @cli.command("unpublish", short_help="Remove a pushed package from the registry") -@click.argument("package", required=True, metavar="[<@organization>/][@]") +@click.argument( + "package", required=True, metavar="[<@organization>/][@]" +) +@click.option( + "--type", + type=click.Choice(list(PackageType.items().values())), + default="library", + help="Package type, default is set to `library`", +) @click.option( "--undo", is_flag=True, help="Undo a remove, putting a version back into the registry", ) -def package_unpublish(package, undo): +def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin spec = PackageSpec(package) response = RegistryClient().unpublish_package( - name=spec.name, owner=spec.organization, version=spec.version, undo=undo + type=type, + name=spec.name, + owner=spec.organization, + version=spec.version, + undo=undo, ) click.secho(response.get("message"), fg="green") diff --git a/platformio/package/pack.py b/platformio/package/pack.py index ecf14a4f..1e18c55a 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -19,49 +19,12 @@ import tarfile import tempfile from platformio import fs -from platformio.compat import get_object_members from platformio.package.exception import PackageException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.unpacker import FileUnpacker -class PackageType(object): - LIBRARY = "library" - PLATFORM = "platform" - TOOL = "tool" - - @classmethod - def items(cls): - return get_object_members(cls) - - @classmethod - def get_manifest_map(cls): - return { - cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), - cls.LIBRARY: ( - ManifestFileType.LIBRARY_JSON, - ManifestFileType.LIBRARY_PROPERTIES, - ManifestFileType.MODULE_JSON, - ), - cls.TOOL: (ManifestFileType.PACKAGE_JSON,), - } - - @classmethod - def from_archive(cls, path): - assert path.endswith("tar.gz") - manifest_map = cls.get_manifest_map() - with tarfile.open(path, mode="r|gz") as tf: - for t in sorted(cls.items().values()): - try: - for manifest in manifest_map[t]: - if tf.getmember(manifest): - return t - except KeyError: - pass - return None - - class PackagePacker(object): EXCLUDE_DEFAULT = [ "._*", diff --git a/platformio/package/spec.py b/platformio/package/spec.py index 0de8227a..e729dae2 100644 --- a/platformio/package/spec.py +++ b/platformio/package/spec.py @@ -12,6 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. +import tarfile + +from platformio.compat import get_object_members +from platformio.package.manifest.parser import ManifestFileType + + +class PackageType(object): + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r|gz") as tf: + for t in sorted(cls.items().values()): + try: + for manifest in manifest_map[t]: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + class PackageSpec(object): def __init__(self, raw=None, organization=None, name=None, version=None): From d38f5aca5c927db071dd253cc007a1204945d497 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 16:20:02 +0300 Subject: [PATCH 011/223] Fix metavar for package CLI --- platformio/commands/package.py | 4 ++-- platformio/telemetry.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 04e78e30..ac5c58ee 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -36,7 +36,7 @@ def cli(): @cli.command("pack", short_help="Create a tarball from a package") -@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +@click.argument("package", required=True, metavar="") def package_pack(package): p = PackagePacker(package) archive_path = p.pack() @@ -44,7 +44,7 @@ def package_pack(package): @cli.command("publish", short_help="Publish a package to the registry") -@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +@click.argument("package", required=True, metavar="") @click.option( "--owner", help="PIO Account username (can be organization username). " diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 83c53c7f..7435bdab 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -146,7 +146,15 @@ class MeasurementProtocol(TelemetryBase): return cmd_path = args[:1] - if args[0] in ("account", "device", "platform", "project", "settings",): + if args[0] in ( + "account", + "device", + "platform", + "package", + "project", + "settings", + "system", + ): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: lib_subcmds = ( From c1965b607b67679f464ecc6dc39fb3b985ce71af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 17:27:05 +0300 Subject: [PATCH 012/223] Add binary stream to package publishing request --- platformio/clients/registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f68c4fb4..ba52c83b 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -36,7 +36,8 @@ class RegistryClient(RESTClient): "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={"private": 1 if private else 0, "released_at": released_at}, headers={ - "Authorization": "Bearer %s" % account.fetch_authentication_token() + "Authorization": "Bearer %s" % account.fetch_authentication_token(), + "Content-Type": "application/octet-stream", }, data=fp, ) From 8e72c48319cd30a25edcfa1d4b902c12722ee43e Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Wed, 27 May 2020 22:30:16 +0300 Subject: [PATCH 013/223] fix datetime validation in package publish command --- platformio/commands/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index ac5c58ee..b42d34b7 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -23,6 +23,8 @@ from platformio.package.spec import PackageSpec, PackageType def validate_datetime(ctx, param, value): # pylint: disable=unused-argument + if not value: + return value try: datetime.strptime(value, "%Y-%m-%d %H:%M:%S") except ValueError as e: From 25a421402b43101722a59fd9a13c4a5c2ac20317 Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Thu, 28 May 2020 12:49:32 +0300 Subject: [PATCH 014/223] fix package type detector --- platformio/package/spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/package/spec.py b/platformio/package/spec.py index e729dae2..0535d4ba 100644 --- a/platformio/package/spec.py +++ b/platformio/package/spec.py @@ -45,12 +45,12 @@ class PackageType(object): manifest_map = cls.get_manifest_map() with tarfile.open(path, mode="r|gz") as tf: for t in sorted(cls.items().values()): - try: - for manifest in manifest_map[t]: + for manifest in manifest_map[t]: + try: if tf.getmember(manifest): return t - except KeyError: - pass + except KeyError: + pass return None From 49960b257dec1d1c4f49c17a03f39fac64b20d5f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 May 2020 16:07:02 +0300 Subject: [PATCH 015/223] Implement fs.calculate_file_hashsum --- platformio/downloader.py | 17 ++++------------- platformio/fs.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/platformio/downloader.py b/platformio/downloader.py index 21f5477b..ccbc5b36 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib import io import math import sys @@ -23,7 +22,7 @@ from time import mktime import click import requests -from platformio import app, util +from platformio import app, fs, util from platformio.exception import ( FDSHASumMismatch, FDSizeMismatch, @@ -103,17 +102,9 @@ class FileDownloader(object): raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) if not sha1: return None - - checksum = hashlib.sha1() - with io.open(self._destination, "rb", buffering=0) as fp: - while True: - chunk = fp.read(io.DEFAULT_BUFFER_SIZE) - if not chunk: - break - checksum.update(chunk) - - if sha1.lower() != checksum.hexdigest().lower(): - raise FDSHASumMismatch(checksum.hexdigest(), self._fname, sha1) + checksum = fs.calculate_file_hashsum("sha1", self._destination) + if sha1.lower() != checksum.lower(): + raise FDSHASumMismatch(checksum, self._fname, sha1) return True def _preserve_filemtime(self, lmdate): diff --git a/platformio/fs.py b/platformio/fs.py index 5122c882..7a592746 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib +import io import json import os import re @@ -72,6 +74,17 @@ def format_filesize(filesize): return "%d%sB" % ((base * filesize / unit), suffix) +def calculate_file_hashsum(algorithm, path): + h = hashlib.new(algorithm) + with io.open(path, "rb", buffering=0) as fp: + while True: + chunk = fp.read(io.DEFAULT_BUFFER_SIZE) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel From 37e795d539c045c4b13f9ec2db9d012f64ccfb0b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 May 2020 16:07:20 +0300 Subject: [PATCH 016/223] Send package checksum when publishing --- platformio/clients/registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index ba52c83b..bbb0725c 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio import __registry_api__ +from platformio import __registry_api__, fs from platformio.clients.account import AccountClient from platformio.clients.rest import RESTClient from platformio.package.spec import PackageType @@ -38,6 +38,7 @@ class RegistryClient(RESTClient): headers={ "Authorization": "Bearer %s" % account.fetch_authentication_token(), "Content-Type": "application/octet-stream", + "X-PIO-SHA256": fs.calculate_file_hashsum("sha256", archive_path), }, data=fp, ) From ae58cc74bdc5a960e36299f7a02a8342cb6acb67 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 May 2020 16:08:37 +0300 Subject: [PATCH 017/223] Rename checksum header to X-PIO-Content-SHA256 --- platformio/clients/registry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index bbb0725c..123b3239 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -38,7 +38,9 @@ class RegistryClient(RESTClient): headers={ "Authorization": "Bearer %s" % account.fetch_authentication_token(), "Content-Type": "application/octet-stream", - "X-PIO-SHA256": fs.calculate_file_hashsum("sha256", archive_path), + "X-PIO-Content-SHA256": fs.calculate_file_hashsum( + "sha256", archive_path + ), }, data=fp, ) From 26ba6e4756e2d3efea54ad1d5e6f1c14e9bda091 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 May 2020 17:06:36 +0300 Subject: [PATCH 018/223] Add new option to package publishing CLI which allows to disable email notiication --- platformio/clients/registry.py | 12 +++++++++--- platformio/commands/package.py | 9 +++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 123b3239..f90282d8 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -17,13 +17,15 @@ from platformio.clients.account import AccountClient from platformio.clients.rest import RESTClient from platformio.package.spec import PackageType +# pylint: disable=too-many-arguments + class RegistryClient(RESTClient): def __init__(self): super(RegistryClient, self).__init__(base_url=__registry_api__) def publish_package( - self, archive_path, owner=None, released_at=None, private=False + self, archive_path, owner=None, released_at=None, private=False, notify=True ): account = AccountClient() if not owner: @@ -34,7 +36,11 @@ class RegistryClient(RESTClient): response = self.send_request( "post", "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), - params={"private": 1 if private else 0, "released_at": released_at}, + params={ + "private": 1 if private else 0, + "notify": 1 if notify else 0, + "released_at": released_at, + }, headers={ "Authorization": "Bearer %s" % account.fetch_authentication_token(), "Content-Type": "application/octet-stream", @@ -46,7 +52,7 @@ class RegistryClient(RESTClient): ) return response - def unpublish_package( # pylint: disable=redefined-builtin,too-many-arguments + def unpublish_package( # pylint: disable=redefined-builtin self, type, name, owner=None, version=None, undo=False ): account = AccountClient() diff --git a/platformio/commands/package.py b/platformio/commands/package.py index b42d34b7..e5eb7db6 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -58,11 +58,16 @@ def package_pack(package): help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52", ) @click.option("--private", is_flag=True, help="Restricted access (not a public)") -def package_publish(package, owner, released_at, private): +@click.option( + "--notify/--no-notify", + default=True, + help="Notify by email when package is processed", +) +def package_publish(package, owner, released_at, private, notify): p = PackagePacker(package) archive_path = p.pack() response = RegistryClient().publish_package( - archive_path, owner, released_at, private + archive_path, owner, released_at, private, notify ) os.remove(archive_path) click.secho(response.get("message"), fg="green") From 9a1d2970cc02c3365079fe1f78e82c8fc20b7f3d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 30 May 2020 01:10:04 +0300 Subject: [PATCH 019/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b26f9eb4..dd9c31e9 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b26f9eb4834bd5de1cc28c9d5a6cd99b4332ebef +Subproject commit dd9c31e97efdd2d37b3386aa7324697ed3b3f6a1 From 9064fcbc775a61fc5fc1de43731d366d53237916 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 14:33:03 +0300 Subject: [PATCH 020/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index dd9c31e9..88a52b67 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit dd9c31e97efdd2d37b3386aa7324697ed3b3f6a1 +Subproject commit 88a52b6700255801b9bb53d2aab2f36c8072c73a From fe52f60389da9d116d2ad0402dff7078aafa5df8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 14:33:53 +0300 Subject: [PATCH 021/223] Bypass PermissionError when cleaning the cache --- platformio/maintenance.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index d2e7ea1c..5f275736 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -139,7 +139,10 @@ def after_upgrade(ctx): return else: click.secho("Please wait while upgrading PlatformIO...", fg="yellow") - app.clean_cache() + try: + app.clean_cache() + except PermissionError: + pass # Update PlatformIO's Core packages update_core_packages(silent=True) From 8c586dc360afca5201a54675a5e39a975d86d37e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 17:16:59 +0300 Subject: [PATCH 022/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 88a52b67..5071271d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 88a52b6700255801b9bb53d2aab2f36c8072c73a +Subproject commit 5071271dff8a7162489fc43fb6d98966d2593c09 From 140fff9c23bc6749ba6e1791f716107e6975244c Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 3 Jun 2020 17:41:30 +0300 Subject: [PATCH 023/223] CLI to manage organizations. Resolve #3532 (#3540) * CLI to manage organizations. Resolve #3532 * fix tests * fix test * add org owner test * fix org test * fix invalid username/orgname error text * refactor auth request in clients * fix * fix send auth request * fix regexp * remove duplicated code. minor fixes. * Remove space Co-authored-by: Ivan Kravets --- platformio/clients/account.py | 70 ++++++++---- platformio/clients/registry.py | 20 ++-- platformio/commands/account.py | 12 ++- platformio/commands/org.py | 128 ++++++++++++++++++++++ tests/commands/test_account.py | 14 ++- tests/commands/test_orgs.py | 191 +++++++++++++++++++++++++++++++++ 6 files changed, 394 insertions(+), 41 deletions(-) create mode 100644 platformio/commands/org.py create mode 100644 tests/commands/test_orgs.py diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 078e2b8a..9e0e6581 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -35,7 +35,7 @@ class AccountAlreadyAuthorized(AccountError): MESSAGE = "You are already authorized with {0} account." -class AccountClient(RESTClient): +class AccountClient(RESTClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 @@ -61,6 +61,14 @@ class AccountClient(RESTClient): del account[key] app.set_state_item("account", account) + def send_auth_request(self, *args, **kwargs): + headers = kwargs.get("headers", {}) + if "Authorization" not in headers: + token = self.fetch_authentication_token() + headers["Authorization"] = "Bearer %s" % token + kwargs["headers"] = headers + return self.send_request(*args, **kwargs) + def login(self, username, password): try: self.fetch_authentication_token() @@ -107,11 +115,9 @@ class AccountClient(RESTClient): return True def change_password(self, old_password, new_password): - token = self.fetch_authentication_token() - self.send_request( + self.send_auth_request( "post", "/v1/password", - headers={"Authorization": "Bearer %s" % token}, data={"old_password": old_password, "new_password": new_password}, ) return True @@ -141,11 +147,9 @@ class AccountClient(RESTClient): ) def auth_token(self, password, regenerate): - token = self.fetch_authentication_token() - result = self.send_request( + result = self.send_auth_request( "post", "/v1/token", - headers={"Authorization": "Bearer %s" % token}, data={"password": password, "regenerate": 1 if regenerate else 0}, ) return result.get("auth_token") @@ -154,21 +158,12 @@ class AccountClient(RESTClient): return self.send_request("post", "/v1/forgot", data={"username": username},) def get_profile(self): - token = self.fetch_authentication_token() - return self.send_request( - "get", "/v1/profile", headers={"Authorization": "Bearer %s" % token}, - ) + return self.send_auth_request("get", "/v1/profile",) def update_profile(self, profile, current_password): - token = self.fetch_authentication_token() profile["current_password"] = current_password self.delete_local_state("summary") - response = self.send_request( - "put", - "/v1/profile", - headers={"Authorization": "Bearer %s" % token}, - data=profile, - ) + response = self.send_auth_request("put", "/v1/profile", data=profile,) return response def get_account_info(self, offline=False): @@ -187,10 +182,7 @@ class AccountClient(RESTClient): "username": account.get("username"), } } - token = self.fetch_authentication_token() - result = self.send_request( - "get", "/v1/summary", headers={"Authorization": "Bearer %s" % token}, - ) + result = self.send_auth_request("get", "/v1/summary",) account["summary"] = dict( profile=result.get("profile"), packages=result.get("packages"), @@ -201,6 +193,40 @@ class AccountClient(RESTClient): app.set_state_item("account", account) return result + def create_org(self, orgname, email, display_name): + response = self.send_auth_request( + "post", + "/v1/orgs", + data={"orgname": orgname, "email": email, "displayname": display_name}, + ) + return response + + def list_orgs(self): + response = self.send_auth_request("get", "/v1/orgs",) + return response + + def update_org(self, orgname, data): + response = self.send_auth_request( + "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v} + ) + return response + + def add_org_owner(self, orgname, username): + response = self.send_auth_request( + "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + ) + return response + + def list_org_owners(self, orgname): + response = self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,) + return response + + def remove_org_owner(self, orgname, username): + response = self.send_auth_request( + "delete", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + ) + return response + def fetch_authentication_token(self): if "PLATFORMIO_AUTH_TOKEN" in os.environ: return os.environ["PLATFORMIO_AUTH_TOKEN"] diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f90282d8..5936d2e9 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -24,6 +24,14 @@ class RegistryClient(RESTClient): def __init__(self): super(RegistryClient, self).__init__(base_url=__registry_api__) + def send_auth_request(self, *args, **kwargs): + headers = kwargs.get("headers", {}) + if "Authorization" not in headers: + token = AccountClient().fetch_authentication_token() + headers["Authorization"] = "Bearer %s" % token + kwargs["headers"] = headers + return self.send_request(*args, **kwargs) + def publish_package( self, archive_path, owner=None, released_at=None, private=False, notify=True ): @@ -33,7 +41,7 @@ class RegistryClient(RESTClient): account.get_account_info(offline=True).get("profile").get("username") ) with open(archive_path, "rb") as fp: - response = self.send_request( + response = self.send_auth_request( "post", "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={ @@ -42,7 +50,6 @@ class RegistryClient(RESTClient): "released_at": released_at, }, headers={ - "Authorization": "Bearer %s" % account.fetch_authentication_token(), "Content-Type": "application/octet-stream", "X-PIO-Content-SHA256": fs.calculate_file_hashsum( "sha256", archive_path @@ -63,12 +70,7 @@ class RegistryClient(RESTClient): path = "/v3/package/%s/%s/%s" % (owner, type, name) if version: path = path + "/version/" + version - response = self.send_request( - "delete", - path, - params={"undo": 1 if undo else 0}, - headers={ - "Authorization": "Bearer %s" % account.fetch_authentication_token() - }, + response = self.send_auth_request( + "delete", path, params={"undo": 1 if undo else 0}, ) return response diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 78c5aa9e..39cca5bd 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -29,13 +29,15 @@ def cli(): pass -def validate_username(value): +def validate_username(value, field="username"): value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){3,38}$", value, flags=re.I): + if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): raise click.BadParameter( - "Invalid username format. " - "Username must contain at least 4 characters including single hyphens," - " and cannot begin or end with a hyphen" + "Invalid %s format. " + "%s may only contain alphanumeric characters " + "or single hyphens, cannot begin or end with a hyphen, " + "and must not be longer than 38 characters." + % (field.lower(), field.capitalize()) ) return value diff --git a/platformio/commands/org.py b/platformio/commands/org.py new file mode 100644 index 00000000..26584923 --- /dev/null +++ b/platformio/commands/org.py @@ -0,0 +1,128 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json + +import click +from tabulate import tabulate + +from platformio.clients.account import AccountClient +from platformio.commands.account import validate_email, validate_username + + +@click.group("org", short_help="Manage Organizations") +def cli(): + pass + + +def validate_orgname(value): + return validate_username(value, "Organization name") + + +@cli.command("create", short_help="Create a new organization") +@click.argument( + "orgname", callback=lambda _, __, value: validate_orgname(value), +) +@click.option( + "--email", callback=lambda _, __, value: validate_email(value) if value else value +) +@click.option("--display-name",) +def org_create(orgname, email, display_name): + client = AccountClient() + client.create_org(orgname, email, display_name) + return click.secho("An organization has been successfully created.", fg="green",) + + +@cli.command("list", short_help="List organizations") +@click.option("--json-output", is_flag=True) +def org_list(json_output): + client = AccountClient() + orgs = client.list_orgs() + if json_output: + return click.echo(json.dumps(orgs)) + click.echo() + click.secho("Organizations", fg="cyan") + click.echo("=" * len("Organizations")) + for org in orgs: + click.echo() + click.secho(org.get("orgname"), bold=True) + click.echo("-" * len(org.get("orgname"))) + data = [] + if org.get("displayname"): + data.append(("Display Name:", org.get("displayname"))) + if org.get("email"): + data.append(("Email:", org.get("email"))) + data.append( + ( + "Owners:", + ", ".join((owner.get("username") for owner in org.get("owners"))), + ) + ) + click.echo(tabulate(data, tablefmt="plain")) + return click.echo() + + +@cli.command("update", short_help="Update organization") +@click.argument("orgname") +@click.option("--new-orgname") +@click.option("--email") +@click.option("--display-name",) +def org_update(orgname, **kwargs): + client = AccountClient() + org = next( + (org for org in client.list_orgs() if org.get("orgname") == orgname), None + ) + if not org: + return click.ClickException("Organization '%s' not found" % orgname) + del org["owners"] + new_org = org.copy() + if not any(kwargs.values()): + for field in org: + new_org[field] = click.prompt( + field.replace("_", " ").capitalize(), default=org[field] + ) + if field == "email": + validate_email(new_org[field]) + if field == "orgname": + validate_orgname(new_org[field]) + else: + new_org.update( + {key.replace("new_", ""): value for key, value in kwargs.items() if value} + ) + client.update_org(orgname, new_org) + return click.secho("An organization has been successfully updated.", fg="green",) + + +@cli.command("add", short_help="Add a new owner to organization") +@click.argument("orgname",) +@click.argument("username",) +def org_add_owner(orgname, username): + client = AccountClient() + client.add_org_owner(orgname, username) + return click.secho( + "A new owner has been successfully added to organization.", fg="green", + ) + + +@cli.command("remove", short_help="Remove an owner from organization") +@click.argument("orgname",) +@click.argument("username",) +def org_remove_owner(orgname, username): + client = AccountClient() + client.remove_org_owner(orgname, username) + return click.secho( + "An owner has been successfully removed from organization.", fg="green", + ) diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index 5b160f0c..1be778eb 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -145,8 +145,12 @@ def test_account_password_change_with_invalid_old_password( ) assert result.exit_code > 0 assert result.exception - assert "Invalid user password" in str(result.exception) - + assert ( + "Invalid request data for new_password -> " + "'Password must contain at least 8 " + "characters including a number and a lowercase letter'" + in str(result.exception) + ) finally: clirunner.invoke(cmd_account, ["logout"]) @@ -174,9 +178,9 @@ def test_account_password_change_with_invalid_new_password_format( assert result.exit_code > 0 assert result.exception assert ( - "Invalid password format. Password must contain at" - " least 8 characters including a number and a lowercase letter" - in str(result.exception) + "Invalid request data for new_password -> " + "'Password must contain at least 8 characters" + " including a number and a lowercase letter'" in str(result.exception) ) finally: diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py new file mode 100644 index 00000000..b49f1f00 --- /dev/null +++ b/tests/commands/test_orgs.py @@ -0,0 +1,191 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import pytest + +from platformio.commands.account import cli as cmd_account +from platformio.commands.org import cli as cmd_org + +pytestmark = pytest.mark.skipif( + not ( + os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") + and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") + ), + reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", +) + + +@pytest.fixture(scope="session") +def credentials(): + return { + "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], + "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], + } + + +def test_org_add(clirunner, credentials, validate_cliresult, isolated_pio_home): + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + if len(json_result) < 3: + for i in range(3 - len(json_result)): + result = clirunner.invoke( + cmd_org, + [ + "create", + "%s-%s" % (i, credentials["login"]), + "--email", + "test@test.com", + "--display-name", + "TEST ORG %s" % i, + ], + ) + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + finally: + clirunner.invoke(cmd_account, ["logout"]) + + +def test_org_list(clirunner, credentials, validate_cliresult, isolated_pio_home): + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + check = False + for org in json_result: + assert "orgname" in org + assert "displayname" in org + assert "email" in org + assert "owners" in org + for owner in org.get("owners"): + assert "username" in owner + check = owner["username"] == credentials["login"] if not check else True + assert "firstname" in owner + assert "lastname" in owner + assert check + finally: + clirunner.invoke(cmd_account, ["logout"]) + + +def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_home): + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + org = json_result[0] + assert "orgname" in org + assert "displayname" in org + assert "email" in org + assert "owners" in org + + old_orgname = org["orgname"] + if len(old_orgname) > 10: + new_orgname = "neworg" + org["orgname"][6:] + + result = clirunner.invoke( + cmd_org, ["update", old_orgname, "--new-orgname", new_orgname], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_org, ["update", new_orgname, "--new-orgname", old_orgname], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + assert json.loads(result.output.strip()) == json_result + finally: + clirunner.invoke(cmd_account, ["logout"]) + + +def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home): + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + org = json_result[0] + assert "orgname" in org + assert "displayname" in org + assert "email" in org + assert "owners" in org + + result = clirunner.invoke(cmd_org, ["add", org["orgname"], "platformio"],) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + check = False + for item in json_result: + if item["orgname"] != org["orgname"]: + continue + for owner in item.get("owners"): + check = owner["username"] == "platformio" if not check else True + assert check + + result = clirunner.invoke(cmd_org, ["remove", org["orgname"], "platformio"],) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + check = False + for item in json_result: + if item["orgname"] != org["orgname"]: + continue + for owner in item.get("owners"): + check = owner["username"] == "platformio" if not check else True + assert not check + + finally: + clirunner.invoke(cmd_account, ["logout"]) From f7dceb782cee937d3ecb12268f259932815a95b7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 21:24:01 +0300 Subject: [PATCH 024/223] Fix PY2.7 when PermissionError is not avialable --- platformio/maintenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 5f275736..16712872 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -141,7 +141,7 @@ def after_upgrade(ctx): click.secho("Please wait while upgrading PlatformIO...", fg="yellow") try: app.clean_cache() - except PermissionError: + except: # pylint: disable=bare-except pass # Update PlatformIO's Core packages From cbcd3f7c4d085e9a23c9d191b03bb21a7d735dfd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 21:40:03 +0300 Subject: [PATCH 025/223] Fix cmd.org test --- tests/commands/test_orgs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py index b49f1f00..33f5291b 100644 --- a/tests/commands/test_orgs.py +++ b/tests/commands/test_orgs.py @@ -157,7 +157,7 @@ def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home assert "email" in org assert "owners" in org - result = clirunner.invoke(cmd_org, ["add", org["orgname"], "platformio"],) + result = clirunner.invoke(cmd_org, ["add", org["orgname"], "ivankravets"],) validate_cliresult(result) result = clirunner.invoke(cmd_org, ["list", "--json-output"],) @@ -169,10 +169,10 @@ def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home if item["orgname"] != org["orgname"]: continue for owner in item.get("owners"): - check = owner["username"] == "platformio" if not check else True + check = owner["username"] == "ivankravets" if not check else True assert check - result = clirunner.invoke(cmd_org, ["remove", org["orgname"], "platformio"],) + result = clirunner.invoke(cmd_org, ["remove", org["orgname"], "ivankravets"],) validate_cliresult(result) result = clirunner.invoke(cmd_org, ["list", "--json-output"],) @@ -184,7 +184,7 @@ def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home if item["orgname"] != org["orgname"]: continue for owner in item.get("owners"): - check = owner["username"] == "platformio" if not check else True + check = owner["username"] == "ivankravets" if not check else True assert not check finally: From 6c97cc61928af4f2433ce2f6a7d6ff1fdd9ec9c2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 22:22:13 +0300 Subject: [PATCH 026/223] Cosmetic changes to Org CLI --- platformio/commands/org.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/platformio/commands/org.py b/platformio/commands/org.py index 26584923..5cf0b13b 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -53,12 +53,9 @@ def org_list(json_output): orgs = client.list_orgs() if json_output: return click.echo(json.dumps(orgs)) - click.echo() - click.secho("Organizations", fg="cyan") - click.echo("=" * len("Organizations")) for org in orgs: click.echo() - click.secho(org.get("orgname"), bold=True) + click.secho(org.get("orgname"), fg="cyan") click.echo("-" * len(org.get("orgname"))) data = [] if org.get("displayname"): @@ -113,7 +110,7 @@ def org_add_owner(orgname, username): client = AccountClient() client.add_org_owner(orgname, username) return click.secho( - "A new owner has been successfully added to organization.", fg="green", + "A new owner has been successfully added to the organization.", fg="green", ) @@ -124,5 +121,5 @@ def org_remove_owner(orgname, username): client = AccountClient() client.remove_org_owner(orgname, username) return click.secho( - "An owner has been successfully removed from organization.", fg="green", + "An owner has been successfully removed from the organization.", fg="green", ) From 87b5fbd237651ec7e7da35d040cde8d69a9803d1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 22:34:37 +0300 Subject: [PATCH 027/223] More cosmetic changes to Org CLI --- platformio/commands/account.py | 2 +- platformio/commands/org.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 39cca5bd..67603224 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -34,7 +34,7 @@ def validate_username(value, field="username"): if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): raise click.BadParameter( "Invalid %s format. " - "%s may only contain alphanumeric characters " + "%s must only contain alphanumeric characters " "or single hyphens, cannot begin or end with a hyphen, " "and must not be longer than 38 characters." % (field.lower(), field.capitalize()) diff --git a/platformio/commands/org.py b/platformio/commands/org.py index 5cf0b13b..d5f7a2f9 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -43,7 +43,9 @@ def validate_orgname(value): def org_create(orgname, email, display_name): client = AccountClient() client.create_org(orgname, email, display_name) - return click.secho("An organization has been successfully created.", fg="green",) + return click.secho( + "The organization %s has been successfully created." % orgname, fg="green", + ) @cli.command("list", short_help="List organizations") @@ -100,7 +102,9 @@ def org_update(orgname, **kwargs): {key.replace("new_", ""): value for key, value in kwargs.items() if value} ) client.update_org(orgname, new_org) - return click.secho("An organization has been successfully updated.", fg="green",) + return click.secho( + "The organization %s has been successfully updated." % orgname, fg="green", + ) @cli.command("add", short_help="Add a new owner to organization") @@ -110,7 +114,9 @@ def org_add_owner(orgname, username): client = AccountClient() client.add_org_owner(orgname, username) return click.secho( - "A new owner has been successfully added to the organization.", fg="green", + "The new owner %s has been successfully added to the %s organization." + % (username, orgname), + fg="green", ) @@ -121,5 +127,7 @@ def org_remove_owner(orgname, username): client = AccountClient() client.remove_org_owner(orgname, username) return click.secho( - "An owner has been successfully removed from the organization.", fg="green", + "The %s owner has been successfully removed from the %s organization." + % (username, orgname), + fg="green", ) From d7f4eb59558da803a228f23b1d620d70b96b3b61 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 22:40:37 +0300 Subject: [PATCH 028/223] Minor grammar fix --- platformio/commands/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 67603224..c254dbee 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -34,7 +34,7 @@ def validate_username(value, field="username"): if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): raise click.BadParameter( "Invalid %s format. " - "%s must only contain alphanumeric characters " + "%s must contain only alphanumeric characters " "or single hyphens, cannot begin or end with a hyphen, " "and must not be longer than 38 characters." % (field.lower(), field.capitalize()) From 3c1b08daab2eba6f8a546e34f299a535fc29a510 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jun 2020 13:57:56 +0300 Subject: [PATCH 029/223] Ignore empty PLATFORMIO_AUTH_TOKEN --- platformio/clients/account.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 9e0e6581..084110da 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -228,8 +228,8 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods return response def fetch_authentication_token(self): - if "PLATFORMIO_AUTH_TOKEN" in os.environ: - return os.environ["PLATFORMIO_AUTH_TOKEN"] + if os.environ.get("PLATFORMIO_AUTH_TOKEN"): + return os.environ.get("PLATFORMIO_AUTH_TOKEN") auth = app.get_state_item("account", {}).get("auth", {}) if auth.get("access_token") and auth.get("access_token_expire"): if auth.get("access_token_expire") > time.time(): From 0c4c113b0a1d3e5e09c0c1079f685272f8195478 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jun 2020 14:09:42 +0300 Subject: [PATCH 030/223] Fix account shpw command when PLATFORMIO_AUTH_TOKEN is used --- platformio/clients/account.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 084110da..545595c1 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -167,15 +167,13 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods return response def get_account_info(self, offline=False): - account = app.get_state_item("account") - if not account: - raise AccountNotAuthorized() + account = app.get_state_item("account") or {} if ( account.get("summary") and account["summary"].get("expire_at", 0) > time.time() ): return account["summary"] - if offline: + if offline and account.get("email"): return { "profile": { "email": account.get("email"), From 42df3c9c3fca2106718ac56f084c6fce4f94e959 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jun 2020 15:27:46 +0300 Subject: [PATCH 031/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 5071271d..b993a3e8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5071271dff8a7162489fc43fb6d98966d2593c09 +Subproject commit b993a3e8ccbb0fe0c5103f667e4b26d7b1c87502 From 94cb8082856331ae6b9ec0b5b45009d5b4284c4c Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Thu, 4 Jun 2020 19:31:30 +0300 Subject: [PATCH 032/223] CLI to manage teams. Resolve #3533 (#3547) * CLI to manage teams.Minor fixes. Resolve #3533 * fix teams tests * disable org and team tests * minor fixes. fix error texts * fix split compatibility --- platformio/clients/account.py | 70 +++++++++--- platformio/commands/org.py | 12 +- platformio/commands/team.py | 201 ++++++++++++++++++++++++++++++++++ tests/commands/test_orgs.py | 106 ++++++------------ tests/commands/test_teams.py | 158 ++++++++++++++++++++++++++ 5 files changed, 453 insertions(+), 94 deletions(-) create mode 100644 platformio/commands/team.py create mode 100644 tests/commands/test_teams.py diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 545595c1..9534777d 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -115,12 +115,11 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods return True def change_password(self, old_password, new_password): - self.send_auth_request( + return self.send_auth_request( "post", "/v1/password", data={"old_password": old_password, "new_password": new_password}, ) - return True def registration( self, username, email, password, firstname, lastname @@ -147,12 +146,11 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods ) def auth_token(self, password, regenerate): - result = self.send_auth_request( + return self.send_auth_request( "post", "/v1/token", data={"password": password, "regenerate": 1 if regenerate else 0}, - ) - return result.get("auth_token") + ).get("auth_token") def forgot_password(self, username): return self.send_request("post", "/v1/forgot", data={"username": username},) @@ -192,38 +190,76 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods return result def create_org(self, orgname, email, display_name): - response = self.send_auth_request( + return self.send_auth_request( "post", "/v1/orgs", data={"orgname": orgname, "email": email, "displayname": display_name}, ) - return response + + def get_org(self, orgname): + return self.send_auth_request("get", "/v1/orgs/%s" % orgname) def list_orgs(self): - response = self.send_auth_request("get", "/v1/orgs",) - return response + return self.send_auth_request("get", "/v1/orgs",) def update_org(self, orgname, data): - response = self.send_auth_request( + return self.send_auth_request( "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v} ) - return response def add_org_owner(self, orgname, username): - response = self.send_auth_request( + return self.send_auth_request( "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, ) - return response def list_org_owners(self, orgname): - response = self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,) - return response + return self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,) def remove_org_owner(self, orgname, username): - response = self.send_auth_request( + return self.send_auth_request( "delete", "/v1/orgs/%s/owners" % orgname, data={"username": username}, ) - return response + + def create_team(self, orgname, teamname, description): + return self.send_auth_request( + "post", + "/v1/orgs/%s/teams" % orgname, + data={"name": teamname, "description": description}, + ) + + def destroy_team(self, orgname, teamname): + return self.send_auth_request( + "delete", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + ) + + def get_team(self, orgname, teamname): + return self.send_auth_request( + "get", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + ) + + def list_teams(self, orgname): + return self.send_auth_request("get", "/v1/orgs/%s/teams" % orgname,) + + def update_team(self, orgname, teamname, data): + return self.send_auth_request( + "put", + "/v1/orgs/%s/teams/%s" % (orgname, teamname), + data={k: v for k, v in data.items() if v}, + ) + + def add_team_member(self, orgname, teamname, username): + return self.send_auth_request( + "post", + "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), + data={"username": username}, + ) + + def remove_team_member(self, orgname, teamname, username): + return self.send_auth_request( + "delete", + "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), + data={"username": username}, + ) def fetch_authentication_token(self): if os.environ.get("PLATFORMIO_AUTH_TOKEN"): diff --git a/platformio/commands/org.py b/platformio/commands/org.py index d5f7a2f9..7d62120f 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -55,6 +55,8 @@ def org_list(json_output): orgs = client.list_orgs() if json_output: return click.echo(json.dumps(orgs)) + if not orgs: + return click.echo("You do not have any organizations") for org in orgs: click.echo() click.secho(org.get("orgname"), fg="cyan") @@ -76,16 +78,14 @@ def org_list(json_output): @cli.command("update", short_help="Update organization") @click.argument("orgname") -@click.option("--new-orgname") +@click.option( + "--new-orgname", callback=lambda _, __, value: validate_orgname(value), +) @click.option("--email") @click.option("--display-name",) def org_update(orgname, **kwargs): client = AccountClient() - org = next( - (org for org in client.list_orgs() if org.get("orgname") == orgname), None - ) - if not org: - return click.ClickException("Organization '%s' not found" % orgname) + org = client.get_org(orgname) del org["owners"] new_org = org.copy() if not any(kwargs.values()): diff --git a/platformio/commands/team.py b/platformio/commands/team.py new file mode 100644 index 00000000..5461cabd --- /dev/null +++ b/platformio/commands/team.py @@ -0,0 +1,201 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json +import re + +import click +from tabulate import tabulate + +from platformio.clients.account import AccountClient + + +def validate_orgname_teamname(value, teamname_validate=False): + if ":" not in value: + raise click.BadParameter( + "Please specify organization and team name in the next" + " format - orgname:teamname. For example, mycompany:DreamTeam" + ) + teamname = str(value.strip().split(":", 1)[1]) + if teamname_validate: + validate_teamname(teamname) + return value + + +def validate_teamname(value): + if not value: + return value + value = str(value).strip() + if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I): + raise click.BadParameter( + "Invalid team name format. " + "Team name must only contain alphanumeric characters, " + "single hyphens, underscores, spaces. It can not " + "begin or end with a hyphen or a underscore and must" + " not be longer than 20 characters." + ) + return value + + +@click.group("team", short_help="Manage Teams") +def cli(): + pass + + +@cli.command("create", short_help="Create a new team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname( + value, teamname_validate=True + ), +) +@click.option("--description",) +def team_create(orgname_teamname, description): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.create_team(orgname, teamname, description) + return click.secho( + "The team %s has been successfully created." % teamname, fg="green", + ) + + +@cli.command("list", short_help="List teams") +@click.argument("orgname", required=False) +@click.option("--json-output", is_flag=True) +def team_list(orgname, json_output): + client = AccountClient() + data = {} + if not orgname: + for item in client.list_orgs(): + teams = client.list_teams(item.get("orgname")) + data[item.get("orgname")] = teams + else: + teams = client.list_teams(orgname) + data[orgname] = teams + if json_output: + return click.echo(json.dumps(data[orgname] if orgname else data)) + if not any(data.values()): + return click.secho("You do not have any teams.", fg="yellow") + for org_name in data: + for team in data[org_name]: + click.echo() + click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan") + click.echo("-" * len("%s:%s" % (org_name, team.get("name")))) + table_data = [] + if team.get("description"): + table_data.append(("Description:", team.get("description"))) + table_data.append( + ( + "Members:", + ", ".join( + (member.get("username") for member in team.get("members")) + ) + if team.get("members") + else "-", + ) + ) + click.echo(tabulate(table_data, tablefmt="plain")) + return click.echo() + + +@cli.command("update", short_help="Update team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.option( + "--name", callback=lambda _, __, value: validate_teamname(value), +) +@click.option("--description",) +def team_update(orgname_teamname, **kwargs): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + team = client.get_team(orgname, teamname) + del team["id"] + del team["members"] + new_team = team.copy() + if not any(kwargs.values()): + for field in team: + new_team[field] = click.prompt( + field.replace("_", " ").capitalize(), default=team[field] + ) + if field == "name": + validate_teamname(new_team[field]) + else: + new_team.update({key: value for key, value in kwargs.items() if value}) + client.update_team(orgname, teamname, new_team) + return click.secho( + "The team %s has been successfully updated." % teamname, fg="green", + ) + + +@cli.command("destroy", short_help="Destroy a team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +def team_destroy(orgname_teamname): + orgname, teamname = orgname_teamname.split(":", 1) + click.confirm( + click.style( + "Are you sure you want to destroy the %s team?" % teamname, fg="yellow" + ), + abort=True, + ) + client = AccountClient() + client.destroy_team(orgname, teamname) + return click.secho( + "The team %s has been successfully destroyed." % teamname, fg="green", + ) + + +@cli.command("add", short_help="Add a new member to team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.argument("username",) +def team_add_member(orgname_teamname, username): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.add_team_member(orgname, teamname, username) + return click.secho( + "The new member %s has been successfully added to the %s team." + % (username, teamname), + fg="green", + ) + + +@cli.command("remove", short_help="Remove a member from team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.argument("username",) +def org_remove_owner(orgname_teamname, username): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.remove_team_member(orgname, teamname, username) + return click.secho( + "The %s member has been successfully removed from the %s team." + % (username, teamname), + fg="green", + ) diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py index 33f5291b..4650caaf 100644 --- a/tests/commands/test_orgs.py +++ b/tests/commands/test_orgs.py @@ -37,7 +37,7 @@ def credentials(): } -def test_org_add(clirunner, credentials, validate_cliresult, isolated_pio_home): +def test_orgs(clirunner, credentials, validate_cliresult, isolated_pio_home): try: result = clirunner.invoke( cmd_account, @@ -66,26 +66,11 @@ def test_org_add(clirunner, credentials, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_org, ["list", "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_org_list(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - assert "Successfully logged in!" in result.output - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 + assert len(json_result) >= 3 check = False for org in json_result: assert "orgname" in org + orgname = org["orgname"] assert "displayname" in org assert "email" in org assert "owners" in org @@ -95,10 +80,41 @@ def test_org_list(clirunner, credentials, validate_cliresult, isolated_pio_home) assert "firstname" in owner assert "lastname" in owner assert check + + result = clirunner.invoke(cmd_org, ["add", orgname, "ivankravets"],) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 3 + check = False + for item in json_result: + if item["orgname"] != orgname: + continue + for owner in item.get("owners"): + check = owner["username"] == "ivankravets" if not check else True + assert check + + result = clirunner.invoke(cmd_org, ["remove", orgname, "ivankravets"],) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 3 + check = False + for item in json_result: + if item["orgname"] != orgname: + continue + for owner in item.get("owners"): + check = owner["username"] == "ivankravets" if not check else True + assert not check finally: clirunner.invoke(cmd_account, ["logout"]) +@pytest.mark.skip def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_home): try: result = clirunner.invoke( @@ -111,7 +127,7 @@ def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_hom result = clirunner.invoke(cmd_org, ["list", "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 + assert len(json_result) >= 3 org = json_result[0] assert "orgname" in org assert "displayname" in org @@ -137,55 +153,3 @@ def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_hom assert json.loads(result.output.strip()) == json_result finally: clirunner.invoke(cmd_account, ["logout"]) - - -def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - assert "Successfully logged in!" in result.output - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 - org = json_result[0] - assert "orgname" in org - assert "displayname" in org - assert "email" in org - assert "owners" in org - - result = clirunner.invoke(cmd_org, ["add", org["orgname"], "ivankravets"],) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 - check = False - for item in json_result: - if item["orgname"] != org["orgname"]: - continue - for owner in item.get("owners"): - check = owner["username"] == "ivankravets" if not check else True - assert check - - result = clirunner.invoke(cmd_org, ["remove", org["orgname"], "ivankravets"],) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 - check = False - for item in json_result: - if item["orgname"] != org["orgname"]: - continue - for owner in item.get("owners"): - check = owner["username"] == "ivankravets" if not check else True - assert not check - - finally: - clirunner.invoke(cmd_account, ["logout"]) diff --git a/tests/commands/test_teams.py b/tests/commands/test_teams.py new file mode 100644 index 00000000..13e30ce0 --- /dev/null +++ b/tests/commands/test_teams.py @@ -0,0 +1,158 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import time + +import pytest + +from platformio.commands.account import cli as cmd_account +from platformio.commands.org import cli as cmd_org +from platformio.commands.team import cli as cmd_team + +pytestmark = pytest.mark.skipif( + not ( + os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") + and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") + ), + reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", +) + + +@pytest.fixture(scope="session") +def credentials(): + return { + "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], + "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], + } + + +def test_teams(clirunner, credentials, validate_cliresult, isolated_pio_home): + orgname = "" + teamname = "test-" + str(int(time.time() * 1000)) + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + if len(json_result) < 3: + for i in range(3 - len(json_result)): + result = clirunner.invoke( + cmd_org, + [ + "create", + "%s-%s" % (i, credentials["login"]), + "--email", + "test@test.com", + "--display-name", + "TEST ORG %s" % i, + ], + ) + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 3 + orgname = json_result[0].get("orgname") + + result = clirunner.invoke( + cmd_team, + [ + "create", + "%s:%s" % (orgname, teamname), + "--description", + "team for CI test", + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 1 + check = False + for team in json_result: + assert team["id"] + assert team["name"] + if team["name"] == teamname: + check = True + assert "description" in team + assert "members" in team + assert check + + result = clirunner.invoke( + cmd_team, ["add", "%s:%s" % (orgname, teamname), credentials["login"]], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + check = False + for team in json_result: + assert team["id"] + assert team["name"] + assert "description" in team + assert "members" in team + if ( + len(team["members"]) > 0 + and team["members"][0]["username"] == credentials["login"] + ): + check = True + assert check + + result = clirunner.invoke( + cmd_team, ["remove", "%s:%s" % (orgname, teamname), credentials["login"]], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (orgname, teamname), + "--description", + "Updated Description", + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 1 + check = False + for team in json_result: + assert team["id"] + assert team["name"] + assert "description" in team + if team.get("description") == "Updated Description": + check = True + assert "members" in team + assert check + finally: + clirunner.invoke( + cmd_team, ["destroy", "%s:%s" % (orgname, teamname),], + ) + clirunner.invoke(cmd_account, ["logout"]) From 6fa7cb4af530125b34be5d794ee4516db2ed70ed Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jun 2020 22:59:05 +0300 Subject: [PATCH 033/223] Add new dev-platform "ASR Microelectronics ASR605x" --- README.rst | 1 + docs | 2 +- examples | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 44c251e0..fe7345d5 100644 --- a/README.rst +++ b/README.rst @@ -81,6 +81,7 @@ Development Platforms --------------------- * `Aceinna IMU `_ +* `ASR Microelectronics ASR605x `_ * `Atmel AVR `_ * `Atmel SAM `_ * `Espressif 32 `_ diff --git a/docs b/docs index b993a3e8..b6bc6eb1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b993a3e8ccbb0fe0c5103f667e4b26d7b1c87502 +Subproject commit b6bc6eb15fdf9cf0d055eeae9d114ddd9e9d6e0d diff --git a/examples b/examples index 7793b677..c442de34 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 7793b677f72ce3c3e9ed92b7915859ca2bfa313f +Subproject commit c442de34a57b54451170dbe39f3411a06a05b3f2 From ced244d30a10885c98fb11804ec365f787238812 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Fri, 5 Jun 2020 11:30:15 +0300 Subject: [PATCH 034/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b6bc6eb1..1bf31621 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b6bc6eb15fdf9cf0d055eeae9d114ddd9e9d6e0d +Subproject commit 1bf316215c614f6dd4a7502e990137b005043212 From 27fd3b0b14995dc87ab345c954d150d14ec0177c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Jun 2020 14:17:19 +0300 Subject: [PATCH 035/223] Improve detecting if PlatformIO Core is run in container --- platformio/app.py | 11 +++++++---- platformio/maintenance.py | 5 ++--- platformio/proc.py | 28 ++++++++++++++-------------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 6c7c7b1a..f4ae15b9 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,9 +25,8 @@ from time import time import requests -from platformio import __version__, exception, fs, lockfile +from platformio import __version__, exception, fs, lockfile, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data -from platformio.proc import is_ci from platformio.project.helpers import ( get_default_projects_dir, get_project_cache_dir, @@ -383,7 +382,7 @@ def is_disabled_progressbar(): return any( [ get_session_var("force_option"), - is_ci(), + proc.is_ci(), getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", ] ) @@ -420,7 +419,11 @@ def get_cid(): def get_user_agent(): - data = ["PlatformIO/%s" % __version__, "CI/%d" % int(is_ci())] + data = [ + "PlatformIO/%s" % __version__, + "CI/%d" % int(proc.is_ci()), + "Container/%d" % int(proc.is_container()), + ] if get_session_var("caller_id"): data.append("Caller/%s" % get_session_var("caller_id")) if os.getenv("PLATFORMIO_IDE"): diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 16712872..0c8ee2df 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -66,10 +66,9 @@ def on_platformio_exception(e): def set_caller(caller=None): + caller = caller or getenv("PLATFORMIO_CALLER") if not caller: - if getenv("PLATFORMIO_CALLER"): - caller = getenv("PLATFORMIO_CALLER") - elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): + if getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): caller = "vscode" elif is_container(): if getenv("C9_UID"): diff --git a/platformio/proc.py b/platformio/proc.py index 80e50201..04f15a57 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -15,7 +15,6 @@ import os import subprocess import sys -from os.path import isdir, isfile, join, normpath from threading import Thread from platformio import exception @@ -143,18 +142,16 @@ def is_ci(): def is_container(): - if not isfile("/proc/1/cgroup"): + if os.path.exists("/.dockerenv"): + return True + if not os.path.isfile("/proc/1/cgroup"): return False with open("/proc/1/cgroup") as fp: - for line in fp: - line = line.strip() - if ":" in line and not line.endswith(":/"): - return True - return False + return ":/docker/" in fp.read() def get_pythonexe_path(): - return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) + return os.environ.get("PYTHONEXEPATH", os.path.normpath(sys.executable)) def copy_pythonpath_to_osenv(): @@ -164,7 +161,10 @@ def copy_pythonpath_to_osenv(): for p in os.sys.path: conditions = [p not in _PYTHONPATH] if not WINDOWS: - conditions.append(isdir(join(p, "click")) or isdir(join(p, "platformio"))) + conditions.append( + os.path.isdir(os.path.join(p, "click")) + or os.path.isdir(os.path.join(p, "platformio")) + ) if all(conditions): _PYTHONPATH.append(p) os.environ["PYTHONPATH"] = os.pathsep.join(_PYTHONPATH) @@ -178,16 +178,16 @@ def where_is_program(program, envpath=None): # try OS's built-in commands try: result = exec_command(["where" if WINDOWS else "which", program], env=env) - if result["returncode"] == 0 and isfile(result["out"].strip()): + if result["returncode"] == 0 and os.path.isfile(result["out"].strip()): return result["out"].strip() except OSError: pass # look up in $PATH for bin_dir in env.get("PATH", "").split(os.pathsep): - if isfile(join(bin_dir, program)): - return join(bin_dir, program) - if isfile(join(bin_dir, "%s.exe" % program)): - return join(bin_dir, "%s.exe" % program) + if os.path.isfile(os.path.join(bin_dir, program)): + return os.path.join(bin_dir, program) + if os.path.isfile(os.path.join(bin_dir, "%s.exe" % program)): + return os.path.join(bin_dir, "%s.exe" % program) return program From f5e6820903ba960184fe4ecd9f5223f140cc0fa0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Jun 2020 14:18:24 +0300 Subject: [PATCH 036/223] Bump version to 4.3.5a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 60621751..b4db9ecf 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, "5a1") +VERSION = (4, 3, "5a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e0e97a36297852128016b2ba8360d77f64b276db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Jun 2020 18:29:11 +0300 Subject: [PATCH 037/223] Cache the latest news in PIO Home for 180 days --- platformio/commands/home/rpc/handlers/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index d1851d13..a216344e 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -24,7 +24,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): def load_latest_tweets(self, data_url): cache_key = app.ContentCache.key_from_args(data_url, "tweets") - cache_valid = "7d" + cache_valid = "180d" with app.ContentCache() as cc: cache_data = cc.get(cache_key) if cache_data: From 7457ef043b91e3eaf3c6001da66b95120c265a98 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Jun 2020 12:00:19 +0300 Subject: [PATCH 038/223] Docs: Sync ASR Micro dev-platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1bf31621..af312fb7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1bf316215c614f6dd4a7502e990137b005043212 +Subproject commit af312fb70f3bf528d6f59169dc92f088764eb2a4 From 78546e9246d515d32b66dc12957bdc1368332138 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Jun 2020 19:26:48 +0300 Subject: [PATCH 039/223] Docs: Add "TensorFlow, Meet The ESP32" to articles list --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index af312fb7..cfba4f45 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit af312fb70f3bf528d6f59169dc92f088764eb2a4 +Subproject commit cfba4f456843f1069d9ebd083e359186054c8659 From a5547491edccaca360f6f1fc4b0e095383be0726 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Tue, 9 Jun 2020 15:50:37 +0300 Subject: [PATCH 040/223] Add account and org destroy commands. Fix tests (#3552) * Add account and org destroy commands. Fix tests * fix tests * fix * fix texts --- .github/workflows/core.yml | 5 +- platformio/clients/account.py | 10 +- platformio/commands/account.py | 17 + platformio/commands/org.py | 22 +- tests/commands/test_account.py | 610 ++++++++------------------------- tests/commands/test_orgs.py | 238 ++++++------- tests/commands/test_teams.py | 189 +++++----- tests/conftest.py | 43 +++ 8 files changed, 454 insertions(+), 680 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c84a97d7..c2e9547c 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -28,8 +28,9 @@ jobs: tox -e lint - name: Integration Tests env: - PLATFORMIO_TEST_ACCOUNT_LOGIN: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_LOGIN }} - PLATFORMIO_TEST_ACCOUNT_PASSWORD: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_PASSWORD }} + TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }} + TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }} + TEST_EMAIL_POP3_SERVER: ${{ secrets.TEST_EMAIL_POP3_SERVER }} run: | tox -e testcore diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 9534777d..31e34f78 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -189,11 +189,14 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods app.set_state_item("account", account) return result - def create_org(self, orgname, email, display_name): + def destroy_account(self): + return self.send_auth_request("delete", "/v1/account") + + def create_org(self, orgname, email, displayname): return self.send_auth_request( "post", "/v1/orgs", - data={"orgname": orgname, "email": email, "displayname": display_name}, + data={"orgname": orgname, "email": email, "displayname": displayname}, ) def get_org(self, orgname): @@ -207,6 +210,9 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v} ) + def destroy_org(self, orgname): + return self.send_auth_request("delete", "/v1/orgs/%s" % orgname,) + def add_org_owner(self, orgname, username): return self.send_auth_request( "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, diff --git a/platformio/commands/account.py b/platformio/commands/account.py index c254dbee..3a1492ec 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -178,6 +178,23 @@ def account_update(current_password, **kwargs): return click.secho("Please re-login.", fg="yellow") +@cli.command("destroy", short_help="Destroy account") +def account_destroy(): + client = AccountClient() + click.confirm( + "Are you sure you want to delete the %s user account?\n" + "Warning! All linked data will be permanently removed and can not be restored." + % client.get_account_info().get("profile").get("username"), + abort=True, + ) + client.destroy_account() + try: + client.logout() + except AccountNotAuthorized: + pass + return click.secho("User account has been destroyed.", fg="green",) + + @cli.command("show", short_help="PIO Account information") @click.option("--offline", is_flag=True) @click.option("--json-output", is_flag=True) diff --git a/platformio/commands/org.py b/platformio/commands/org.py index 7d62120f..a7e0f1e9 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -39,10 +39,10 @@ def validate_orgname(value): @click.option( "--email", callback=lambda _, __, value: validate_email(value) if value else value ) -@click.option("--display-name",) -def org_create(orgname, email, display_name): +@click.option("--displayname",) +def org_create(orgname, email, displayname): client = AccountClient() - client.create_org(orgname, email, display_name) + client.create_org(orgname, email, displayname) return click.secho( "The organization %s has been successfully created." % orgname, fg="green", ) @@ -82,7 +82,7 @@ def org_list(json_output): "--new-orgname", callback=lambda _, __, value: validate_orgname(value), ) @click.option("--email") -@click.option("--display-name",) +@click.option("--displayname",) def org_update(orgname, **kwargs): client = AccountClient() org = client.get_org(orgname) @@ -107,6 +107,20 @@ def org_update(orgname, **kwargs): ) +@cli.command("destroy", short_help="Destroy organization") +@click.argument("orgname") +def account_destroy(orgname): + client = AccountClient() + click.confirm( + "Are you sure you want to delete the %s organization account?\n" + "Warning! All linked data will be permanently removed and can not be restored." + % orgname, + abort=True, + ) + client.destroy_org(orgname) + return click.secho("Organization %s has been destroyed." % orgname, fg="green",) + + @cli.command("add", short_help="Add a new owner to organization") @click.argument("orgname",) @click.argument("username",) diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index 1be778eb..221b724a 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -17,34 +17,29 @@ import os import time import pytest +import requests from platformio.commands.account import cli as cmd_account - -pytestmark = pytest.mark.skipif( - not ( - os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") - and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") - ), - reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", -) +from platformio.commands.package import cli as cmd_package +from platformio.downloader import FileDownloader +from platformio.unpacker import FileUnpacker -@pytest.fixture(scope="session") -def credentials(): - return { - "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], - "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], - } - - -def test_account_register_with_already_exists_username( - clirunner, credentials, isolated_pio_home +@pytest.mark.skipif( + not os.environ.get("TEST_EMAIL_LOGIN"), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) # pylint:disable=too-many-arguments +def test_account( + clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory ): - username = credentials["login"] - email = "test@test.com" - if "@" in credentials["login"]: - username = "Testusername" - email = credentials["login"] + username = "test-piocore-%s" % str(int(time.time() * 1000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" + + # pio account register result = clirunner.invoke( cmd_account, [ @@ -54,345 +49,33 @@ def test_account_register_with_already_exists_username( "-e", email, "-p", - credentials["password"], + password, "--firstname", - "First", + firstname, "--lastname", - "Last", + lastname, ], ) - assert result.exit_code > 0 - assert result.exception - assert "User with same username already exists" in str( - result.exception - ) or "User with same email already exists" in str(result.exception) + validate_cliresult(result) + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' 0 - assert result.exception - assert "Invalid user credentials" in str(result.exception) - - -def test_account_login(clirunner, credentials, validate_cliresult, isolated_pio_home): + # pio account login + result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + validate_cliresult(result) try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - assert "Successfully logged in!" in result.output - - with open(str(isolated_pio_home.join("appstate.json"))) as fp: - appstate = json.load(fp) - assert appstate.get("account") - assert appstate.get("account").get("email") - assert appstate.get("account").get("username") - assert appstate.get("account").get("auth") - assert appstate.get("account").get("auth").get("access_token") - assert appstate.get("account").get("auth").get("access_token_expire") - assert appstate.get("account").get("auth").get("refresh_token") - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are already authorized with" in str(result.exception) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_logout(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["logout"]) - validate_cliresult(result) - assert "Successfully logged out" in result.output - - result = clirunner.invoke(cmd_account, ["logout"]) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_password_change_with_invalid_old_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - ["password", "--old-password", "test", "--new-password", "test"], - ) - assert result.exit_code > 0 - assert result.exception - assert ( - "Invalid request data for new_password -> " - "'Password must contain at least 8 " - "characters including a number and a lowercase letter'" - in str(result.exception) - ) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_password_change_with_invalid_new_password_format( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "test", - ], - ) - assert result.exit_code > 0 - assert result.exception - assert ( - "Invalid request data for new_password -> " - "'Password must contain at least 8 characters" - " including a number and a lowercase letter'" in str(result.exception) - ) - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_password_change( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "Testpassword123", - ], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "Testpassword123", - ], - ) - validate_cliresult(result) - assert "Password successfully changed!" in result.output - - result = clirunner.invoke(cmd_account, ["logout"]) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, ["login", "-u", credentials["login"], "-p", "Testpassword123"], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - "Testpassword123", - "--new-password", - credentials["password"], - ], - ) - validate_cliresult(result) - assert "Password successfully changed!" in result.output - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_token_with_invalid_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["token", "--password", "test",],) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user password" in str(result.exception) - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_token(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - validate_cliresult(result) - assert "Personal Authentication Token:" in result.output - token = result.output.strip().split(": ")[-1] - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - token = json_result.get("result") - - clirunner.invoke(cmd_account, ["logout"]) - - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - os.environ["PLATFORMIO_AUTH_TOKEN"] = token - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - - os.environ.pop("PLATFORMIO_AUTH_TOKEN") - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_token_with_refreshing( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") - token = json_result.get("result") - - result = clirunner.invoke( - cmd_account, - [ - "token", - "--password", - credentials["password"], - "--json-output", - "--regenerate", - ], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") - assert token != json_result.get("result") - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke(cmd_account, ["show"],) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - + # pio account summary result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) validate_cliresult(result) json_result = json.loads(result.output.strip()) @@ -405,9 +88,8 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi result = clirunner.invoke(cmd_account, ["show"]) validate_cliresult(result) - assert credentials["login"] in result.output - assert "Community" in result.output - assert "100 Concurrent Remote Agents" in result.output + assert username in result.output + # assert "100 Concurrent Remote Agents" in result.output result = clirunner.invoke(cmd_account, ["show", "--json-output"]) validate_cliresult(result) @@ -416,9 +98,9 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi assert json_result.get("profile") assert json_result.get("profile").get("username") assert json_result.get("profile").get("email") - assert credentials["login"] == json_result.get("profile").get( + assert username == json_result.get("profile").get( "username" - ) or credentials["login"] == json_result.get("profile").get("email") + ) or username == json_result.get("profile").get("email") assert json_result.get("profile").get("firstname") assert json_result.get("profile").get("lastname") assert json_result.get("packages") @@ -433,147 +115,121 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi assert json_result.get("profile") assert json_result.get("profile").get("username") assert json_result.get("profile").get("email") - assert credentials["login"] == json_result.get("profile").get( + assert username == json_result.get("profile").get( "username" - ) or credentials["login"] == json_result.get("profile").get("email") + ) or username == json_result.get("profile").get("email") assert json_result.get("profile").get("firstname") assert json_result.get("profile").get("lastname") assert json_result.get("packages") assert json_result.get("packages")[0].get("name") assert json_result.get("packages")[0].get("path") assert json_result.get("subscriptions") is not None - finally: + + # pio account token + result = clirunner.invoke(cmd_account, ["token", "--password", password,],) + validate_cliresult(result) + assert "Personal Authentication Token:" in result.output + token = result.output.strip().split(": ")[-1] + + result = clirunner.invoke( + cmd_account, ["token", "--password", password, "--json-output"], + ) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result + assert json_result.get("status") == "success" + assert json_result.get("result") == token + token = json_result.get("result") + clirunner.invoke(cmd_account, ["logout"]) - -@pytest.mark.skip_ci -def test_account_profile_update_with_invalid_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) + result = clirunner.invoke(cmd_account, ["token", "--password", password,],) assert result.exit_code > 0 assert result.exception assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) + os.environ["PLATFORMIO_AUTH_TOKEN"] = token + result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], + cmd_account, ["token", "--password", password, "--json-output"], + ) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result + assert json_result.get("status") == "success" + assert json_result.get("result") == token + + os.environ.pop("PLATFORMIO_AUTH_TOKEN") + + result = clirunner.invoke( + cmd_account, ["login", "-u", username, "-p", password], ) validate_cliresult(result) - firstname = "First " + str(int(time.time() * 1000)) - + # pio account password + new_password = "Testpassword123" result = clirunner.invoke( cmd_account, - ["update", "--current-password", "test", "--firstname", firstname], + ["password", "--old-password", password, "--new-password", new_password,], ) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user password" in str(result.exception) - finally: + validate_cliresult(result) + assert "Password successfully changed!" in result.output + clirunner.invoke(cmd_account, ["logout"]) - -@pytest.mark.skip_ci -def test_account_profile_update_only_firstname_and_lastname( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], + cmd_account, ["login", "-u", username, "-p", new_password], ) validate_cliresult(result) + result = clirunner.invoke( + cmd_account, + ["password", "--old-password", new_password, "--new-password", password,], + ) + validate_cliresult(result) + + # pio account update firstname = "First " + str(int(time.time() * 1000)) lastname = "Last" + str(int(time.time() * 1000)) - result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - credentials["password"], - "--firstname", - firstname, - "--lastname", - lastname, - ], - ) - validate_cliresult(result) - assert "Profile successfully updated!" in result.output - - result = clirunner.invoke(cmd_account, ["show", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result.get("profile").get("firstname") == firstname - assert json_result.get("profile").get("lastname") == lastname - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_profile_update( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["show", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - - firstname = "First " + str(int(time.time() * 1000)) - lastname = "Last" + str(int(time.time() * 1000)) - - old_username = json_result.get("profile").get("username") new_username = "username" + str(int(time.time() * 1000))[-5:] - + new_email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) result = clirunner.invoke( cmd_account, [ "update", "--current-password", - credentials["password"], + password, "--firstname", firstname, "--lastname", lastname, "--username", new_username, + "--email", + new_email, ], ) validate_cliresult(result) assert "Profile successfully updated!" in result.output - assert "Please re-login." in result.output + assert ( + "Please check your mail to verify your new email address and re-login. " + in result.output + ) + + result = receive_email(new_email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' 0 @@ -583,27 +239,39 @@ def test_account_profile_update( ) result = clirunner.invoke( - cmd_account, ["login", "-u", new_username, "-p", credentials["password"]], + cmd_account, ["login", "-u", new_username, "-p", password], ) validate_cliresult(result) - result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - credentials["password"], - "--username", - old_username, - ], - ) - validate_cliresult(result) - assert "Profile successfully updated!" in result.output - assert "Please re-login." in result.output + # pio account destroy with linked resource - result = clirunner.invoke( - cmd_account, ["login", "-u", old_username, "-p", credentials["password"]], + package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" + + tmp_dir = tmpdir_factory.mktemp("package") + fd = FileDownloader(package_url, str(tmp_dir)) + pkg_dir = tmp_dir.mkdir("raw_package") + fd.start(with_progress=False, silent=True) + with FileUnpacker(fd.get_filepath()) as unpacker: + unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) + + result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) + validate_cliresult(result) + try: + result = receive_email(new_email) + assert "Congrats" in result + assert "was published" in result + except: # pylint:disable=bare-except + pass + + result = clirunner.invoke(cmd_account, ["destroy"], "y") + assert result.exit_code != 0 + assert ( + "We can not destroy the %s account due to 1 linked resources from registry" + % username ) + + result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) validate_cliresult(result) finally: - clirunner.invoke(cmd_account, ["logout"]) + result = clirunner.invoke(cmd_account, ["destroy"], "y") + validate_cliresult(result) diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py index 4650caaf..3af38e83 100644 --- a/tests/commands/test_orgs.py +++ b/tests/commands/test_orgs.py @@ -14,142 +14,150 @@ import json import os +import time import pytest +import requests from platformio.commands.account import cli as cmd_account from platformio.commands.org import cli as cmd_org -pytestmark = pytest.mark.skipif( - not ( - os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") - and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") - ), - reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", -) +@pytest.mark.skipif( + not os.environ.get("TEST_EMAIL_LOGIN"), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) # pylint:disable=too-many-arguments +def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): + username = "test-piocore-%s" % str(int(time.time() * 1000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" -@pytest.fixture(scope="session") -def credentials(): - return { - "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], - "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], - } + # pio account register + result = clirunner.invoke( + cmd_account, + [ + "register", + "-u", + username, + "-e", + email, + "-p", + password, + "--firstname", + firstname, + "--lastname", + lastname, + ], + ) + validate_cliresult(result) + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split('= 3 - check = False - for org in json_result: - assert "orgname" in org - orgname = org["orgname"] - assert "displayname" in org - assert "email" in org - assert "owners" in org - for owner in org.get("owners"): - assert "username" in owner - check = owner["username"] == credentials["login"] if not check else True - assert "firstname" in owner - assert "lastname" in owner - assert check + assert json_result == [ + { + "orgname": new_orgname, + "displayname": new_display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} + ], + } + ] - result = clirunner.invoke(cmd_org, ["add", orgname, "ivankravets"],) + result = clirunner.invoke( + cmd_org, + [ + "update", + new_orgname, + "--new-orgname", + orgname, + "--displayname", + display_name, + ], + ) validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) >= 3 - check = False - for item in json_result: - if item["orgname"] != orgname: - continue - for owner in item.get("owners"): - check = owner["username"] == "ivankravets" if not check else True - assert check - - result = clirunner.invoke(cmd_org, ["remove", orgname, "ivankravets"],) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) >= 3 - check = False - for item in json_result: - if item["orgname"] != orgname: - continue - for owner in item.get("owners"): - check = owner["username"] == "ivankravets" if not check else True - assert not check finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip -def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) + result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") validate_cliresult(result) - assert "Successfully logged in!" in result.output - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + result = clirunner.invoke(cmd_account, ["destroy"], "y") validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) >= 3 - org = json_result[0] - assert "orgname" in org - assert "displayname" in org - assert "email" in org - assert "owners" in org - - old_orgname = org["orgname"] - if len(old_orgname) > 10: - new_orgname = "neworg" + org["orgname"][6:] - - result = clirunner.invoke( - cmd_org, ["update", old_orgname, "--new-orgname", new_orgname], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_org, ["update", new_orgname, "--new-orgname", old_orgname], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - assert json.loads(result.output.strip()) == json_result - finally: - clirunner.invoke(cmd_account, ["logout"]) diff --git a/tests/commands/test_teams.py b/tests/commands/test_teams.py index 13e30ce0..92d5226d 100644 --- a/tests/commands/test_teams.py +++ b/tests/commands/test_teams.py @@ -17,123 +17,128 @@ import os import time import pytest +import requests from platformio.commands.account import cli as cmd_account from platformio.commands.org import cli as cmd_org from platformio.commands.team import cli as cmd_team -pytestmark = pytest.mark.skipif( - not ( - os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") - and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") - ), - reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", -) +@pytest.mark.skipif( + not os.environ.get("TEST_EMAIL_LOGIN"), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) # pylint:disable=too-many-arguments +def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): + username = "test-piocore-%s" % str(int(time.time() * 1000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" -@pytest.fixture(scope="session") -def credentials(): - return { - "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], - "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], - } + # pio account register + result = clirunner.invoke( + cmd_account, + [ + "register", + "-u", + username, + "-e", + email, + "-p", + password, + "--firstname", + firstname, + "--lastname", + lastname, + ], + ) + validate_cliresult(result) + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split('= 3 - orgname = json_result[0].get("orgname") - + # pio team create result = clirunner.invoke( cmd_team, [ "create", "%s:%s" % (orgname, teamname), "--description", - "team for CI test", + team_description, ], ) validate_cliresult(result) + # pio team list result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) >= 1 - check = False - for team in json_result: - assert team["id"] - assert team["name"] - if team["name"] == teamname: - check = True - assert "description" in team - assert "members" in team - assert check + for item in json_result: + del item["id"] + assert json_result == [ + {"name": teamname, "description": team_description, "members": []} + ] + # pio team add (member) result = clirunner.invoke( - cmd_team, ["add", "%s:%s" % (orgname, teamname), credentials["login"]], + cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username], ) validate_cliresult(result) result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) - json_result = json.loads(result.output.strip()) - check = False - for team in json_result: - assert team["id"] - assert team["name"] - assert "description" in team - assert "members" in team - if ( - len(team["members"]) > 0 - and team["members"][0]["username"] == credentials["login"] - ): - check = True - assert check + assert second_username in result.output + # pio team remove (member) result = clirunner.invoke( - cmd_team, ["remove", "%s:%s" % (orgname, teamname), credentials["login"]], + cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username], ) validate_cliresult(result) result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) + assert second_username not in result.output + # pio team update + new_teamname = "new-" + str(int(time.time() * 1000)) + newteam_description = "Updated Description" result = clirunner.invoke( cmd_team, [ "update", "%s:%s" % (orgname, teamname), + "--name", + new_teamname, "--description", - "Updated Description", + newteam_description, ], ) validate_cliresult(result) @@ -141,18 +146,30 @@ def test_teams(clirunner, credentials, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) >= 1 - check = False - for team in json_result: - assert team["id"] - assert team["name"] - assert "description" in team - if team.get("description") == "Updated Description": - check = True - assert "members" in team - assert check - finally: - clirunner.invoke( - cmd_team, ["destroy", "%s:%s" % (orgname, teamname),], + for item in json_result: + del item["id"] + assert json_result == [ + {"name": new_teamname, "description": newteam_description, "members": []} + ] + + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (orgname, new_teamname), + "--name", + teamname, + "--description", + team_description, + ], ) - clirunner.invoke(cmd_account, ["logout"]) + validate_cliresult(result) + finally: + result = clirunner.invoke( + cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y" + ) + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") + validate_cliresult(result) + result = clirunner.invoke(cmd_account, ["destroy"], "y") + validate_cliresult(result) diff --git a/tests/conftest.py b/tests/conftest.py index f0529146..4b2259ac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import email import os +import poplib +import time import pytest from click.testing import CliRunner @@ -53,3 +56,43 @@ def isolated_pio_home(request, tmpdir_factory): @pytest.fixture(scope="function") def without_internet(monkeypatch): monkeypatch.setattr(util, "_internet_on", lambda: False) + + +@pytest.fixture +def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals + def _receive_email(from_who): + test_email = os.environ.get("TEST_EMAIL_LOGIN") + test_password = os.environ.get("TEST_EMAIL_PASSWORD") + pop_server = os.environ.get("TEST_EMAIL_POP3_SERVER") or "pop.gmail.com" + if "gmail" in pop_server: + test_email = "recent:" + test_email + + def get_body(msg): + if msg.is_multipart(): + return get_body(msg.get_payload(0)) + return msg.get_payload(None, True) + + result = None + start_time = time.time() + while not result: + time.sleep(5) + server = poplib.POP3_SSL(pop_server) + server.user(test_email) + server.pass_(test_password) + _, mails, _ = server.list() + for index, _ in enumerate(mails): + _, lines, _ = server.retr(index + 1) + msg_content = b"\n".join(lines) + msg = email.message_from_string( + msg_content.decode("ASCII", errors="surrogateescape") + ) + if from_who not in msg.get("To"): + continue + server.dele(index + 1) + result = get_body(msg).decode() + if time.time() - start_time > 60: + break + server.quit() + return result + + return _receive_email From e0023bb9086b6732290b7a28eb3bf56c18f31af8 Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Tue, 9 Jun 2020 17:05:11 +0300 Subject: [PATCH 041/223] increase tests email receiving time --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4b2259ac..542527b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,7 +90,7 @@ def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals continue server.dele(index + 1) result = get_body(msg).decode() - if time.time() - start_time > 60: + if time.time() - start_time > 120: break server.quit() return result From 3c8e0b17a7c42691e8fe4e0c418e88e926d8c453 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Jun 2020 18:43:50 +0300 Subject: [PATCH 042/223] Added support for custom targets --- HISTORY.rst | 9 ++- docs | 2 +- platformio/builder/main.py | 3 +- platformio/builder/tools/pioide.py | 4 +- platformio/builder/tools/piomisc.py | 58 ++++---------- platformio/builder/tools/piotarget.py | 109 ++++++++++++++++++++++++++ 6 files changed, 139 insertions(+), 46 deletions(-) create mode 100644 platformio/builder/tools/piotarget.py diff --git a/HISTORY.rst b/HISTORY.rst index 250e680a..65392477 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,9 +6,16 @@ Release Notes PlatformIO Core 4 ----------------- -4.3.5 (2020-??-??) +4.4.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* New `Account Management System `__ (preview) + + - Manage own organizations + - Manage organization teams + - Manage resource access + +* Added support for `custom targets `__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. 4.3.4 (2020-05-23) diff --git a/docs b/docs index cfba4f45..439f402c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit cfba4f456843f1069d9ebd083e359186054c8659 +Subproject commit 439f402c5b882af01b48068180810a58fc6db5ae diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 7184da7c..a0a8ab12 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -55,6 +55,7 @@ DEFAULT_ENV_OPTIONS = dict( "c++", "link", "platformio", + "piotarget", "pioplatform", "pioproject", "piomaxlen", @@ -217,7 +218,7 @@ if "idedata" in COMMAND_LINE_TARGETS: click.echo( "\n%s\n" % dump_json_to_unicode( - projenv.DumpIDEData() # pylint: disable=undefined-variable + projenv.DumpIDEData(env) # pylint: disable=undefined-variable ) ) env.Exit(0) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 65203ab7..acb36ae4 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -143,7 +143,8 @@ def _escape_build_flag(flags): return [flag if " " not in flag else '"%s"' % flag for flag in flags] -def DumpIDEData(env): +def DumpIDEData(env, globalenv): + """ env here is `projenv`""" env["__escape_build_flag"] = _escape_build_flag @@ -169,6 +170,7 @@ def DumpIDEData(env): ], "svd_path": _get_svd_path(env), "compiler_type": env.GetCompilerType(), + "targets": globalenv.DumpTargets(), } env_ = env.Clone() diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 1079f402..aa5158fe 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -16,15 +16,12 @@ from __future__ import absolute_import import atexit import io +import os import re import sys -from os import environ, remove, walk -from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp import click -from SCons.Action import Action # pylint: disable=import-error -from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import fs, util from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape @@ -126,11 +123,11 @@ class InoToCPPConverter(object): '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( out_file, tmp_path ), - "Converting " + basename(out_file[:-4]), + "Converting " + os.path.basename(out_file[:-4]), ) ) atexit.register(_delete_file, tmp_path) - return isfile(out_file) + return os.path.isfile(out_file) def _join_multiline_strings(self, contents): if "\\\n" not in contents: @@ -233,7 +230,9 @@ class InoToCPPConverter(object): def ConvertInoToCpp(env): src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR")) - ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")) + ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( + os.path.join(src_dir, "*.pde") + ) if not ino_nodes: return c = InoToCPPConverter(env) @@ -244,8 +243,8 @@ def ConvertInoToCpp(env): def _delete_file(path): try: - if isfile(path): - remove(path) + if os.path.isfile(path): + os.remove(path) except: # pylint: disable=bare-except pass @@ -255,7 +254,7 @@ def _get_compiler_type(env): if env.subst("$CC").endswith("-gcc"): return "gcc" try: - sysenv = environ.copy() + sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command([env.subst("$CC"), "-v"], env=sysenv) except OSError: @@ -277,8 +276,8 @@ def GetCompilerType(env): def GetActualLDScript(env): def _lookup_in_ldpath(script): for d in env.get("LIBPATH", []): - path = join(env.subst(d), script) - if isfile(path): + path = os.path.join(env.subst(d), script) + if os.path.isfile(path): return path return None @@ -297,7 +296,7 @@ def GetActualLDScript(env): else: continue script = env.subst(raw_script.replace('"', "").strip()) - if isfile(script): + if os.path.isfile(script): return script path = _lookup_in_ldpath(script) if path: @@ -319,29 +318,6 @@ def GetActualLDScript(env): env.Exit(1) -def VerboseAction(_, act, actstr): - if int(ARGUMENTS.get("PIOVERBOSE", 0)): - return act - return Action(act, actstr) - - -def PioClean(env, clean_dir): - if not isdir(clean_dir): - print("Build environment is clean") - env.Exit(0) - clean_rel_path = relpath(clean_dir) - for root, _, files in walk(clean_dir): - for f in files: - dst = join(root, f) - remove(dst) - print( - "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst)) - ) - print("Done cleaning") - fs.rmtree(clean_dir) - env.Exit(0) - - def ConfigureDebugFlags(env): def _cleanup_debug_flags(scope): if scope not in env: @@ -370,16 +346,16 @@ def ConfigureDebugFlags(env): def ConfigureTestTarget(env): env.Append( CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], - CPPPATH=[join("$BUILD_DIR", "UnityTestLib")], + CPPPATH=[os.path.join("$BUILD_DIR", "UnityTestLib")], ) unitylib = env.BuildLibrary( - join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity") + os.path.join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity") ) env.Prepend(LIBS=[unitylib]) src_filter = ["+<*.cpp>", "+<*.c>"] if "PIOTEST_RUNNING_NAME" in env: - src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], sep)) + src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], os.path.sep)) env.Replace(PIOTEST_SRC_FILTER=src_filter) @@ -393,7 +369,7 @@ def GetExtraScripts(env, scope): if not items: return items with fs.cd(env.subst("$PROJECT_DIR")): - return [realpath(item) for item in items] + return [os.path.realpath(item) for item in items] def exists(_): @@ -404,8 +380,6 @@ def generate(env): env.AddMethod(ConvertInoToCpp) env.AddMethod(GetCompilerType) env.AddMethod(GetActualLDScript) - env.AddMethod(VerboseAction) - env.AddMethod(PioClean) env.AddMethod(ConfigureDebugFlags) env.AddMethod(ConfigureTestTarget) env.AddMethod(GetExtraScripts) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py new file mode 100644 index 00000000..cbe90455 --- /dev/null +++ b/platformio/builder/tools/piotarget.py @@ -0,0 +1,109 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from SCons.Action import Action # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error + +from platformio import fs + + +def VerboseAction(_, act, actstr): + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + return act + return Action(act, actstr) + + +def PioClean(env, clean_dir): + if not os.path.isdir(clean_dir): + print("Build environment is clean") + env.Exit(0) + clean_rel_path = os.path.relpath(clean_dir) + for root, _, files in os.walk(clean_dir): + for f in files: + dst = os.path.join(root, f) + os.remove(dst) + print( + "Removed %s" + % (dst if clean_rel_path.startswith(".") else os.path.relpath(dst)) + ) + print("Done cleaning") + fs.rmtree(clean_dir) + env.Exit(0) + + +def _add_pio_target( # pylint: disable=too-many-arguments + env, + scope, + name, + dependencies, + actions, + title=None, + description=None, + always_build=True, +): + if "__PIO_TARGETS" not in env: + env["__PIO_TARGETS"] = {} + assert name not in env["__PIO_TARGETS"] + env["__PIO_TARGETS"][name] = dict( + name=name, scope=scope, title=title, description=description + ) + target = env.Alias(name, dependencies, actions) + if always_build: + AlwaysBuild(target) + return target + + +def AddSystemTarget(env, *args, **kwargs): + return _add_pio_target(env, "system", *args, **kwargs) + + +def AddCustomTarget(env, *args, **kwargs): + return _add_pio_target(env, "custom", *args, **kwargs) + + +def DumpTargets(env): + print("DumpTargets", id(env)) + targets = env.get("__PIO_TARGETS") or {} + # pre-fill default system targets + if ( + not any(t["scope"] == "system" for t in targets.values()) + and env.PioPlatform().is_embedded() + ): + targets["upload"] = dict(name="upload", scope="system", title="Upload") + targets["compiledb"] = dict( + name="compiledb", + scope="system", + title="Compilation database", + description="Generate compilation database `compile_commands.json`", + ) + targets["clean"] = dict(name="clean", scope="system", title="Clean") + return list(targets.values()) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(VerboseAction) + env.AddMethod(PioClean) + env.AddMethod(AddSystemTarget) + env.AddMethod(AddCustomTarget) + env.AddMethod(DumpTargets) + return env From 89cc6f9bf353bfe417f781de2a96b0dcf51674cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Jun 2020 18:44:49 +0300 Subject: [PATCH 043/223] Bump version to 4.4.0a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b4db9ecf..3dab3e3a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, "5a2") +VERSION = (4, 4, "0a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 062a82c89e07f04989b5882ef2bf6f5f74784b43 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Jun 2020 20:59:23 +0300 Subject: [PATCH 044/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 439f402c..4ceaa801 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 439f402c5b882af01b48068180810a58fc6db5ae +Subproject commit 4ceaa801e6f1eb25b40edcb709746c85c86706fe From e6fbd6acf1a335b5198a1931f4a6c43a6c19bc70 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 9 Jun 2020 23:26:49 +0300 Subject: [PATCH 045/223] Remove debug code --- platformio/builder/tools/piotarget.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index cbe90455..c4ddbade 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -78,7 +78,6 @@ def AddCustomTarget(env, *args, **kwargs): def DumpTargets(env): - print("DumpTargets", id(env)) targets = env.get("__PIO_TARGETS") or {} # pre-fill default system targets if ( From a182cca5e9a339b1a754c7d2f09e516bd9c2be62 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 10 Jun 2020 11:07:19 +0300 Subject: [PATCH 046/223] tests fix (#3555) * replace timestamp with randint in tests * replace pop3 with imap --- .github/workflows/core.yml | 2 +- tests/commands/test_account.py | 10 +++++----- tests/commands/test_orgs.py | 8 ++++---- tests/commands/test_teams.py | 10 +++++----- tests/conftest.py | 29 +++++++++++++++-------------- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c2e9547c..8332e944 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -30,7 +30,7 @@ jobs: env: TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }} TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }} - TEST_EMAIL_POP3_SERVER: ${{ secrets.TEST_EMAIL_POP3_SERVER }} + TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }} run: | tox -e testcore diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index 221b724a..dff24f52 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -14,7 +14,7 @@ import json import os -import time +import random import pytest import requests @@ -32,7 +32,7 @@ from platformio.unpacker import FileUnpacker def test_account( clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory ): - username = "test-piocore-%s" % str(int(time.time() * 1000)) + username = "test-piocore-%s" % str(random.randint(0, 100000)) splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) firstname = "Test" @@ -191,10 +191,10 @@ def test_account( validate_cliresult(result) # pio account update - firstname = "First " + str(int(time.time() * 1000)) - lastname = "Last" + str(int(time.time() * 1000)) + firstname = "First " + str(random.randint(0, 100000)) + lastname = "Last" + str(random.randint(0, 100000)) - new_username = "username" + str(int(time.time() * 1000))[-5:] + new_username = "username" + str(random.randint(0, 100000)) new_email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) result = clirunner.invoke( cmd_account, diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py index 3af38e83..d2001d0c 100644 --- a/tests/commands/test_orgs.py +++ b/tests/commands/test_orgs.py @@ -14,7 +14,7 @@ import json import os -import time +import random import pytest import requests @@ -28,7 +28,7 @@ from platformio.commands.org import cli as cmd_org reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", ) # pylint:disable=too-many-arguments def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(int(time.time() * 1000)) + username = "test-piocore-%s" % str(random.randint(0, 100000)) splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) firstname = "Test" @@ -71,7 +71,7 @@ def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) validate_cliresult(result) - orgname = "testorg-piocore-%s" % str(int(time.time() * 1000)) + orgname = "testorg-piocore-%s" % str(random.randint(0, 100000)) display_name = "Test Org for PIO Core" second_username = "ivankravets" try: @@ -114,7 +114,7 @@ def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): assert second_username not in result.output # pio org update - new_orgname = "neworg-piocore-%s" % str(int(time.time() * 1000)) + new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) new_display_name = "Test Org for PIO Core" result = clirunner.invoke( diff --git a/tests/commands/test_teams.py b/tests/commands/test_teams.py index 92d5226d..57085502 100644 --- a/tests/commands/test_teams.py +++ b/tests/commands/test_teams.py @@ -14,7 +14,7 @@ import json import os -import time +import random import pytest import requests @@ -29,7 +29,7 @@ from platformio.commands.team import cli as cmd_team reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", ) # pylint:disable=too-many-arguments def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(int(time.time() * 1000)) + username = "test-piocore-%s" % str(random.randint(0, 100000)) splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) firstname = "Test" @@ -72,7 +72,7 @@ def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) validate_cliresult(result) - orgname = "testorg-piocore-%s" % str(int(time.time() * 1000)) + orgname = "testorg-piocore-%s" % str(random.randint(0, 100000)) display_name = "Test Org for PIO Core" # pio org create @@ -81,7 +81,7 @@ def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): ) validate_cliresult(result) - teamname = "test-" + str(int(time.time() * 1000)) + teamname = "test-" + str(random.randint(0, 100000)) team_description = "team for CI test" second_username = "ivankravets" try: @@ -128,7 +128,7 @@ def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): assert second_username not in result.output # pio team update - new_teamname = "new-" + str(int(time.time() * 1000)) + new_teamname = "new-" + str(random.randint(0, 100000)) newteam_description = "Updated Description" result = clirunner.invoke( cmd_team, diff --git a/tests/conftest.py b/tests/conftest.py index 542527b1..9fa3578b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,8 +13,8 @@ # limitations under the License. import email +import imaplib import os -import poplib import time import pytest @@ -63,9 +63,7 @@ def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals def _receive_email(from_who): test_email = os.environ.get("TEST_EMAIL_LOGIN") test_password = os.environ.get("TEST_EMAIL_PASSWORD") - pop_server = os.environ.get("TEST_EMAIL_POP3_SERVER") or "pop.gmail.com" - if "gmail" in pop_server: - test_email = "recent:" + test_email + imap_server = os.environ.get("TEST_EMAIL_IMAP_SERVER") or "imap.gmail.com" def get_body(msg): if msg.is_multipart(): @@ -76,23 +74,26 @@ def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals start_time = time.time() while not result: time.sleep(5) - server = poplib.POP3_SSL(pop_server) - server.user(test_email) - server.pass_(test_password) - _, mails, _ = server.list() - for index, _ in enumerate(mails): - _, lines, _ = server.retr(index + 1) - msg_content = b"\n".join(lines) + server = imaplib.IMAP4_SSL(imap_server) + server.login(test_email, test_password) + server.select("INBOX") + _, mails = server.search(None, "ALL") + for index in mails[0].split(): + _, data = server.fetch(index, "(RFC822)") msg = email.message_from_string( - msg_content.decode("ASCII", errors="surrogateescape") + data[0][1].decode("ASCII", errors="surrogateescape") ) if from_who not in msg.get("To"): continue - server.dele(index + 1) + if "gmail" in imap_server: + server.store(index, "+X-GM-LABELS", "\\Trash") + server.store(index, "+FLAGS", "\\Deleted") + server.expunge() result = get_body(msg).decode() if time.time() - start_time > 120: break - server.quit() + server.close() + server.logout() return result return _receive_email From 0d8272890c901f5bb4d56d875ce743a01fc4cf61 Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Wed, 10 Jun 2020 12:02:34 +0300 Subject: [PATCH 047/223] merge account, org and team tests into one file --- ...st_account.py => test_account_org_team.py} | 293 ++++++++++++++++++ tests/commands/test_orgs.py | 163 ---------- tests/commands/test_teams.py | 175 ----------- 3 files changed, 293 insertions(+), 338 deletions(-) rename tests/commands/{test_account.py => test_account_org_team.py} (51%) delete mode 100644 tests/commands/test_orgs.py delete mode 100644 tests/commands/test_teams.py diff --git a/tests/commands/test_account.py b/tests/commands/test_account_org_team.py similarity index 51% rename from tests/commands/test_account.py rename to tests/commands/test_account_org_team.py index dff24f52..49530f5d 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account_org_team.py @@ -20,7 +20,9 @@ import pytest import requests from platformio.commands.account import cli as cmd_account +from platformio.commands.org import cli as cmd_org from platformio.commands.package import cli as cmd_package +from platformio.commands.team import cli as cmd_team from platformio.downloader import FileDownloader from platformio.unpacker import FileUnpacker @@ -275,3 +277,294 @@ def test_account( finally: result = clirunner.invoke(cmd_account, ["destroy"], "y") validate_cliresult(result) + + +@pytest.mark.skipif( + not os.environ.get("TEST_EMAIL_LOGIN"), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) # pylint:disable=too-many-arguments +def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): + username = "test-piocore-%s" % str(random.randint(0, 100000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" + + # pio account register + result = clirunner.invoke( + cmd_account, + [ + "register", + "-u", + username, + "-e", + email, + "-p", + password, + "--firstname", + firstname, + "--lastname", + lastname, + ], + ) + validate_cliresult(result) + + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import random - -import pytest -import requests - -from platformio.commands.account import cli as cmd_account -from platformio.commands.org import cli as cmd_org - - -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), - reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(random.randint(0, 100000)) - splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") - email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) - firstname = "Test" - lastname = "User" - password = "Qwerty123!" - - # pio account register - result = clirunner.invoke( - cmd_account, - [ - "register", - "-u", - username, - "-e", - email, - "-p", - password, - "--firstname", - firstname, - "--lastname", - lastname, - ], - ) - validate_cliresult(result) - - # email verification - result = receive_email(email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() - ) - session = requests.Session() - result = session.get(link).text - link = result.split(' -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import random - -import pytest -import requests - -from platformio.commands.account import cli as cmd_account -from platformio.commands.org import cli as cmd_org -from platformio.commands.team import cli as cmd_team - - -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), - reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(random.randint(0, 100000)) - splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") - email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) - firstname = "Test" - lastname = "User" - password = "Qwerty123!" - - # pio account register - result = clirunner.invoke( - cmd_account, - [ - "register", - "-u", - username, - "-e", - email, - "-p", - password, - "--firstname", - firstname, - "--lastname", - lastname, - ], - ) - validate_cliresult(result) - - # email verification - result = receive_email(email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() - ) - session = requests.Session() - result = session.get(link).text - link = result.split(' Date: Wed, 10 Jun 2020 12:22:28 +0300 Subject: [PATCH 048/223] cleaning --- tests/commands/test_account_org_team.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index 49530f5d..770a692e 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -275,8 +275,7 @@ def test_account( result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) validate_cliresult(result) finally: - result = clirunner.invoke(cmd_account, ["destroy"], "y") - validate_cliresult(result) + clirunner.invoke(cmd_account, ["destroy"], "y") @pytest.mark.skipif( @@ -413,10 +412,8 @@ def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): ) validate_cliresult(result) finally: - result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") - validate_cliresult(result) - result = clirunner.invoke(cmd_account, ["destroy"], "y") - validate_cliresult(result) + clirunner.invoke(cmd_org, ["destroy", orgname], "y") + clirunner.invoke(cmd_account, ["destroy"], "y") @pytest.mark.skipif( @@ -560,11 +557,6 @@ def test_team(clirunner, validate_cliresult, receive_email, isolated_pio_home): ) validate_cliresult(result) finally: - result = clirunner.invoke( - cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y" - ) - validate_cliresult(result) - result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") - validate_cliresult(result) - result = clirunner.invoke(cmd_account, ["destroy"], "y") - validate_cliresult(result) + clirunner.invoke(cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y") + clirunner.invoke(cmd_org, ["destroy", orgname], "y") + clirunner.invoke(cmd_account, ["destroy"], "y") From 9e3ba11e8ad748b6871efd894223c23c3103dbe0 Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Wed, 10 Jun 2020 12:36:07 +0300 Subject: [PATCH 049/223] skip account tests --- tests/commands/test_account_org_team.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index 770a692e..a7711ee2 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -26,6 +26,8 @@ from platformio.commands.team import cli as cmd_team from platformio.downloader import FileDownloader from platformio.unpacker import FileUnpacker +pytestmark = pytest.mark.skip() + @pytest.mark.skipif( not os.environ.get("TEST_EMAIL_LOGIN"), From b71b939307c5eb5139fa683ef404453f1635ba90 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jun 2020 14:25:53 +0300 Subject: [PATCH 050/223] Rename "AddSystemTarget" to "AddPlatformTarget" --- platformio/builder/tools/piotarget.py | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index c4ddbade..1dc5ff32 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -47,21 +47,21 @@ def PioClean(env, clean_dir): env.Exit(0) -def _add_pio_target( # pylint: disable=too-many-arguments +def AddTarget( # pylint: disable=too-many-arguments env, - scope, name, dependencies, actions, title=None, description=None, + group="Generic", always_build=True, ): if "__PIO_TARGETS" not in env: env["__PIO_TARGETS"] = {} assert name not in env["__PIO_TARGETS"] env["__PIO_TARGETS"][name] = dict( - name=name, scope=scope, title=title, description=description + name=name, title=title, description=description, group=group ) target = env.Alias(name, dependencies, actions) if always_build: @@ -69,29 +69,28 @@ def _add_pio_target( # pylint: disable=too-many-arguments return target -def AddSystemTarget(env, *args, **kwargs): - return _add_pio_target(env, "system", *args, **kwargs) +def AddPlatformTarget(env, *args, **kwargs): + return env.AddTarget(group="Platform", *args, **kwargs) def AddCustomTarget(env, *args, **kwargs): - return _add_pio_target(env, "custom", *args, **kwargs) + return env.AddTarget(group="Custom", *args, **kwargs) def DumpTargets(env): targets = env.get("__PIO_TARGETS") or {} - # pre-fill default system targets - if ( - not any(t["scope"] == "system" for t in targets.values()) - and env.PioPlatform().is_embedded() + # pre-fill default targets if embedded dev-platform + if env.PioPlatform().is_embedded() and not any( + t["group"] == "Platform" for t in targets.values() ): - targets["upload"] = dict(name="upload", scope="system", title="Upload") + targets["upload"] = dict(name="upload", group="Platform", title="Upload") targets["compiledb"] = dict( name="compiledb", - scope="system", title="Compilation database", description="Generate compilation database `compile_commands.json`", + group="Advanced", ) - targets["clean"] = dict(name="clean", scope="system", title="Clean") + targets["clean"] = dict(name="clean", title="Clean", group="Generic") return list(targets.values()) @@ -102,7 +101,8 @@ def exists(_): def generate(env): env.AddMethod(VerboseAction) env.AddMethod(PioClean) - env.AddMethod(AddSystemTarget) + env.AddMethod(AddTarget) + env.AddMethod(AddPlatformTarget) env.AddMethod(AddCustomTarget) env.AddMethod(DumpTargets) return env From ef8a9835b070826b093791af3ef7ca4e1304d68f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jun 2020 14:26:48 +0300 Subject: [PATCH 051/223] Bump version to 4.4.0a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 3dab3e3a..d373bbad 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a1") +VERSION = (4, 4, "0a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f571ad9d4756424e0d1875964bc8028d0997626e Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 11 Jun 2020 11:03:48 +0300 Subject: [PATCH 052/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4ceaa801..37b313fd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4ceaa801e6f1eb25b40edcb709746c85c86706fe +Subproject commit 37b313fd1576995a61d49a0520ea8d063b1a4da6 From 2722e2741525a6db6583c25203e9779a7a6b9e59 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 11 Jun 2020 15:15:46 +0300 Subject: [PATCH 053/223] Sync docs --- docs | 2 +- platformio/builder/tools/piotarget.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 37b313fd..ec54ae99 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 37b313fd1576995a61d49a0520ea8d063b1a4da6 +Subproject commit ec54ae991784ce854dfe939bcaf4c2545abcf55e diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 1dc5ff32..7106a40d 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -86,7 +86,7 @@ def DumpTargets(env): targets["upload"] = dict(name="upload", group="Platform", title="Upload") targets["compiledb"] = dict( name="compiledb", - title="Compilation database", + title="Compilation Database", description="Generate compilation database `compile_commands.json`", group="Advanced", ) From 266612bbdf753326deda5e1897aed80d23e46127 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 11 Jun 2020 15:27:51 +0300 Subject: [PATCH 054/223] Run CI on pull requests --- .github/workflows/core.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/examples.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 8332e944..c6e15bfd 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -1,6 +1,6 @@ name: Core -on: [push] +on: [push, pull_request] jobs: build: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bfe2c116..39de401e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,6 +1,6 @@ name: Docs -on: [push] +on: [push, pull_request] jobs: build: diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b5452909..f1db5c38 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -1,6 +1,6 @@ name: Examples -on: [push] +on: [push, pull_request] jobs: build: From 405dcda8247cdfbd66c9a93b78faa683693b7aa7 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Thu, 11 Jun 2020 16:02:38 +0300 Subject: [PATCH 055/223] Feature/update account tests (#3556) * update account tests * change second user * refactoring * clean * fix tests email receiving * fix --- tests/commands/test_account_org_team.py | 949 +++++++++++------------- tests/conftest.py | 4 +- 2 files changed, 441 insertions(+), 512 deletions(-) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index a7711ee2..cc97b33f 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -26,16 +26,29 @@ from platformio.commands.team import cli as cmd_team from platformio.downloader import FileDownloader from platformio.unpacker import FileUnpacker -pytestmark = pytest.mark.skip() - - -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), +pytestmark = pytest.mark.skipif( + not (os.environ.get("TEST_EMAIL_LOGIN") and os.environ.get("TEST_EMAIL_PASSWORD")), reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_account( - clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory -): +) + +username = None +email = None +firstname = None +lastname = None +password = None + +orgname = None +display_name = None +second_username = None + +teamname = None +team_description = None + + +def test_prepare(): + global username, splited_email, email, firstname, lastname + global password, orgname, display_name, second_username, teamname, team_description + username = "test-piocore-%s" % str(random.randint(0, 100000)) splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) @@ -43,522 +56,436 @@ def test_account( lastname = "User" password = "Qwerty123!" - # pio account register - result = clirunner.invoke( - cmd_account, - [ - "register", - "-u", - username, - "-e", - email, - "-p", - password, - "--firstname", - firstname, - "--lastname", - lastname, - ], - ) - validate_cliresult(result) - - # email verification - result = receive_email(email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() - ) - session = requests.Session() - result = session.get(link).text - link = result.split(' 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - os.environ["PLATFORMIO_AUTH_TOKEN"] = token - - result = clirunner.invoke( - cmd_account, ["token", "--password", password, "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - - os.environ.pop("PLATFORMIO_AUTH_TOKEN") - - result = clirunner.invoke( - cmd_account, ["login", "-u", username, "-p", password], - ) - validate_cliresult(result) - - # pio account password - new_password = "Testpassword123" - result = clirunner.invoke( - cmd_account, - ["password", "--old-password", password, "--new-password", new_password,], - ) - validate_cliresult(result) - assert "Password successfully changed!" in result.output - - clirunner.invoke(cmd_account, ["logout"]) - - result = clirunner.invoke( - cmd_account, ["login", "-u", username, "-p", new_password], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - ["password", "--old-password", new_password, "--new-password", password,], - ) - validate_cliresult(result) - - # pio account update - firstname = "First " + str(random.randint(0, 100000)) - lastname = "Last" + str(random.randint(0, 100000)) - - new_username = "username" + str(random.randint(0, 100000)) - new_email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) - result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - password, - "--firstname", - firstname, - "--lastname", - lastname, - "--username", - new_username, - "--email", - new_email, - ], - ) - validate_cliresult(result) - assert "Profile successfully updated!" in result.output - assert ( - "Please check your mail to verify your new email address and re-login. " - in result.output - ) - - result = receive_email(new_email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() - ) - session = requests.Session() - result = session.get(link).text - link = result.split(' 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, ["login", "-u", new_username, "-p", password], - ) - validate_cliresult(result) - - # pio account destroy with linked resource - - package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" - - tmp_dir = tmpdir_factory.mktemp("package") - fd = FileDownloader(package_url, str(tmp_dir)) - pkg_dir = tmp_dir.mkdir("raw_package") - fd.start(with_progress=False, silent=True) - with FileUnpacker(fd.get_filepath()) as unpacker: - unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) - - result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) - validate_cliresult(result) - try: - result = receive_email(new_email) - assert "Congrats" in result - assert "was published" in result - except: # pylint:disable=bare-except - pass - - result = clirunner.invoke(cmd_account, ["destroy"], "y") - assert result.exit_code != 0 - assert ( - "We can not destroy the %s account due to 1 linked resources from registry" - % username - ) - - result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) - validate_cliresult(result) - finally: - clirunner.invoke(cmd_account, ["destroy"], "y") - - -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), - reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(random.randint(0, 100000)) - splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") - email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) - firstname = "Test" - lastname = "User" - password = "Qwerty123!" - - # pio account register - result = clirunner.invoke( - cmd_account, - [ - "register", - "-u", - username, - "-e", - email, - "-p", - password, - "--firstname", - firstname, - "--lastname", - lastname, - ], - ) - validate_cliresult(result) - - # email verification - result = receive_email(email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() - ) - session = requests.Session() - result = session.get(link).text - link = result.split(' 0 + assert result.exception + assert "You are not authorized! Please log in to PIO Account" in str( + result.exception + ) + + os.environ["PLATFORMIO_AUTH_TOKEN"] = token + + result = clirunner.invoke( + cmd_account, ["token", "--password", password, "--json-output"], + ) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result + assert json_result.get("status") == "success" + assert json_result.get("result") == token + + os.environ.pop("PLATFORMIO_AUTH_TOKEN") + + result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + validate_cliresult(result) + + +def test_account_change_password(clirunner, validate_cliresult, isolated_pio_home): + new_password = "Testpassword123" + result = clirunner.invoke( + cmd_account, + ["password", "--old-password", password, "--new-password", new_password,], + ) + validate_cliresult(result) + assert "Password successfully changed!" in result.output + + clirunner.invoke(cmd_account, ["logout"]) + + result = clirunner.invoke( + cmd_account, ["login", "-u", username, "-p", new_password], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_account, + ["password", "--old-password", new_password, "--new-password", password,], + ) + validate_cliresult(result) + + +def test_account_update( + clirunner, validate_cliresult, receive_email, isolated_pio_home +): + global username + global email + global firstname + global lastname + + firstname = "First " + str(random.randint(0, 100000)) + lastname = "Last" + str(random.randint(0, 100000)) + + username = "username" + str(random.randint(0, 100000)) + email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) + result = clirunner.invoke( + cmd_account, + [ + "update", + "--current-password", + password, + "--firstname", + firstname, + "--lastname", + lastname, + "--username", + username, + "--email", + email, + ], + ) + validate_cliresult(result) + assert "Profile successfully updated!" in result.output + assert ( + "Please check your mail to verify your new email address and re-login. " + in result.output + ) + + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' 0 + assert result.exception + assert "You are not authorized! Please log in to PIO Account" in str( + result.exception + ) + + result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + validate_cliresult(result) + + +def test_account_destroy_with_linked_resources( + clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory +): + package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" + + tmp_dir = tmpdir_factory.mktemp("package") + fd = FileDownloader(package_url, str(tmp_dir)) + pkg_dir = tmp_dir.mkdir("raw_package") + fd.start(with_progress=False, silent=True) + with FileUnpacker(fd.get_filepath()) as unpacker: + unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) + + result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) + validate_cliresult(result) try: - # pio team create - result = clirunner.invoke( - cmd_team, - [ - "create", - "%s:%s" % (orgname, teamname), - "--description", - team_description, + result = receive_email(email) + assert "Congrats" in result + assert "was published" in result + except: # pylint:disable=bare-except + pass + + result = clirunner.invoke(cmd_account, ["destroy"], "y") + assert result.exit_code != 0 + assert ( + "We can not destroy the %s account due to 1 linked resources from registry" + % username + ) + + result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) + validate_cliresult(result) + + +def test_org_create(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke( + cmd_org, ["create", "--email", email, "--displayname", display_name, orgname], + ) + validate_cliresult(result) + + +def test_org_list(clirunner, validate_cliresult, isolated_pio_home): + # pio org list + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result == [ + { + "orgname": orgname, + "displayname": display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} ], - ) - validate_cliresult(result) + } + ] - # pio team list - result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - for item in json_result: - del item["id"] - assert json_result == [ - {"name": teamname, "description": team_description, "members": []} - ] - # pio team add (member) - result = clirunner.invoke( - cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username], - ) - validate_cliresult(result) +def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke(cmd_org, ["add", orgname, second_username]) + validate_cliresult(result) - result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) - validate_cliresult(result) - assert second_username in result.output + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + assert second_username in result.output - # pio team remove (member) - result = clirunner.invoke( - cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username], - ) - validate_cliresult(result) - result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) - validate_cliresult(result) - assert second_username not in result.output +def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke(cmd_org, ["remove", orgname, second_username]) + validate_cliresult(result) - # pio team update - new_teamname = "new-" + str(random.randint(0, 100000)) - newteam_description = "Updated Description" - result = clirunner.invoke( - cmd_team, - [ - "update", - "%s:%s" % (orgname, teamname), - "--name", - new_teamname, - "--description", - newteam_description, + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + assert second_username not in result.output + + +def test_org_update(clirunner, validate_cliresult, isolated_pio_home): + new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) + new_display_name = "Test Org for PIO Core" + + result = clirunner.invoke( + cmd_org, + [ + "update", + orgname, + "--new-orgname", + new_orgname, + "--displayname", + new_display_name, + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result == [ + { + "orgname": new_orgname, + "displayname": new_display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} ], - ) - validate_cliresult(result) + } + ] - result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - for item in json_result: - del item["id"] - assert json_result == [ - {"name": new_teamname, "description": newteam_description, "members": []} - ] + result = clirunner.invoke( + cmd_org, + [ + "update", + new_orgname, + "--new-orgname", + orgname, + "--displayname", + display_name, + ], + ) + validate_cliresult(result) - result = clirunner.invoke( - cmd_team, - [ - "update", - "%s:%s" % (orgname, new_teamname), - "--name", - teamname, - "--description", - team_description, - ], - ) - validate_cliresult(result) - finally: - clirunner.invoke(cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y") - clirunner.invoke(cmd_org, ["destroy", orgname], "y") - clirunner.invoke(cmd_account, ["destroy"], "y") + +def test_team_create(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke( + cmd_team, + ["create", "%s:%s" % (orgname, teamname), "--description", team_description,], + ) + validate_cliresult(result) + + +def test_team_list(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + for item in json_result: + del item["id"] + assert json_result == [ + {"name": teamname, "description": team_description, "members": []} + ] + + +def test_team_add_member(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke( + cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + assert second_username in result.output + + +def test_team_remove(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke( + cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + assert second_username not in result.output + + +def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_home): + new_teamname = "new-" + str(random.randint(0, 100000)) + newteam_description = "Updated Description" + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (orgname, teamname), + "--name", + new_teamname, + "--description", + newteam_description, + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + for item in json_result: + del item["id"] + assert json_result == [ + {"name": new_teamname, "description": newteam_description, "members": []} + ] + + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (orgname, new_teamname), + "--name", + teamname, + "--description", + team_description, + ], + ) + validate_cliresult(result) + + +def test_cleanup(clirunner, validate_cliresult, receive_email, isolated_pio_home): + result = clirunner.invoke(cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y") + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") + validate_cliresult(result) + result = clirunner.invoke(cmd_account, ["destroy"], "y") + validate_cliresult(result) diff --git a/tests/conftest.py b/tests/conftest.py index 9fa3578b..eda52184 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,7 +79,9 @@ def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals server.select("INBOX") _, mails = server.search(None, "ALL") for index in mails[0].split(): - _, data = server.fetch(index, "(RFC822)") + status, data = server.fetch(index, "(RFC822)") + if status != "OK" or not data or not isinstance(data[0], tuple): + continue msg = email.message_from_string( data[0][1].decode("ASCII", errors="surrogateescape") ) From 660b57cdd3d1a4e2af60c128bba3658da43beb44 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 11 Jun 2020 21:16:06 +0300 Subject: [PATCH 056/223] Update PIO Home front-end to 3.2.3 --- platformio/managers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 53f435fd..27bee8c2 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.proc import get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": "~3.2.1", + "contrib-piohome": "~3.2.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", From fdb83c24be01d19ed48d42b332b082348a6b2cfd Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 11 Jun 2020 23:53:52 +0300 Subject: [PATCH 057/223] Clean autogenerated files before running tests // Resolve #3523 Fixes possible conflicts between auxiliary test transport files when project contains multiple environments with different platforms --- HISTORY.rst | 1 + platformio/commands/test/processor.py | 31 ++++++++++-------- tests/commands/test_test.py | 45 ++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 65392477..0add5842 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO Core 4 * Added support for `custom targets `__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. +* Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 9024ed0e..bb7c4a12 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -13,7 +13,7 @@ # limitations under the License. import atexit -from os import remove +from os import remove, listdir from os.path import isdir, isfile, join from string import Template @@ -194,24 +194,29 @@ class TestProcessorBase(object): ] ) - def delete_tmptest_file(file_): - try: - remove(file_) - except: # pylint: disable=bare-except - if isfile(file_): - click.secho( - "Warning: Could not remove temporary file '%s'. " - "Please remove it manually." % file_, - fg="yellow", - ) + tmp_file_prefix = "tmp_pio_test_transport" + + def delete_tmptest_files(test_dir): + for item in listdir(test_dir): + if item.startswith(tmp_file_prefix) and isfile(join(test_dir, item)): + try: + remove(join(test_dir, item)) + except: # pylint: disable=bare-except + click.secho( + "Warning: Could not remove temporary file '%s'. " + "Please remove it manually." % join(test_dir, item), + fg="yellow", + ) transport_options = TRANSPORT_OPTIONS[self.get_transport()] tpl = Template(file_tpl).substitute(transport_options) data = Template(tpl).substitute(baudrate=self.get_baudrate()) + + delete_tmptest_files(test_dir) tmp_file = join( - test_dir, "output_export." + transport_options.get("language", "c") + test_dir, "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")) ) with open(tmp_file, "w") as fp: fp.write(data) - atexit.register(delete_tmptest_file, tmp_file) + atexit.register(delete_tmptest_files, test_dir) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index a201e723..38fb7eb2 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -17,6 +17,7 @@ from os.path import join import pytest from platformio import util +from platformio.commands.test.command import cli as cmd_test def test_local_env(): @@ -31,7 +32,49 @@ def test_local_env(): ] ) if result["returncode"] != 1: - pytest.fail(result) + pytest.fail(str(result)) assert all([s in result["err"] for s in ("PASSED", "IGNORED", "FAILED")]), result[ "out" ] + + +def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): + + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:teensy31] +platform = teensy +framework = mbed +board = teensy31 + +[env:native] +platform = native + +[env:espressif32] +platform = espressif32 +framework = arduino +board = esp32dev +""" + ) + + project_dir.mkdir("test").join("test_main.cpp").write( + """ +#ifdef ARDUINO +void setup() {} +void loop() {} +#else +int main() { + UNITY_BEGIN(); + UNITY_END(); +} +#endif +""" + ) + + result = clirunner.invoke( + cmd_test, ["-d", str(project_dir), "--without-testing", "--without-uploading"], + ) + + validate_cliresult(result) + assert "Multiple ways to build" not in result.output From 28d9f25f9abd000a1be45bb61c98c11b27a2a61e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Jun 2020 23:47:12 +0300 Subject: [PATCH 058/223] Added a new "-e, --environment" option to "platformio project init" command --- HISTORY.rst | 1 + docs | 2 +- examples | 2 +- platformio/commands/project.py | 28 ++++++++++++- platformio/ide/projectgenerator.py | 64 +++++++++++------------------- tests/commands/test_init.py | 37 +++++++++++++---- 6 files changed, 83 insertions(+), 51 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0add5842..b725d474 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO Core 4 * Added support for `custom targets `__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. +* Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) 4.3.4 (2020-05-23) diff --git a/docs b/docs index ec54ae99..2178afaf 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ec54ae991784ce854dfe939bcaf4c2545abcf55e +Subproject commit 2178afaf333874a1f6d3dff4b45d0613517049b5 diff --git a/examples b/examples index c442de34..bcdcf466 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit c442de34a57b54451170dbe39f3411a06a05b3f2 +Subproject commit bcdcf46691c1fe79e8cb2cc74e1a156f6bac5e90 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 3d73f4ff..c83b44a7 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -93,6 +93,7 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 ) @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) +@click.option("-e", "--environment", help="Update using existing environment") @click.option("-O", "--project-option", multiple=True) @click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) @@ -102,6 +103,7 @@ def project_init( project_dir, board, ide, + environment, project_option, env_prefix, silent, @@ -139,7 +141,11 @@ def project_init( ) if ide: - pg = ProjectGenerator(project_dir, ide, board) + config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + config.validate() + pg = ProjectGenerator( + config, environment or get_best_envname(config, board), ide + ) pg.generate() if is_new_project: @@ -444,3 +450,23 @@ def _install_dependent_platforms(ctx, platforms): ctx.invoke( cli_platform_install, platforms=list(set(platforms) - set(installed_platforms)) ) + + +def get_best_envname(config, board_ids=None): + envname = None + default_envs = config.default_envs() + if default_envs: + envname = default_envs[0] + if not board_ids: + return envname + + for env in config.envs(): + if not board_ids: + return env + if not envname: + envname = env + items = config.items(env=env, as_dict=True) + if "board" in items and items.get("board") in board_ids: + return env + + return envname diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 8bf735b0..30eaa97d 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -15,47 +15,31 @@ import codecs import os import sys -from os.path import basename, isdir, isfile, join, realpath, relpath import bottle from platformio import fs, util from platformio.proc import where_is_program -from platformio.project.config import ProjectConfig from platformio.project.helpers import load_project_ide_data class ProjectGenerator(object): - def __init__(self, project_dir, ide, boards): - self.config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) - self.config.validate() - self.project_dir = project_dir + def __init__(self, config, env_name, ide): + self.config = config + self.project_dir = os.path.dirname(config.path) + self.env_name = str(env_name) self.ide = str(ide) - self.env_name = str(self.get_best_envname(boards)) @staticmethod def get_supported_ides(): - tpls_dir = join(fs.get_source_dir(), "ide", "tpls") - return sorted([d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) - - def get_best_envname(self, boards=None): - envname = None - default_envs = self.config.default_envs() - if default_envs: - envname = default_envs[0] - if not boards: - return envname - - for env in self.config.envs(): - if not boards: - return env - if not envname: - envname = env - items = self.config.items(env=env, as_dict=True) - if "board" in items and items.get("board") in boards: - return env - - return envname + tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls") + return sorted( + [ + d + for d in os.listdir(tpls_dir) + if os.path.isdir(os.path.join(tpls_dir, d)) + ] + ) @staticmethod def filter_includes(includes_map, ignore_scopes=None, to_unix_path=True): @@ -75,12 +59,12 @@ class ProjectGenerator(object): tpl_vars = { "config": self.config, "systype": util.get_systype(), - "project_name": basename(self.project_dir), + "project_name": os.path.basename(self.project_dir), "project_dir": self.project_dir, "env_name": self.env_name, - "user_home_dir": realpath(fs.expanduser("~")), + "user_home_dir": os.path.realpath(fs.expanduser("~")), "platformio_path": sys.argv[0] - if isfile(sys.argv[0]) + if os.path.isfile(sys.argv[0]) else where_is_program("platformio"), "env_path": os.getenv("PATH"), "env_pathsep": os.pathsep, @@ -97,7 +81,7 @@ class ProjectGenerator(object): "src_files": self.get_src_files(), "project_src_dir": self.config.get_optional_dir("src"), "project_lib_dir": self.config.get_optional_dir("lib"), - "project_libdeps_dir": join( + "project_libdeps_dir": os.path.join( self.config.get_optional_dir("libdeps"), self.env_name ), } @@ -120,12 +104,12 @@ class ProjectGenerator(object): with fs.cd(self.project_dir): for root, _, files in os.walk(self.config.get_optional_dir("src")): for f in files: - result.append(relpath(join(root, f))) + result.append(os.path.relpath(os.path.join(root, f))) return result def get_tpls(self): tpls = [] - tpls_dir = join(fs.get_source_dir(), "ide", "tpls", self.ide) + tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls", self.ide) for root, _, files in os.walk(tpls_dir): for f in files: if not f.endswith(".tpl"): @@ -133,7 +117,7 @@ class ProjectGenerator(object): _relpath = root.replace(tpls_dir, "") if _relpath.startswith(os.sep): _relpath = _relpath[1:] - tpls.append((_relpath, join(root, f))) + tpls.append((_relpath, os.path.join(root, f))) return tpls def generate(self): @@ -141,12 +125,12 @@ class ProjectGenerator(object): for tpl_relpath, tpl_path in self.get_tpls(): dst_dir = self.project_dir if tpl_relpath: - dst_dir = join(self.project_dir, tpl_relpath) - if not isdir(dst_dir): + dst_dir = os.path.join(self.project_dir, tpl_relpath) + if not os.path.isdir(dst_dir): os.makedirs(dst_dir) - file_name = basename(tpl_path)[:-4] + file_name = os.path.basename(tpl_path)[:-4] contents = self._render_tpl(tpl_path, tpl_vars) - self._merge_contents(join(dst_dir, file_name), contents) + self._merge_contents(os.path.join(dst_dir, file_name), contents) @staticmethod def _render_tpl(tpl_path, tpl_vars): @@ -155,7 +139,7 @@ class ProjectGenerator(object): @staticmethod def _merge_contents(dst_path, contents): - if basename(dst_path) == ".gitignore" and isfile(dst_path): + if os.path.basename(dst_path) == ".gitignore" and os.path.isfile(dst_path): return with codecs.open(dst_path, "w", encoding="utf8") as fp: fp.write(contents) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index b874ead7..09bd8cf9 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -62,29 +62,50 @@ def test_init_ide_without_board(clirunner, tmpdir): assert isinstance(result.exception, ProjectEnvsNotAvailableError) -def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): +def test_init_ide_vscode(clirunner, validate_cliresult, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke( - cmd_init, ["--ide", "atom", "-b", "uno", "-b", "teensy31"] + cmd_init, ["--ide", "vscode", "-b", "uno", "-b", "teensy31"] ) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert all( - [tmpdir.join(f).check() for f in (".clang_complete", ".gcc-flags.json")] + [ + tmpdir.join(".vscode").join(f).check() + for f in ("c_cpp_properties.json", "launch.json") + ] + ) + assert ( + "framework-arduino-avr" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() ) - assert "framework-arduino" in tmpdir.join(".clang_complete").read() # switch to NodeMCU - result = clirunner.invoke(cmd_init, ["--ide", "atom", "-b", "nodemcuv2"]) + result = clirunner.invoke(cmd_init, ["--ide", "vscode", "-b", "nodemcuv2"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert "arduinoespressif" in tmpdir.join(".clang_complete").read() + assert ( + "framework-arduinoespressif8266" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) + + # switch to teensy31 via env name + result = clirunner.invoke(cmd_init, ["--ide", "vscode", "-e", "teensy31"]) + validate_cliresult(result) + validate_pioproject(str(tmpdir)) + assert ( + "framework-arduinoteensy" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) # switch to the first board - result = clirunner.invoke(cmd_init, ["--ide", "atom"]) + result = clirunner.invoke(cmd_init, ["--ide", "vscode"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert "framework-arduino" in tmpdir.join(".clang_complete").read() + assert ( + "framework-arduino-avr" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) def test_init_ide_eclipse(clirunner, validate_cliresult): From cf2fa37e56cac5838fd8084a3b7c99a4e68818fa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 13 Jun 2020 13:18:54 +0300 Subject: [PATCH 059/223] Bump version to 4.4.0a3 --- docs | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 2178afaf..3bdbfb58 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2178afaf333874a1f6d3dff4b45d0613517049b5 +Subproject commit 3bdbfb58e293ad0e2149fc1e83ad4abba8b7aad9 diff --git a/platformio/__init__.py b/platformio/__init__.py index d373bbad..a88d3fbc 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a2") +VERSION = (4, 4, "0a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From cb70e510166968d40d371923329a10e24687a440 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 13 Jun 2020 16:21:15 +0300 Subject: [PATCH 060/223] Update changelog for Custom Targets --- HISTORY.rst | 11 +++++++++-- docs | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b725d474..1d404086 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,13 +9,20 @@ PlatformIO Core 4 4.4.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ -* New `Account Management System `__ (preview) +* New `Account Management System `__ - Manage own organizations - Manage organization teams - Manage resource access -* Added support for `custom targets `__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.) +* New `Custom Targets `__ + + - Pre/Post processing based on a dependent sources (other target, source file, etc.) + - Command launcher with own arguments + - Launch command with custom options declared in `"platformio.ini" `__ + - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) + + * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) diff --git a/docs b/docs index 3bdbfb58..8e182191 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3bdbfb58e293ad0e2149fc1e83ad4abba8b7aad9 +Subproject commit 8e1821918ff774f97f97e65c006a7bb669d48c90 From df0e6016bb255e6a3c2becacce2347308edbdd63 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 15 Jun 2020 21:25:24 +0300 Subject: [PATCH 061/223] Handle possible NodeList in source files when processing Middlewares // Resolve #3531 env.Object() returns a list of objects that breaks the processing of subsequent middlewares since we only expected File nodes. --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 1d404086..8b98626a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,6 +26,7 @@ PlatformIO Core 4 * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) +* Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c0cc11de..bb86dfc0 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -20,6 +20,7 @@ import sys from SCons import Builder, Util # pylint: disable=import-error from SCons.Node import FS # pylint: disable=import-error +from SCons.Node import NodeList # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error @@ -285,6 +286,8 @@ def CollectBuildFiles( for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []): tmp = [] for node in sources: + if isinstance(node, NodeList): + node = node[0] if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): tmp.append(node) continue From d3fd1157437e5ad4d983a49378140220585c622b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 15 Jun 2020 22:05:28 +0300 Subject: [PATCH 062/223] Black format --- platformio/commands/test/processor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index bb7c4a12..cfc0f3ca 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -13,7 +13,7 @@ # limitations under the License. import atexit -from os import remove, listdir +from os import listdir, remove from os.path import isdir, isfile, join from string import Template @@ -214,7 +214,8 @@ class TestProcessorBase(object): delete_tmptest_files(test_dir) tmp_file = join( - test_dir, "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")) + test_dir, + "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")), ) with open(tmp_file, "w") as fp: fp.write(data) From a9c13aa20e9b24e4bd8ec39c48f5958aa6a55794 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 15 Jun 2020 22:05:59 +0300 Subject: [PATCH 063/223] Implement "ManifestParserFactory.new_from_archive" API --- platformio/package/manifest/parser.py | 12 ++++++++++++ platformio/package/spec.py | 2 +- tests/package/test_manifest.py | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index bf017721..837fabd6 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -17,6 +17,7 @@ import io import json import os import re +import tarfile import requests @@ -109,6 +110,17 @@ class ManifestParserFactory(object): remote_url, ) + @staticmethod + def new_from_archive(path): + assert path.endswith("tar.gz") + with tarfile.open(path, mode="r:gz") as tf: + for t in sorted(ManifestFileType.items().values()): + try: + return ManifestParserFactory.new(tf.extractfile(t).read(), t) + except KeyError: + pass + raise UnknownManifestError("Unknown manifest file type in %s archive" % path) + @staticmethod def new( # pylint: disable=redefined-builtin contents, type, remote_url=None, package_dir=None diff --git a/platformio/package/spec.py b/platformio/package/spec.py index 0535d4ba..f031c71c 100644 --- a/platformio/package/spec.py +++ b/platformio/package/spec.py @@ -43,7 +43,7 @@ class PackageType(object): def from_archive(cls, path): assert path.endswith("tar.gz") manifest_map = cls.get_manifest_map() - with tarfile.open(path, mode="r|gz") as tf: + with tarfile.open(path, mode="r:gz") as tf: for t in sorted(cls.items().values()): for manifest in manifest_map[t]: try: diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 1ad66a75..e497c0b4 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -14,6 +14,7 @@ import os import re +import tarfile import jsondiff import pytest @@ -790,6 +791,21 @@ def test_examples_from_dir(tmpdir_factory): ) +def test_parser_from_archive(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("package.json").write('{"name": "package.json"}') + pkg_dir.join("library.json").write('{"name": "library.json"}') + pkg_dir.join("library.properties").write("name=library.properties") + + archive_path = os.path.join(str(pkg_dir), "package.tar.gz") + with tarfile.open(archive_path, mode="w|gz") as tf: + for item in os.listdir(str(pkg_dir)): + tf.add(os.path.join(str(pkg_dir), item), item) + + data = parser.ManifestParserFactory.new_from_archive(archive_path).as_dict() + assert data["name"] == "library.json" + + def test_broken_schemas(): # missing required field with pytest.raises( From 21f3dd11f48e3b4593a676e371429f0c415fff33 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 16 Jun 2020 12:27:49 +0300 Subject: [PATCH 064/223] Fix printing relative paths on Windows // Resolve #3542 Fixes "ValueError" when running "clean" target if "build_dir" points to a folder on a different logical drive --- HISTORY.rst | 1 + platformio/builder/tools/piotarget.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8b98626a..709cc9b7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO Core 4 * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) +* Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 7106a40d..a6dcaf3b 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -20,7 +20,7 @@ from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error -from platformio import fs +from platformio import compat, fs def VerboseAction(_, act, actstr): @@ -30,17 +30,24 @@ def VerboseAction(_, act, actstr): def PioClean(env, clean_dir): + def _relpath(path): + if compat.WINDOWS: + prefix = os.getcwd()[:2].lower() + if ":" not in prefix or not path.lower().startswith(prefix): + return path + return os.path.relpath(path) + if not os.path.isdir(clean_dir): print("Build environment is clean") env.Exit(0) - clean_rel_path = os.path.relpath(clean_dir) + clean_rel_path = _relpath(clean_dir) for root, _, files in os.walk(clean_dir): for f in files: dst = os.path.join(root, f) os.remove(dst) print( "Removed %s" - % (dst if clean_rel_path.startswith(".") else os.path.relpath(dst)) + % (dst if not clean_rel_path.startswith(".") else _relpath(dst)) ) print("Done cleaning") fs.rmtree(clean_dir) From cad0ae01130a45d14b3ca556f015c8dd9e6dd070 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 16 Jun 2020 15:06:04 +0300 Subject: [PATCH 065/223] Update slogan to "No more vendor lock-in!" --- README.rst | 2 ++ docs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index fe7345d5..17d15bf6 100644 --- a/README.rst +++ b/README.rst @@ -39,6 +39,8 @@ PlatformIO `PlatformIO `_ a new generation ecosystem for embedded development +**A place where Developers and Teams have true Freedom! No more vendor lock-in!** + * Open source, maximum permissive Apache 2.0 license * Cross-platform IDE and Unified Debugger * Static Code Analyzer and Remote Unit Testing diff --git a/docs b/docs index 8e182191..c07127eb 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8e1821918ff774f97f97e65c006a7bb669d48c90 +Subproject commit c07127ebc702a6558890cc43691a0daf03591ad5 From 1e90c821dcfe5f4ac2048bc43b0207b148fd5b5f Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 17 Jun 2020 00:24:55 +0300 Subject: [PATCH 066/223] Disable package upload test (#3562) --- tests/commands/test_account_org_team.py | 60 ++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index cc97b33f..b4e576f0 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -283,36 +283,36 @@ def test_account_update( validate_cliresult(result) -def test_account_destroy_with_linked_resources( - clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory -): - package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" - - tmp_dir = tmpdir_factory.mktemp("package") - fd = FileDownloader(package_url, str(tmp_dir)) - pkg_dir = tmp_dir.mkdir("raw_package") - fd.start(with_progress=False, silent=True) - with FileUnpacker(fd.get_filepath()) as unpacker: - unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) - - result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) - validate_cliresult(result) - try: - result = receive_email(email) - assert "Congrats" in result - assert "was published" in result - except: # pylint:disable=bare-except - pass - - result = clirunner.invoke(cmd_account, ["destroy"], "y") - assert result.exit_code != 0 - assert ( - "We can not destroy the %s account due to 1 linked resources from registry" - % username - ) - - result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) - validate_cliresult(result) +# def test_account_destroy_with_linked_resources( +# clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory +# ): +# package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" +# +# tmp_dir = tmpdir_factory.mktemp("package") +# fd = FileDownloader(package_url, str(tmp_dir)) +# pkg_dir = tmp_dir.mkdir("raw_package") +# fd.start(with_progress=False, silent=True) +# with FileUnpacker(fd.get_filepath()) as unpacker: +# unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) +# +# result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) +# validate_cliresult(result) +# try: +# result = receive_email(email) +# assert "Congrats" in result +# assert "was published" in result +# except: # pylint:disable=bare-except +# pass +# +# result = clirunner.invoke(cmd_account, ["destroy"], "y") +# assert result.exit_code != 0 +# assert ( +# "We can not destroy the %s account due to 1 linked resources from registry" +# % username +# ) +# +# result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) +# validate_cliresult(result) def test_org_create(clirunner, validate_cliresult, isolated_pio_home): From 42e8ea29ff677ff72e7eabefdffd88693c6d1ce9 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 17 Jun 2020 13:53:53 +0300 Subject: [PATCH 067/223] CLI to manage access level on PlatformIO resources. Resolve #3534 (#3563) --- platformio/clients/registry.py | 20 ++++ platformio/commands/access.py | 137 ++++++++++++++++++++++++ tests/commands/test_account_org_team.py | 3 - 3 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 platformio/commands/access.py diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 5936d2e9..1a3626de 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -74,3 +74,23 @@ class RegistryClient(RESTClient): "delete", path, params={"undo": 1 if undo else 0}, ) return response + + def update_resource(self, urn, private): + return self.send_auth_request( + "put", "/v3/resources/%s" % urn, data={"private": int(private)}, + ) + + def grant_access_for_resource(self, urn, client, level): + return self.send_auth_request( + "put", + "/v3/resources/%s/access" % urn, + data={"client": client, "level": level}, + ) + + def revoke_access_from_resource(self, urn, client): + return self.send_auth_request( + "delete", "/v3/resources/%s/access" % urn, data={"client": client}, + ) + + def list_own_resources(self): + return self.send_auth_request("get", "/v3/resources",) diff --git a/platformio/commands/access.py b/platformio/commands/access.py new file mode 100644 index 00000000..92efce28 --- /dev/null +++ b/platformio/commands/access.py @@ -0,0 +1,137 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json +import re + +import click +from tabulate import tabulate + +from platformio.clients.registry import RegistryClient +from platformio.commands.account import validate_username +from platformio.commands.team import validate_orgname_teamname + + +def validate_client(value): + if ":" in value: + validate_orgname_teamname(value) + else: + validate_username(value) + return value + + +@click.group("access", short_help="Manage Resource Access") +def cli(): + pass + + +def validate_urn(value): + value = str(value).strip() + if not re.match(r"^reg:pkg:(\d+)$", value, flags=re.I): + raise click.BadParameter("Invalid URN format.") + return value + + +@cli.command("public", short_help="Make resource public") +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +def access_public(urn, urn_type): + client = RegistryClient() + client.update_resource(urn=urn, private=0) + return click.secho( + "The resource %s has been successfully updated." % urn, fg="green", + ) + + +@cli.command("private", short_help="Make resource private") +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +def access_private(urn, urn_type): + client = RegistryClient() + client.update_resource(urn=urn, private=1) + return click.secho( + "The resource %s has been successfully updated." % urn, fg="green", + ) + + +@cli.command("grant", short_help="Grant access") +@click.argument("level", type=click.Choice(["admin", "maintainer", "guest"])) +@click.argument( + "client", + metavar="[ORGNAME:TEAMNAME|USERNAME]", + callback=lambda _, __, value: validate_client(value), +) +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +def access_grant(level, client, urn, urn_type): + reg_client = RegistryClient() + reg_client.grant_access_for_resource(urn=urn, client=client, level=level) + return click.secho( + "Access for resource %s has been granted for %s" % (urn, client), fg="green", + ) + + +@cli.command("revoke", short_help="Revoke access") +@click.argument( + "client", + metavar="[ORGNAME:TEAMNAME|USERNAME]", + callback=lambda _, __, value: validate_client(value), +) +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +def access_revoke(client, urn, urn_type): + reg_client = RegistryClient() + reg_client.revoke_access_from_resource(urn=urn, client=client) + return click.secho( + "Access for resource %s has been revoked for %s" % (urn, client), fg="green", + ) + + +@cli.command("list", short_help="List resources") +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--json-output", is_flag=True) +def access_list(urn_type, json_output): + reg_client = RegistryClient() + resources = reg_client.list_own_resources() + if json_output: + return click.echo(json.dumps(resources)) + if not resources: + return click.secho("You do not have any resources.", fg="yellow") + for resource in resources: + click.echo() + click.secho(resource.get("name"), fg="cyan") + click.echo("-" * len(resource.get("name"))) + table_data = [] + table_data.append(("URN:", resource.get("urn"))) + table_data.append(("Owner:", resource.get("owner"))) + table_data.append( + ( + "Access level(s):", + ", ".join( + (level.capitalize() for level in resource.get("access_levels")) + ), + ) + ) + click.echo(tabulate(table_data, tablefmt="plain")) + return click.echo() diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index b4e576f0..297e4c12 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -21,10 +21,7 @@ import requests from platformio.commands.account import cli as cmd_account from platformio.commands.org import cli as cmd_org -from platformio.commands.package import cli as cmd_package from platformio.commands.team import cli as cmd_team -from platformio.downloader import FileDownloader -from platformio.unpacker import FileUnpacker pytestmark = pytest.mark.skipif( not (os.environ.get("TEST_EMAIL_LOGIN") and os.environ.get("TEST_EMAIL_PASSWORD")), From e853d61e162017603346aada3fc068b0cda9d484 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 17 Jun 2020 18:55:40 +0300 Subject: [PATCH 068/223] Add orgname filter for access list (#3564) * add orgname filter for access list * fix * fix namings --- platformio/clients/registry.py | 6 ++++-- platformio/commands/access.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 1a3626de..c48094ee 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -92,5 +92,7 @@ class RegistryClient(RESTClient): "delete", "/v3/resources/%s/access" % urn, data={"client": client}, ) - def list_own_resources(self): - return self.send_auth_request("get", "/v3/resources",) + def list_resources(self, owner): + return self.send_auth_request( + "get", "/v3/resources", params={"owner": owner} if owner else None + ) diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 92efce28..2ca72fd6 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -109,11 +109,12 @@ def access_revoke(client, urn, urn_type): @cli.command("list", short_help="List resources") +@click.argument("owner", required=False) @click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") @click.option("--json-output", is_flag=True) -def access_list(urn_type, json_output): +def access_list(owner, urn_type, json_output): reg_client = RegistryClient() - resources = reg_client.list_own_resources() + resources = reg_client.list_resources(owner=owner) if json_output: return click.echo(json.dumps(resources)) if not resources: From 03d99657585df7860a4bdf4e455f7aca4c66c8f3 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 17 Jun 2020 23:46:50 +0300 Subject: [PATCH 069/223] Replace urn with prn (#3565) * Replace urn with prn * fix * fix text --- platformio/commands/access.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 2ca72fd6..4d801755 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -40,7 +40,7 @@ def cli(): def validate_urn(value): value = str(value).strip() - if not re.match(r"^reg:pkg:(\d+)$", value, flags=re.I): + if not re.match(r"^reg:pkg:(\d+):(\w+)$", value, flags=re.I): raise click.BadParameter("Invalid URN format.") return value @@ -49,7 +49,7 @@ def validate_urn(value): @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_public(urn, urn_type): client = RegistryClient() client.update_resource(urn=urn, private=0) @@ -62,7 +62,7 @@ def access_public(urn, urn_type): @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_private(urn, urn_type): client = RegistryClient() client.update_resource(urn=urn, private=1) @@ -81,7 +81,7 @@ def access_private(urn, urn_type): @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_grant(level, client, urn, urn_type): reg_client = RegistryClient() reg_client.grant_access_for_resource(urn=urn, client=client, level=level) @@ -99,7 +99,7 @@ def access_grant(level, client, urn, urn_type): @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_revoke(client, urn, urn_type): reg_client = RegistryClient() reg_client.revoke_access_from_resource(urn=urn, client=client) @@ -110,7 +110,7 @@ def access_revoke(client, urn, urn_type): @cli.command("list", short_help="List resources") @click.argument("owner", required=False) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="prn:reg:pkg") @click.option("--json-output", is_flag=True) def access_list(owner, urn_type, json_output): reg_client = RegistryClient() From 260c36727cc5c24c68d975150646cc8d469144ee Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Wed, 17 Jun 2020 23:56:22 +0300 Subject: [PATCH 070/223] fix pio access urn format --- platformio/commands/access.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 4d801755..6a59be7a 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -40,7 +40,7 @@ def cli(): def validate_urn(value): value = str(value).strip() - if not re.match(r"^reg:pkg:(\d+):(\w+)$", value, flags=re.I): + if not re.match(r"^prn:reg:pkg:(\d+):(\w+)$", value, flags=re.I): raise click.BadParameter("Invalid URN format.") return value @@ -110,7 +110,7 @@ def access_revoke(client, urn, urn_type): @cli.command("list", short_help="List resources") @click.argument("owner", required=False) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="prn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") @click.option("--json-output", is_flag=True) def access_list(owner, urn_type, json_output): reg_client = RegistryClient() From c20a1f24cd54a6f2d436c973d5b2b805f6fb9e7b Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 18 Jun 2020 20:36:59 +0300 Subject: [PATCH 071/223] Don't print relative paths with double-dot --- platformio/builder/tools/piotarget.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index a6dcaf3b..948776fc 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -33,7 +33,11 @@ def PioClean(env, clean_dir): def _relpath(path): if compat.WINDOWS: prefix = os.getcwd()[:2].lower() - if ":" not in prefix or not path.lower().startswith(prefix): + if ( + ":" not in prefix + or not path.lower().startswith(prefix) + or os.path.relpath(path).startswith("..") + ): return path return os.path.relpath(path) From 87d5997b46766230f655638fcb2b8cec699b16d9 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 22 Jun 2020 14:42:45 +0300 Subject: [PATCH 072/223] Add a test that ensures setUp and tearDown functions can be compiled --- tests/commands/test_test.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 38fb7eb2..4110188f 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -78,3 +78,87 @@ int main() { validate_cliresult(result) assert "Multiple ways to build" not in result.output + + +def test_setup_teardown_are_compilable(clirunner, validate_cliresult, tmpdir): + + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:embedded] +platform = ststm32 +framework = stm32cube +board = nucleo_f401re +test_transport = custom + +[env:native] +platform = native + +""" + ) + + test_dir = project_dir.mkdir("test") + test_dir.join("test_main.c").write( + """ +#include +#include + +void setUp(){ + printf("setUp called"); +} +void tearDown(){ + printf("tearDown called"); +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, 1); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); +} +""" + ) + + native_result = clirunner.invoke( + cmd_test, ["-d", str(project_dir), "-e", "native"], + ) + + test_dir.join("unittest_transport.h").write( + """ +#ifdef __cplusplus +extern "C" { +#endif + +void unittest_uart_begin(){} +void unittest_uart_putchar(char c){} +void unittest_uart_flush(){} +void unittest_uart_end(){} + +#ifdef __cplusplus +} +#endif +""" + ) + + embedded_result = clirunner.invoke( + cmd_test, + [ + "-d", + str(project_dir), + "--without-testing", + "--without-uploading", + "-e", + "embedded", + ], + ) + + validate_cliresult(native_result) + validate_cliresult(embedded_result) + + assert all(f in native_result.output for f in ("setUp called", "tearDown called")) + assert all( + "[FAILED]" not in out for out in (native_result.output, embedded_result.output) + ) From 967a856061d1f6f89625eca890e386c5ea6ee11d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 15:25:02 +0300 Subject: [PATCH 073/223] Do not allow ":" and "/" chars in a package name --- platformio/package/manifest/schema.py | 8 +++++++- tests/package/test_manifest.py | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 11d3f902..3502550a 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -149,7 +149,13 @@ class ExampleSchema(StrictSchema): class ManifestSchema(BaseSchema): # Required fields - name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + name = fields.Str( + required=True, + validate=[ + validate.Length(min=1, max=100), + validate.Regexp(r"^[^:/]+$", error="The next chars [:/] are not allowed"), + ], + ) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) # Optional fields diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index e497c0b4..73acfdaf 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -841,3 +841,7 @@ def test_broken_schemas(): version="1.2.3", ) ) + + # invalid package name + with pytest.raises(ManifestValidationError, match=("are not allowed")): + ManifestSchema().load_manifest(dict(name="C/C++ :library", version="1.2.3")) From f19491f90970186422d660e4a6b0871bd6996bd4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 17:55:02 +0300 Subject: [PATCH 074/223] Docs: Sync articles --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index c07127eb..53c8b74e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c07127ebc702a6558890cc43691a0daf03591ad5 +Subproject commit 53c8b74e709cae5e4678d41e491af118edd426ea From 9f05519ccd98e621e10e22593f7a84278bbec073 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 19:53:31 +0300 Subject: [PATCH 075/223] =?UTF-8?q?List=20available=20project=20targets=20?= =?UTF-8?q?with=20a=20new=20"platformio=20run=20=E2=80=93list-targets"=20c?= =?UTF-8?q?ommand=20//=20Resolve=20#3544?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/run/command.py | 45 ++++++++++++++++++++++++++---- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 709cc9b7..2e9af05c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 4 - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) +* List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) diff --git a/docs b/docs index 53c8b74e..e2ed4006 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 53c8b74e709cae5e4678d41e491af118edd426ea +Subproject commit e2ed4006983b5400dee022def8774b13e15466ee diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 378eaf0d..c2142723 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import operator +import os from multiprocessing import cpu_count -from os import getcwd -from os.path import isfile from time import time import click @@ -26,7 +26,7 @@ from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libde from platformio.commands.run.processor import EnvironmentProcessor from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING from platformio.project.config import ProjectConfig -from platformio.project.helpers import find_project_dir_above +from platformio.project.helpers import find_project_dir_above, load_project_ide_data # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -43,7 +43,7 @@ except NotImplementedError: @click.option( "-d", "--project-dir", - default=getcwd, + default=os.getcwd, type=click.Path( exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True ), @@ -68,6 +68,7 @@ except NotImplementedError: @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--disable-auto-clean", is_flag=True) +@click.option("--list-targets", is_flag=True) @click.pass_context def cli( ctx, @@ -80,11 +81,12 @@ def cli( silent, verbose, disable_auto_clean, + list_targets, ): app.set_session_var("custom_project_conf", project_conf) # find project directory on upper level - if isfile(project_dir): + if os.path.isfile(project_dir): project_dir = find_project_dir_above(project_dir) is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta @@ -93,6 +95,9 @@ def cli( config = ProjectConfig.get_instance(project_conf) config.validate(environment) + if list_targets: + return print_target_list(list(environment) or config.envs()) + # clean obsolete build dir if not disable_auto_clean: build_dir = config.get_optional_dir("build") @@ -261,3 +266,33 @@ def print_processing_summary(results): is_error=failed_nums, fg="red" if failed_nums else "green", ) + + +def print_target_list(envs): + tabular_data = [] + for env, data in load_project_ide_data(os.getcwd(), envs).items(): + tabular_data.extend( + sorted( + [ + ( + click.style(env, fg="cyan"), + t["group"], + click.style(t.get("name"), fg="yellow"), + t["title"], + t.get("description"), + ) + for t in data.get("targets", []) + ], + key=operator.itemgetter(1, 2), + ) + ) + tabular_data.append((None, None, None, None, None)) + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Group", "Name", "Title", "Description") + ], + ), + ) From 3aae791bee05aa82a1569a0a3b945e5db3248c25 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 20:02:43 +0300 Subject: [PATCH 076/223] Change slogan to "collaborative platform" --- README.rst | 2 +- docs | 2 +- platformio/__init__.py | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 17d15bf6..104a7deb 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ PlatformIO .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core -`PlatformIO `_ a new generation ecosystem for embedded development +`PlatformIO `_ a new generation collaborative platform for embedded development **A place where Developers and Teams have true Freedom! No more vendor lock-in!** diff --git a/docs b/docs index e2ed4006..478d089d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e2ed4006983b5400dee022def8774b13e15466ee +Subproject commit 478d089d27feb4588d0aeb55a4edd33317fe9af4 diff --git a/platformio/__init__.py b/platformio/__init__.py index a88d3fbc..95560a17 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -17,13 +17,15 @@ __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" __description__ = ( - "A new generation ecosystem for embedded development. " + "A new generation collaborative platform for embedded development. " "Cross-platform IDE and Unified Debugger. " "Static Code Analyzer and Remote Unit Testing. " "Multi-platform and Multi-architecture Build System. " "Firmware File Explorer and Memory Inspection. " - "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, " - "RISC-V, FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3" + "Professional development environment for Embedded, IoT, Arduino, CMSIS, ESP-IDF, " + "FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, STM32Cube, Zephyr RTOS, ARM, AVR, " + "Espressif (ESP8266/ESP32), FPGA, MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), " + "NXP i.MX RT, PIC32, RISC-V, STMicroelectronics (STM8/STM32), Teensy" ) __url__ = "https://platformio.org" From 5ee90f4e618aa32c5c5c39ef76ffdfac0465ec66 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 23:04:36 +0300 Subject: [PATCH 077/223] Display system-wide information using `platformio system info` command // Resolve #3521 --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/system/command.py | 49 +++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2e9af05c..1e07cb28 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 4 - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) +* Display system-wide information using `platformio system info `__ command (`issue #3521 `_) * List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment diff --git a/docs b/docs index 478d089d..f33f42cc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 478d089d27feb4588d0aeb55a4edd33317fe9af4 +Subproject commit f33f42cc9c5ec30267dc8c0c845aeda63adda598 diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 48336bfd..fed66c40 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -12,17 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import json +import platform import subprocess +import sys import click +from tabulate import tabulate -from platformio import proc +from platformio import __version__, proc, util from platformio.commands.system.completion import ( get_completion_install_path, install_completion_code, uninstall_completion_code, ) +from platformio.managers.lib import LibraryManager +from platformio.managers.package import PackageManager +from platformio.managers.platform import PlatformManager +from platformio.project.config import ProjectConfig @click.group("system", short_help="Miscellaneous system commands") @@ -30,6 +37,44 @@ def cli(): pass +@cli.command("info", short_help="Display system-wide information") +@click.option("--json-output", is_flag=True) +def system_info(json_output): + project_config = ProjectConfig() + data = {} + data["core_version"] = {"title": "PlatformIO Core", "value": __version__} + data["python_version"] = { + "title": "Python", + "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)), + } + data["system"] = {"title": "System Type", "value": util.get_systype()} + data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)} + data["core_dir"] = { + "title": "PlatformIO Core Directory", + "value": project_config.get_optional_dir("core"), + } + data["global_lib_nums"] = { + "title": "Global Libraries", + "value": len(LibraryManager().get_installed()), + } + data["dev_platform_nums"] = { + "title": "Development Platforms", + "value": len(PlatformManager().get_installed()), + } + data["package_tool_nums"] = { + "title": "Package Tools", + "value": len( + PackageManager(project_config.get_optional_dir("packages")).get_installed() + ), + } + + click.echo( + json.dumps(data) + if json_output + else tabulate([(item["title"], item["value"]) for item in data.values()]) + ) + + @cli.group("completion", short_help="Shell completion support") def completion(): # pylint: disable=import-error,import-outside-toplevel From a172a17c815e8fcbe0f8473c6bac1ea1d9714817 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 23:09:28 +0300 Subject: [PATCH 078/223] Bump version to 4.4.0a4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 95560a17..86f894d3 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a3") +VERSION = (4, 4, "0a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 164ae2bcbc1dfde2b7e2fd745553bb4a86ff431f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Jun 2020 11:20:29 +0300 Subject: [PATCH 079/223] Extend system info with Python and PIO Core executables // Issue #3521 --- platformio/commands/system/command.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index fed66c40..e5d66721 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -53,6 +53,16 @@ def system_info(json_output): "title": "PlatformIO Core Directory", "value": project_config.get_optional_dir("core"), } + data["platformio_exe"] = { + "title": "PlatformIO Core Executable", + "value": proc.where_is_program( + "platformio.exe" if proc.WINDOWS else "platformio" + ), + } + data["python_exe"] = { + "title": "Python Executable", + "value": proc.get_pythonexe_path(), + } data["global_lib_nums"] = { "title": "Global Libraries", "value": len(LibraryManager().get_installed()), From 9fb4cde2a5dac33e5897738b0e7828feefe0921f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Jun 2020 11:26:22 +0300 Subject: [PATCH 080/223] Do not generate ".travis.yml" for a new project, let the user have a choice --- HISTORY.rst | 1 + platformio/commands/project.py | 78 ---------------------------------- 2 files changed, 1 insertion(+), 78 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1e07cb28..2d9bb800 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO Core 4 * List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment +* Do not generate ".travis.yml" for a new project, let the user have a choice * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) * Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) diff --git a/platformio/commands/project.py b/platformio/commands/project.py index c83b44a7..5bd5efcb 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -149,7 +149,6 @@ def project_init( pg.generate() if is_new_project: - init_ci_conf(project_dir) init_cvs_ignore(project_dir) if silent: @@ -310,83 +309,6 @@ More information about PIO Unit Testing: ) -def init_ci_conf(project_dir): - conf_path = os.path.join(project_dir, ".travis.yml") - if os.path.isfile(conf_path): - return - with open(conf_path, "w") as fp: - fp.write( - """# Continuous Integration (CI) is the practice, in software -# engineering, of merging all developer working copies with a shared mainline -# several times a day < https://docs.platformio.org/page/ci/index.html > -# -# Documentation: -# -# * Travis CI Embedded Builds with PlatformIO -# < https://docs.travis-ci.com/user/integration/platformio/ > -# -# * PlatformIO integration with Travis CI -# < https://docs.platformio.org/page/ci/travis.html > -# -# * User Guide for `platformio ci` command -# < https://docs.platformio.org/page/userguide/cmd_ci.html > -# -# -# Please choose one of the following templates (proposed below) and uncomment -# it (remove "# " before each line) or use own configuration according to the -# Travis CI documentation (see above). -# - - -# -# Template #1: General project. Test it using existing `platformio.ini`. -# - -# language: python -# python: -# - "2.7" -# -# sudo: false -# cache: -# directories: -# - "~/.platformio" -# -# install: -# - pip install -U platformio -# - platformio update -# -# script: -# - platformio run - - -# -# Template #2: The project is intended to be used as a library with examples. -# - -# language: python -# python: -# - "2.7" -# -# sudo: false -# cache: -# directories: -# - "~/.platformio" -# -# env: -# - PLATFORMIO_CI_SRC=path/to/test/file.c -# - PLATFORMIO_CI_SRC=examples/file.ino -# - PLATFORMIO_CI_SRC=path/to/test/directory -# -# install: -# - pip install -U platformio -# - platformio update -# -# script: -# - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N -""", - ) - - def init_cvs_ignore(project_dir): conf_path = os.path.join(project_dir, ".gitignore") if os.path.isfile(conf_path): From 82735dd5718947322cc4a099b7891839b2c17297 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Jun 2020 11:46:00 +0300 Subject: [PATCH 081/223] Fixed an issue with improper processing of source files added via multiple Build Middlewares // Resolve #3531 --- platformio/builder/tools/platformio.py | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index bb86dfc0..560cbe37 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -20,7 +20,6 @@ import sys from SCons import Builder, Util # pylint: disable=import-error from SCons.Node import FS # pylint: disable=import-error -from SCons.Node import NodeList # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error @@ -283,20 +282,21 @@ def CollectBuildFiles( if fs.path_endswith_ext(item, SRC_BUILD_EXT): sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) - for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []): - tmp = [] - for node in sources: - if isinstance(node, NodeList): - node = node[0] - if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): - tmp.append(node) - continue - n = callback(node) - if n: - tmp.append(n) - sources = tmp + middlewares = env.get("__PIO_BUILD_MIDDLEWARES") + if not middlewares: + return sources - return sources + new_sources = [] + for node in sources: + new_node = node + for callback, pattern in middlewares: + if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): + continue + new_node = callback(new_node) + if new_node: + new_sources.append(new_node) + + return new_sources def AddBuildMiddleware(env, callback, pattern=None): From 5dadb8749e86e97d429927e0c82add4455acea4a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Jun 2020 12:33:00 +0300 Subject: [PATCH 082/223] Change slogan to "PlatformIO is a professional collaborative platform for embedded development" --- HISTORY.rst | 3 ++- README.rst | 2 +- docs | 2 +- platformio/__init__.py | 10 +++++----- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2d9bb800..cea8fb1b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ PlatformIO Core 4 4.4.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +**A professional collaborative platform for embedded development** + * New `Account Management System `__ - Manage own organizations @@ -22,7 +24,6 @@ PlatformIO Core 4 - Launch command with custom options declared in `"platformio.ini" `__ - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) - * Display system-wide information using `platformio system info `__ command (`issue #3521 `_) * List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. diff --git a/README.rst b/README.rst index 104a7deb..fcff06c8 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ PlatformIO .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core -`PlatformIO `_ a new generation collaborative platform for embedded development +`PlatformIO `_ is a professional collaborative platform for embedded development **A place where Developers and Teams have true Freedom! No more vendor lock-in!** diff --git a/docs b/docs index f33f42cc..2c2dce47 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f33f42cc9c5ec30267dc8c0c845aeda63adda598 +Subproject commit 2c2dce47ab2ee3eb20893d6fb8e01a34fef5c5ee diff --git a/platformio/__init__.py b/platformio/__init__.py index 86f894d3..7ad284a1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -17,15 +17,15 @@ __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" __description__ = ( - "A new generation collaborative platform for embedded development. " + "A professional collaborative platform for embedded development. " "Cross-platform IDE and Unified Debugger. " "Static Code Analyzer and Remote Unit Testing. " "Multi-platform and Multi-architecture Build System. " "Firmware File Explorer and Memory Inspection. " - "Professional development environment for Embedded, IoT, Arduino, CMSIS, ESP-IDF, " - "FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, STM32Cube, Zephyr RTOS, ARM, AVR, " - "Espressif (ESP8266/ESP32), FPGA, MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), " - "NXP i.MX RT, PIC32, RISC-V, STMicroelectronics (STM8/STM32), Teensy" + "IoT, Arduino, CMSIS, ESP-IDF, FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, " + "STM32Cube, Zephyr RTOS, ARM, AVR, Espressif (ESP8266/ESP32), FPGA, " + "MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), NXP i.MX RT, PIC32, RISC-V, " + "STMicroelectronics (STM8/STM32), Teensy" ) __url__ = "https://platformio.org" From efc2242046b1bbe4d3644ac687234f9b67fa19d3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Jun 2020 14:51:53 +0300 Subject: [PATCH 083/223] Remove empty data from board information --- platformio/managers/platform.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index c667a956..5fa38a43 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -865,7 +865,7 @@ class PlatformBoardConfig(object): return self._manifest def get_brief_data(self): - return { + result = { "id": self.id, "name": self._manifest["name"], "platform": self._manifest.get("platform"), @@ -881,12 +881,16 @@ class PlatformBoardConfig(object): ), "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), "rom": self._manifest.get("upload", {}).get("maximum_size", 0), - "connectivity": self._manifest.get("connectivity"), "frameworks": self._manifest.get("frameworks"), - "debug": self.get_debug_data(), "vendor": self._manifest["vendor"], "url": self._manifest["url"], } + if self._manifest.get("connectivity"): + result["connectivity"] = self._manifest.get("connectivity") + debug = self.get_debug_data() + if debug: + result["debug"] = debug + return result def get_debug_data(self): if not self._manifest.get("debug", {}).get("tools"): @@ -895,7 +899,7 @@ class PlatformBoardConfig(object): for name, options in self._manifest["debug"]["tools"].items(): tools[name] = {} for key, value in options.items(): - if key in ("default", "onboard"): + if key in ("default", "onboard") and value: tools[name][key] = value return {"tools": tools} From 7bc22353cc8b64ac70380b68afb45b8151c7ffd5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Jun 2020 18:04:04 +0300 Subject: [PATCH 084/223] Docs: Sync dev-platforms --- docs | 2 +- examples | 2 +- platformio/managers/platform.py | 2 +- scripts/docspregen.py | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs b/docs index 2c2dce47..438d910c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2c2dce47ab2ee3eb20893d6fb8e01a34fef5c5ee +Subproject commit 438d910cc4d35337df90a6b23955f89a0675b52c diff --git a/examples b/examples index bcdcf466..2fdead5d 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit bcdcf46691c1fe79e8cb2cc74e1a156f6bac5e90 +Subproject commit 2fdead5df5f146c6f7c70eeef63669bebab7a225 diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 5fa38a43..52842ca0 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -609,7 +609,7 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): @property def vendor_url(self): - return self._manifest.get("url") + return self._manifest.get("homepage") @property def docs_url(self): diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 3628f609..3c498309 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -88,12 +88,12 @@ def generate_boards_table(boards, skip_columns=None): lines.append(prefix + name) for data in sorted(boards, key=lambda item: item['name']): - has_onboard_debug = (data['debug'] and any( + has_onboard_debug = (data.get('debug') and any( t.get("onboard") for (_, t) in data['debug']['tools'].items())) debug = "No" if has_onboard_debug: debug = "On-board" - elif data['debug']: + elif data.get('debug'): debug = "External" variables = dict(id=data['id'], @@ -170,11 +170,11 @@ def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): skip_board_columns.append("Debug") lines = [] onboard_debug = [ - b for b in boards if b['debug'] and any( + b for b in boards if b.get('debug') and any( t.get("onboard") for (_, t) in b['debug']['tools'].items()) ] external_debug = [ - b for b in boards if b['debug'] and b not in onboard_debug + b for b in boards if b.get('debug') and b not in onboard_debug ] if not onboard_debug and not external_debug: return lines @@ -723,7 +723,7 @@ You can change upload protocol using :ref:`projectconf_upload_protocol` option: # lines.append("Debugging") lines.append("---------") - if not board['debug']: + if not board.get('debug'): lines.append( ":ref:`piodebug` currently does not support {name} board.".format( **variables)) @@ -781,7 +781,7 @@ def update_debugging(): platforms = [] frameworks = [] for data in BOARDS: - if not data['debug']: + if not data.get('debug'): continue for tool in data['debug']['tools']: From a1ec3e0a22179c9b743ae76cb04454794fb29191 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Jun 2020 23:23:55 +0300 Subject: [PATCH 085/223] Remove "vendor_url" and "docs_url" from Platform API --- platformio/commands/platform.py | 3 +-- platformio/managers/platform.py | 8 -------- scripts/docspregen.py | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index c4a9ca5d..deaeb431 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -76,9 +76,8 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru description=p.description, version=p.version, homepage=p.homepage, + url=p.homepage, repository=p.repository_url, - url=p.vendor_url, - docs=p.docs_url, license=p.license, forDesktop=not p.is_embedded(), frameworks=sorted(list(p.frameworks) if p.frameworks else []), diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 52842ca0..ada4f4ac 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -607,14 +607,6 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): def homepage(self): return self._manifest.get("homepage") - @property - def vendor_url(self): - return self._manifest.get("homepage") - - @property - def docs_url(self): - return self._manifest.get("docs") - @property def repository_url(self): return self._manifest.get("repository", {}).get("url") diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 3c498309..f2dc2757 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -316,7 +316,7 @@ def generate_platform(name, rst_dir): lines.append(p.description) lines.append(""" For more detailed information please visit `vendor site <%s>`_.""" % - campaign_url(p.vendor_url)) + campaign_url(p.homepage)) lines.append(""" .. contents:: Contents :local: From 0bec1f1585b89508727f8151f070885257eef1f3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 26 Jun 2020 18:38:17 +0300 Subject: [PATCH 086/223] Extend system info with "file system" and "locale" encodings --- platformio/commands/system/command.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index e5d66721..a684e14a 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -20,7 +20,7 @@ import sys import click from tabulate import tabulate -from platformio import __version__, proc, util +from platformio import __version__, compat, proc, util from platformio.commands.system.completion import ( get_completion_install_path, install_completion_code, @@ -49,6 +49,14 @@ def system_info(json_output): } data["system"] = {"title": "System Type", "value": util.get_systype()} data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)} + data["filesystem_encoding"] = { + "title": "File System Encoding", + "value": compat.get_filesystem_encoding(), + } + data["locale_encoding"] = { + "title": "Locale Encoding", + "value": compat.get_locale_encoding(), + } data["core_dir"] = { "title": "PlatformIO Core Directory", "value": project_config.get_optional_dir("core"), From bc2eb0d79f45f2ede4ed38e1a5e35ab83d993024 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 26 Jun 2020 19:49:25 +0300 Subject: [PATCH 087/223] Parse dev-platform keywords --- platformio/package/manifest/parser.py | 34 ++++++++++++++------------- tests/package/test_manifest.py | 7 +++--- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 837fabd6..720d4ea3 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -159,6 +159,21 @@ class BaseManifestParser(object): def as_dict(self): return self._data + @staticmethod + def str_to_list(value, sep=",", lowercase=True): + if isinstance(value, string_types): + value = value.split(sep) + assert isinstance(value, list) + result = [] + for item in value: + item = item.strip() + if not item: + continue + if lowercase: + item = item.lower() + result.append(item) + return result + @staticmethod def normalize_author(author): assert isinstance(author, dict) @@ -296,7 +311,7 @@ class LibraryJsonManifestParser(BaseManifestParser): # normalize Union[str, list] fields for k in ("keywords", "platforms", "frameworks"): if k in data: - data[k] = self._str_to_list(data[k], sep=",") + data[k] = self.str_to_list(data[k], sep=",") if "authors" in data: data["authors"] = self._parse_authors(data["authors"]) @@ -309,21 +324,6 @@ class LibraryJsonManifestParser(BaseManifestParser): return data - @staticmethod - def _str_to_list(value, sep=",", lowercase=True): - if isinstance(value, string_types): - value = value.split(sep) - assert isinstance(value, list) - result = [] - for item in value: - item = item.strip() - if not item: - continue - if lowercase: - item = item.lower() - result.append(item) - return result - @staticmethod def _process_renamed_fields(data): if "url" in data: @@ -617,6 +617,8 @@ class PlatformJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") if "frameworks" in data: data["frameworks"] = self._parse_frameworks(data["frameworks"]) if "packages" in data: diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 73acfdaf..13dff94e 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -544,8 +544,8 @@ def test_platform_json_schema(): "name": "atmelavr", "title": "Atmel AVR", "description": "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of performance, power efficiency and design flexibility. Optimized to speed time to market-and easily adapt to new ones-they are based on the industrys most code-efficient architecture for C and assembly programming.", - "url": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", - "homepage": "http://platformio.org/platforms/atmelavr", + "keywords": "arduino, atmel, avr", + "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "engines": { "platformio": "<5" @@ -603,7 +603,8 @@ def test_platform_json_schema(): "on the industrys most code-efficient architecture for C and " "assembly programming." ), - "homepage": "http://platformio.org/platforms/atmelavr", + "keywords": ["arduino", "atmel", "avr"], + "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "repository": { "url": "https://github.com/platformio/platform-atmelavr.git", From 29fb803be1559a7dc85a5cb847375a7db0d09ea7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Jun 2020 12:36:57 +0300 Subject: [PATCH 088/223] Enable PIO Core tests on Python 3.8 --- .github/workflows/core.yml | 2 +- platformio/commands/project.py | 3 +-- tests/commands/test_account_org_team.py | 36 ++++++++++++------------- tests/commands/test_check.py | 2 +- tests/commands/test_ci.py | 14 ++++++---- tests/commands/test_lib.py | 16 +++++------ tests/commands/test_platform.py | 22 +++++++-------- tests/commands/test_test.py | 8 +++--- tests/conftest.py | 23 ++++++++++++---- tests/package/test_manifest.py | 3 ++- tests/test_builder.py | 2 +- tests/test_ino2cpp.py | 4 +-- tests/test_maintenance.py | 12 ++++----- tests/test_managers.py | 4 +-- tests/test_misc.py | 4 +-- 15 files changed, 87 insertions(+), 68 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c6e15bfd..e0aed6a9 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [2.7, 3.7] + python-version: [2.7, 3.7, 3.8] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 5bd5efcb..27e33455 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-arguments,too-many-locals, too-many-branches +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,line-too-long import os @@ -238,7 +238,6 @@ https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html def init_lib_readme(lib_dir): - # pylint: disable=line-too-long with open(os.path.join(lib_dir, "README"), "w") as fp: fp.write( """ diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index 297e4c12..b6d5b6d4 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -62,7 +62,7 @@ def test_prepare(): def test_account_register( - clirunner, validate_cliresult, receive_email, isolated_pio_home + clirunner, validate_cliresult, receive_email, isolated_pio_core ): result = clirunner.invoke( cmd_account, @@ -97,14 +97,14 @@ def test_account_register( def test_account_login( - clirunner, validate_cliresult, isolated_pio_home, + clirunner, validate_cliresult, isolated_pio_core, ): result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) validate_cliresult(result) def test_account_summary( - clirunner, validate_cliresult, isolated_pio_home, + clirunner, validate_cliresult, isolated_pio_core, ): result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) validate_cliresult(result) @@ -156,7 +156,7 @@ def test_account_summary( assert json_result.get("subscriptions") is not None -def test_account_token(clirunner, validate_cliresult, isolated_pio_home): +def test_account_token(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_account, ["token", "--password", password,],) validate_cliresult(result) assert "Personal Authentication Token:" in result.output @@ -198,7 +198,7 @@ def test_account_token(clirunner, validate_cliresult, isolated_pio_home): validate_cliresult(result) -def test_account_change_password(clirunner, validate_cliresult, isolated_pio_home): +def test_account_change_password(clirunner, validate_cliresult, isolated_pio_core): new_password = "Testpassword123" result = clirunner.invoke( cmd_account, @@ -222,7 +222,7 @@ def test_account_change_password(clirunner, validate_cliresult, isolated_pio_hom def test_account_update( - clirunner, validate_cliresult, receive_email, isolated_pio_home + clirunner, validate_cliresult, receive_email, isolated_pio_core ): global username global email @@ -281,7 +281,7 @@ def test_account_update( # def test_account_destroy_with_linked_resources( -# clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory +# clirunner, validate_cliresult, receive_email, isolated_pio_core, tmpdir_factory # ): # package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" # @@ -312,14 +312,14 @@ def test_account_update( # validate_cliresult(result) -def test_org_create(clirunner, validate_cliresult, isolated_pio_home): +def test_org_create(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_org, ["create", "--email", email, "--displayname", display_name, orgname], ) validate_cliresult(result) -def test_org_list(clirunner, validate_cliresult, isolated_pio_home): +def test_org_list(clirunner, validate_cliresult, isolated_pio_core): # pio org list result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) @@ -336,7 +336,7 @@ def test_org_list(clirunner, validate_cliresult, isolated_pio_home): ] -def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_home): +def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_org, ["add", orgname, second_username]) validate_cliresult(result) @@ -345,7 +345,7 @@ def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_home): assert second_username in result.output -def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_home): +def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_org, ["remove", orgname, second_username]) validate_cliresult(result) @@ -354,7 +354,7 @@ def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_home): assert second_username not in result.output -def test_org_update(clirunner, validate_cliresult, isolated_pio_home): +def test_org_update(clirunner, validate_cliresult, isolated_pio_core): new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) new_display_name = "Test Org for PIO Core" @@ -399,7 +399,7 @@ def test_org_update(clirunner, validate_cliresult, isolated_pio_home): validate_cliresult(result) -def test_team_create(clirunner, validate_cliresult, isolated_pio_home): +def test_team_create(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["create", "%s:%s" % (orgname, teamname), "--description", team_description,], @@ -407,7 +407,7 @@ def test_team_create(clirunner, validate_cliresult, isolated_pio_home): validate_cliresult(result) -def test_team_list(clirunner, validate_cliresult, isolated_pio_home): +def test_team_list(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) @@ -418,7 +418,7 @@ def test_team_list(clirunner, validate_cliresult, isolated_pio_home): ] -def test_team_add_member(clirunner, validate_cliresult, isolated_pio_home): +def test_team_add_member(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username], ) @@ -429,7 +429,7 @@ def test_team_add_member(clirunner, validate_cliresult, isolated_pio_home): assert second_username in result.output -def test_team_remove(clirunner, validate_cliresult, isolated_pio_home): +def test_team_remove(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username], ) @@ -440,7 +440,7 @@ def test_team_remove(clirunner, validate_cliresult, isolated_pio_home): assert second_username not in result.output -def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_home): +def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_core): new_teamname = "new-" + str(random.randint(0, 100000)) newteam_description = "Updated Description" result = clirunner.invoke( @@ -479,7 +479,7 @@ def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_ validate_cliresult(result) -def test_cleanup(clirunner, validate_cliresult, receive_email, isolated_pio_home): +def test_cleanup(clirunner, validate_cliresult, receive_email, isolated_pio_core): result = clirunner.invoke(cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y") validate_cliresult(result) result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index c5125b1e..94064bda 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -357,7 +357,7 @@ int main() { assert low_result.exit_code != 0 -def test_check_pvs_studio_free_license(clirunner, tmpdir): +def test_check_pvs_studio_free_license(clirunner, isolated_pio_core, tmpdir): config = """ [env:test] platform = teensy diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 0f7aceed..8a597413 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -24,7 +24,7 @@ def test_ci_empty(clirunner): assert "Invalid value: Missing argument 'src'" in result.output -def test_ci_boards(clirunner, validate_cliresult): +def test_ci_boards(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_ci, [ @@ -38,7 +38,7 @@ def test_ci_boards(clirunner, validate_cliresult): validate_cliresult(result) -def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): +def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, @@ -54,7 +54,9 @@ def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): assert not isfile(join(build_dir, "platformio.ini")) -def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): +def test_ci_keep_build_dir( + clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core +): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, @@ -88,7 +90,7 @@ def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): assert "board: metro" in result.output -def test_ci_project_conf(clirunner, validate_cliresult): +def test_ci_project_conf(clirunner, validate_cliresult, isolated_pio_core): project_dir = join("examples", "wiring-blink") result = clirunner.invoke( cmd_ci, @@ -102,7 +104,9 @@ def test_ci_project_conf(clirunner, validate_cliresult): assert "uno" in result.output -def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): +def test_ci_lib_and_board( + clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core +): storage_dir = str(tmpdir_factory.mktemp("lib")) result = clirunner.invoke( cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"] diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 752c2c30..3341ef71 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -34,7 +34,7 @@ def test_search(clirunner, validate_cliresult): assert int(match.group(1)) > 1 -def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_home): +def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ @@ -54,7 +54,7 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_hom assert result.exit_code != 0 assert isinstance(result.exception, exception.LibNotFound) - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "ArduinoJson_ID64", "ArduinoJson_ID64@5.10.1", @@ -68,7 +68,7 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_hom assert set(items1) == set(items2) -def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home): +def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ @@ -93,12 +93,12 @@ def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home ) assert result.exit_code != 0 - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire_ID1", "ESP32WebServer"] assert set(items1) >= set(items2) -def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_home): +def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ @@ -114,7 +114,7 @@ def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_h ], ) validate_cliresult(result) - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", @@ -260,7 +260,7 @@ def test_global_lib_update(clirunner, validate_cliresult): assert isinstance(result.exception, exception.UnknownPackage) -def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): +def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): # uninstall using package directory result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) validate_cliresult(result) @@ -284,7 +284,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): ) validate_cliresult(result) - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "rs485-nodeproto", "platformio-libmirror", diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index b2db5d83..377c1e28 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -18,7 +18,7 @@ from platformio import exception from platformio.commands import platform as cli_platform -def test_search_json_output(clirunner, validate_cliresult, isolated_pio_home): +def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_search, ["arduino", "--json-output"] ) @@ -48,7 +48,7 @@ def test_install_unknown_from_registry(clirunner): assert isinstance(result.exception, exception.UnknownPackage) -def test_install_known_version(clirunner, validate_cliresult, isolated_pio_home): +def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package", "--with-package", "tool-avrdude"], @@ -56,10 +56,10 @@ def test_install_known_version(clirunner, validate_cliresult, isolated_pio_home) validate_cliresult(result) assert "atmelavr @ 1.2.0" in result.output assert "Installing tool-avrdude @" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 -def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): +def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, [ @@ -69,7 +69,7 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): ) validate_cliresult(result) assert "espressif8266" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 def test_list_json_output(clirunner, validate_cliresult): @@ -88,7 +88,7 @@ def test_list_raw_output(clirunner, validate_cliresult): assert all([s in result.output for s in ("atmelavr", "espressif8266")]) -def test_update_check(clirunner, validate_cliresult, isolated_pio_home): +def test_update_check(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_update, ["--only-check", "--json-output"] ) @@ -96,20 +96,20 @@ def test_update_check(clirunner, validate_cliresult, isolated_pio_home): output = json.loads(result.output) assert len(output) == 1 assert output[0]["name"] == "atmelavr" - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 -def test_update_raw(clirunner, validate_cliresult, isolated_pio_home): +def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cli_platform.platform_update) validate_cliresult(result) assert "Uninstalling atmelavr @ 1.2.0:" in result.output assert "PlatformManager: Installing atmelavr @" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 -def test_uninstall(clirunner, validate_cliresult, isolated_pio_home): +def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_uninstall, ["atmelavr", "espressif8266"] ) validate_cliresult(result) - assert not isolated_pio_home.join("platforms").listdir() + assert not isolated_pio_core.join("platforms").listdir() diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 4110188f..608b0310 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -20,7 +20,7 @@ from platformio import util from platformio.commands.test.command import cli as cmd_test -def test_local_env(): +def test_local_env(isolated_pio_core): result = util.exec_command( [ "platformio", @@ -38,7 +38,7 @@ def test_local_env(): ] -def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): +def test_multiple_env_build(clirunner, validate_cliresult, isolated_pio_core, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write( @@ -80,7 +80,9 @@ int main() { assert "Multiple ways to build" not in result.output -def test_setup_teardown_are_compilable(clirunner, validate_cliresult, tmpdir): +def test_setup_teardown_are_compilable( + clirunner, validate_cliresult, isolated_pio_core, tmpdir +): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write( diff --git a/tests/conftest.py b/tests/conftest.py index eda52184..0978b2ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,15 +42,28 @@ def clirunner(): @pytest.fixture(scope="module") -def isolated_pio_home(request, tmpdir_factory): - home_dir = tmpdir_factory.mktemp(".platformio") - os.environ["PLATFORMIO_CORE_DIR"] = str(home_dir) +def isolated_pio_core(request, tmpdir_factory): + core_dir = tmpdir_factory.mktemp(".platformio") + backup_env_vars = { + "PLATFORMIO_CORE_DIR": {"new": str(core_dir)}, + "PLATFORMIO_WORKSPACE_DIR": {"new": None}, + } + for key, item in backup_env_vars.items(): + backup_env_vars[key]["old"] = os.environ.get(key) + if item["new"] is not None: + os.environ[key] = item["new"] + elif key in os.environ: + del os.environ[key] def fin(): - del os.environ["PLATFORMIO_CORE_DIR"] + for key, item in backup_env_vars.items(): + if item["old"] is None: + del os.environ[key] + else: + os.environ[key] = item["old"] request.addfinalizer(fin) - return home_dir + return core_dir @pytest.fixture(scope="function") diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 13dff94e..cef5900d 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -83,7 +83,7 @@ def test_library_json_parser(): }, "dependencies": [ {"name": "deps1", "version": "1.0.0"}, - {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino, espidf"}, + {"name": "@owner/deps2", "version": "1.0.0", "platforms": "atmelavr, espressif32", "frameworks": "arduino, espidf"}, {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } @@ -101,6 +101,7 @@ def test_library_json_parser(): { "name": "@owner/deps2", "version": "1.0.0", + "platforms": ["atmelavr", "espressif32"], "frameworks": ["arduino", "espidf"], }, {"name": "deps1", "version": "1.0.0"}, diff --git a/tests/test_builder.py b/tests/test_builder.py index f220e50c..dd0fe1f2 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -15,7 +15,7 @@ from platformio.commands.run.command import cli as cmd_run -def test_build_flags(clirunner, validate_cliresult, tmpdir): +def test_build_flags(clirunner, isolated_pio_core, validate_cliresult, tmpdir): build_flags = [ ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), diff --git a/tests/test_ino2cpp.py b/tests/test_ino2cpp.py index d1434df7..af91da8d 100644 --- a/tests/test_ino2cpp.py +++ b/tests/test_ino2cpp.py @@ -31,12 +31,12 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("piotest_dir", test_dirs) -def test_example(clirunner, validate_cliresult, piotest_dir): +def test_example(clirunner, isolated_pio_core, validate_cliresult, piotest_dir): result = clirunner.invoke(cmd_ci, [piotest_dir, "-b", "uno"]) validate_cliresult(result) -def test_warning_line(clirunner, validate_cliresult): +def test_warning_line(clirunner, isolated_pio_core, validate_cliresult): result = clirunner.invoke(cmd_ci, [join(INOTEST_DIR, "basic"), "-b", "uno"]) validate_cliresult(result) assert 'basic.ino:16:14: warning: #warning "Line number is 16"' in result.output diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index c004f28f..34d4ce68 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -22,7 +22,7 @@ from platformio.commands import upgrade as cmd_upgrade from platformio.managers.platform import PlatformManager -def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): +def test_check_pio_upgrade(clirunner, isolated_pio_core, validate_cliresult): def _patch_pio_version(version): maintenance.__version__ = version cmd_upgrade.VERSION = version.split(".", 3) @@ -51,7 +51,7 @@ def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): _patch_pio_version(origin_version) -def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): +def test_check_lib_updates(clirunner, isolated_pio_core, validate_cliresult): # install obsolete library result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<6.13"]) validate_cliresult(result) @@ -66,7 +66,7 @@ def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): assert "There are the new updates for libraries (ArduinoJson)" in result.output -def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_cliresult): +def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_libraries", "Yes"] @@ -96,11 +96,11 @@ def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_clire assert prev_data[0]["version"] != json.loads(result.output)[0]["version"] -def test_check_platform_updates(clirunner, isolated_pio_home, validate_cliresult): +def test_check_platform_updates(clirunner, isolated_pio_core, validate_cliresult): # install obsolete platform result = clirunner.invoke(cli_pio, ["platform", "install", "native"]) validate_cliresult(result) - manifest_path = isolated_pio_home.join("platforms", "native", "platform.json") + manifest_path = isolated_pio_core.join("platforms", "native", "platform.json") manifest = json.loads(manifest_path.read()) manifest["version"] = "0.0.0" manifest_path.write(json.dumps(manifest)) @@ -117,7 +117,7 @@ def test_check_platform_updates(clirunner, isolated_pio_home, validate_cliresult assert "There are the new updates for platforms (native)" in result.output -def test_check_and_update_platforms(clirunner, isolated_pio_home, validate_cliresult): +def test_check_and_update_platforms(clirunner, isolated_pio_core, validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_platforms", "Yes"] diff --git a/tests/test_managers.py b/tests/test_managers.py index f4ab2ed8..308523cd 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -165,7 +165,7 @@ def test_pkg_input_parser(): assert PackageManager.parse_pkg_uri(params) == result -def test_install_packages(isolated_pio_home, tmpdir): +def test_install_packages(isolated_pio_core, tmpdir): packages = [ dict(id=1, name="name_1", version="shasum"), dict(id=1, name="name_1", version="2.0.0"), @@ -198,7 +198,7 @@ def test_install_packages(isolated_pio_home, tmpdir): "name_2@src-f863b537ab00f4c7b5011fc44b120e1f", ] assert set( - [p.basename for p in isolated_pio_home.join("packages").listdir()] + [p.basename for p in isolated_pio_core.join("packages").listdir()] ) == set(pkg_dirnames) diff --git a/tests/test_misc.py b/tests/test_misc.py index aee9f113..d01cc46d 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -29,12 +29,12 @@ def test_ping_internet_ips(): requests.get("http://%s" % host, allow_redirects=False, timeout=2) -def test_api_internet_offline(without_internet, isolated_pio_home): +def test_api_internet_offline(without_internet, isolated_pio_core): with pytest.raises(exception.InternetIsOffline): util.get_api_result("/stats") -def test_api_cache(monkeypatch, isolated_pio_home): +def test_api_cache(monkeypatch, isolated_pio_core): api_kwargs = {"url": "/stats", "cache_valid": "10s"} result = util.get_api_result(**api_kwargs) assert result and "boards" in result From b046f21e0de3ddd9d933e143606a1cbda188a57d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Jun 2020 12:46:04 +0300 Subject: [PATCH 089/223] Fix "RuntimeError: dictionary keys changed during iteration" when parsing "library.json" dependencies --- platformio/package/manifest/parser.py | 2 -- tests/package/test_manifest.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 720d4ea3..193fbde1 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -384,8 +384,6 @@ class LibraryJsonManifestParser(BaseManifestParser): for k, v in dependency.items(): if k not in ("platforms", "frameworks", "authors"): continue - if "*" in v: - del raw[i][k] raw[i][k] = util.items_to_list(v) else: raw[i] = {"name": dependency} diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index cef5900d..332cb7b5 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -83,7 +83,7 @@ def test_library_json_parser(): }, "dependencies": [ {"name": "deps1", "version": "1.0.0"}, - {"name": "@owner/deps2", "version": "1.0.0", "platforms": "atmelavr, espressif32", "frameworks": "arduino, espidf"}, + {"name": "@owner/deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"}, {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } @@ -101,7 +101,7 @@ def test_library_json_parser(): { "name": "@owner/deps2", "version": "1.0.0", - "platforms": ["atmelavr", "espressif32"], + "platforms": ["*"], "frameworks": ["arduino", "espidf"], }, {"name": "deps1", "version": "1.0.0"}, From dd18abcac36721b0ba4d6c031af81720064aaf84 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Jun 2020 12:59:12 +0300 Subject: [PATCH 090/223] Fix tests --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0978b2ef..438ced13 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,10 +57,10 @@ def isolated_pio_core(request, tmpdir_factory): def fin(): for key, item in backup_env_vars.items(): - if item["old"] is None: - del os.environ[key] - else: + if item["old"] is not None: os.environ[key] = item["old"] + elif key in os.environ: + del os.environ[key] request.addfinalizer(fin) return core_dir From e9a15b4e9bdd03934ebd3a376f4ec0749c1be3c4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Jun 2020 21:42:13 +0300 Subject: [PATCH 091/223] Parse package.json manifest keywords --- platformio/package/manifest/parser.py | 2 ++ tests/package/test_manifest.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 193fbde1..86c98587 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -641,6 +641,8 @@ class PackageJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") data = self._parse_system(data) data = self._parse_homepage(data) return data diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 332cb7b5..0c89e012 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -627,6 +627,7 @@ def test_package_json_schema(): { "name": "tool-scons", "description": "SCons software construction tool", + "keywords": "SCons, build", "url": "http://www.scons.org", "version": "3.30101.0" } @@ -642,6 +643,7 @@ def test_package_json_schema(): { "name": "tool-scons", "description": "SCons software construction tool", + "keywords": ["scons", "build"], "homepage": "http://www.scons.org", "version": "3.30101.0", }, From 2b8aebbdf987339352518e38de09825292e7450d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 29 Jun 2020 15:06:21 +0300 Subject: [PATCH 092/223] Extend test for parsing package manifest when "system" is used as a list --- tests/package/test_manifest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 0c89e012..884d6af0 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -628,7 +628,8 @@ def test_package_json_schema(): "name": "tool-scons", "description": "SCons software construction tool", "keywords": "SCons, build", - "url": "http://www.scons.org", + "homepage": "http://www.scons.org", + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], "version": "3.30101.0" } """ @@ -645,6 +646,7 @@ def test_package_json_schema(): "description": "SCons software construction tool", "keywords": ["scons", "build"], "homepage": "http://www.scons.org", + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], "version": "3.30101.0", }, ) From 4cbad399f787211e2d13ba9e83757cd969a596a6 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 29 Jun 2020 19:22:22 +0300 Subject: [PATCH 093/223] Remove mbed framework from several tests --- tests/commands/test_check.py | 2 +- tests/commands/test_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 94064bda..116e4a52 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -412,7 +412,7 @@ int main() { """ ) - frameworks = ["arduino", "mbed", "stm32cube"] + frameworks = ["arduino", "stm32cube"] if sys.version_info[0] == 3: # Zephyr only supports Python 3 frameworks.append("zephyr") diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 608b0310..04a531ed 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -45,7 +45,7 @@ def test_multiple_env_build(clirunner, validate_cliresult, isolated_pio_core, tm """ [env:teensy31] platform = teensy -framework = mbed +framework = arduino board = teensy31 [env:native] From 1ac6c5033402ea6756d9be03c1004ec7224eb186 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 29 Jun 2020 20:52:15 +0300 Subject: [PATCH 094/223] Update multi-environment test for PIO test command --- tests/commands/test_test.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 04a531ed..2b6dc348 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import join +import os +import sys import pytest from platformio import util from platformio.commands.test.command import cli as cmd_test +from platformio.compat import WINDOWS def test_local_env(isolated_pio_core): @@ -26,7 +28,7 @@ def test_local_env(isolated_pio_core): "platformio", "test", "-d", - join("examples", "unit-testing", "calculator"), + os.path.join("examples", "unit-testing", "calculator"), "-e", "native", ] @@ -51,24 +53,27 @@ board = teensy31 [env:native] platform = native -[env:espressif32] -platform = espressif32 +[env:espressif8266] +platform = espressif8266 framework = arduino -board = esp32dev +board = nodemcuv2 """ ) project_dir.mkdir("test").join("test_main.cpp").write( """ +#include #ifdef ARDUINO -void setup() {} -void loop() {} +void setup() #else -int main() { +int main() +#endif +{ UNITY_BEGIN(); UNITY_END(); + } -#endif +void loop() {} """ ) From 5cdca9d49045b2c8b8513994571dc8e5ef583d99 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 29 Jun 2020 21:14:34 +0300 Subject: [PATCH 095/223] Optimize tests --- tests/commands/test_ci.py | 14 +++++--------- tests/commands/test_test.py | 10 +++------- tests/conftest.py | 24 +++++++++++++++--------- tests/test_builder.py | 2 +- tests/test_ino2cpp.py | 4 ++-- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 8a597413..0f7aceed 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -24,7 +24,7 @@ def test_ci_empty(clirunner): assert "Invalid value: Missing argument 'src'" in result.output -def test_ci_boards(clirunner, validate_cliresult, isolated_pio_core): +def test_ci_boards(clirunner, validate_cliresult): result = clirunner.invoke( cmd_ci, [ @@ -38,7 +38,7 @@ def test_ci_boards(clirunner, validate_cliresult, isolated_pio_core): validate_cliresult(result) -def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core): +def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, @@ -54,9 +54,7 @@ def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult, isolated_pi assert not isfile(join(build_dir, "platformio.ini")) -def test_ci_keep_build_dir( - clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core -): +def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, @@ -90,7 +88,7 @@ def test_ci_keep_build_dir( assert "board: metro" in result.output -def test_ci_project_conf(clirunner, validate_cliresult, isolated_pio_core): +def test_ci_project_conf(clirunner, validate_cliresult): project_dir = join("examples", "wiring-blink") result = clirunner.invoke( cmd_ci, @@ -104,9 +102,7 @@ def test_ci_project_conf(clirunner, validate_cliresult, isolated_pio_core): assert "uno" in result.output -def test_ci_lib_and_board( - clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core -): +def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): storage_dir = str(tmpdir_factory.mktemp("lib")) result = clirunner.invoke( cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"] diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 2b6dc348..16e0556c 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -13,16 +13,14 @@ # limitations under the License. import os -import sys import pytest from platformio import util from platformio.commands.test.command import cli as cmd_test -from platformio.compat import WINDOWS -def test_local_env(isolated_pio_core): +def test_local_env(): result = util.exec_command( [ "platformio", @@ -40,7 +38,7 @@ def test_local_env(isolated_pio_core): ] -def test_multiple_env_build(clirunner, validate_cliresult, isolated_pio_core, tmpdir): +def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write( @@ -85,9 +83,7 @@ void loop() {} assert "Multiple ways to build" not in result.output -def test_setup_teardown_are_compilable( - clirunner, validate_cliresult, isolated_pio_core, tmpdir -): +def test_setup_teardown_are_compilable(clirunner, validate_cliresult, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write( diff --git a/tests/conftest.py b/tests/conftest.py index 438ced13..56a59cbd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,16 +36,9 @@ def validate_cliresult(): return decorator -@pytest.fixture(scope="module") -def clirunner(): - return CliRunner() - - -@pytest.fixture(scope="module") -def isolated_pio_core(request, tmpdir_factory): - core_dir = tmpdir_factory.mktemp(".platformio") +@pytest.fixture(scope="session") +def clirunner(request): backup_env_vars = { - "PLATFORMIO_CORE_DIR": {"new": str(core_dir)}, "PLATFORMIO_WORKSPACE_DIR": {"new": None}, } for key, item in backup_env_vars.items(): @@ -62,6 +55,19 @@ def isolated_pio_core(request, tmpdir_factory): elif key in os.environ: del os.environ[key] + request.addfinalizer(fin) + + return CliRunner() + + +@pytest.fixture(scope="module") +def isolated_pio_core(request, tmpdir_factory): + core_dir = tmpdir_factory.mktemp(".platformio") + os.environ["PLATFORMIO_CORE_DIR"] = str(core_dir) + + def fin(): + del os.environ["PLATFORMIO_CORE_DIR"] + request.addfinalizer(fin) return core_dir diff --git a/tests/test_builder.py b/tests/test_builder.py index dd0fe1f2..f220e50c 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -15,7 +15,7 @@ from platformio.commands.run.command import cli as cmd_run -def test_build_flags(clirunner, isolated_pio_core, validate_cliresult, tmpdir): +def test_build_flags(clirunner, validate_cliresult, tmpdir): build_flags = [ ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), diff --git a/tests/test_ino2cpp.py b/tests/test_ino2cpp.py index af91da8d..d1434df7 100644 --- a/tests/test_ino2cpp.py +++ b/tests/test_ino2cpp.py @@ -31,12 +31,12 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("piotest_dir", test_dirs) -def test_example(clirunner, isolated_pio_core, validate_cliresult, piotest_dir): +def test_example(clirunner, validate_cliresult, piotest_dir): result = clirunner.invoke(cmd_ci, [piotest_dir, "-b", "uno"]) validate_cliresult(result) -def test_warning_line(clirunner, isolated_pio_core, validate_cliresult): +def test_warning_line(clirunner, validate_cliresult): result = clirunner.invoke(cmd_ci, [join(INOTEST_DIR, "basic"), "-b", "uno"]) validate_cliresult(result) assert 'basic.ino:16:14: warning: #warning "Line number is 16"' in result.output From 2c24e9eff6b0797ff44673136801dbfa5f5156aa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 30 Jun 2020 14:28:37 +0300 Subject: [PATCH 096/223] Fall back to latin-1 encoding when failed with UTF-8 while parsing manifest --- platformio/package/manifest/parser.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 86c98587..0b826479 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -61,8 +61,14 @@ class ManifestFileType(object): class ManifestParserFactory(object): @staticmethod def read_manifest_contents(path): - with io.open(path, encoding="utf-8") as fp: - return fp.read() + last_err = None + for encoding in ("utf-8", "latin-1"): + try: + with io.open(path, encoding=encoding) as fp: + return fp.read() + except UnicodeDecodeError as e: + last_err = e + raise last_err @classmethod def new_from_file(cls, path, remote_url=False): From 7f48c8c14ec3cc7cf36071f2e23ca0a6011c5d73 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 30 Jun 2020 15:06:40 +0300 Subject: [PATCH 097/223] Fix PyLint for PY 2.7 --- platformio/package/manifest/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 0b826479..cb0051e0 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -68,7 +68,7 @@ class ManifestParserFactory(object): return fp.read() except UnicodeDecodeError as e: last_err = e - raise last_err + raise last_err # pylint: disable=raising-bad-type @classmethod def new_from_file(cls, path, remote_url=False): From 899a6734ee58581a3b2b38583b3d74a30afb5afa Mon Sep 17 00:00:00 2001 From: Rosen Stoyanov Date: Tue, 30 Jun 2020 21:48:44 +0300 Subject: [PATCH 098/223] Add .ccls to .gitignore (vim and emacs) (#3576) * Add .ccls to .gitignore (vim) * Add .ccls to .gitignore (emacs) --- platformio/ide/tpls/emacs/.gitignore.tpl | 1 + platformio/ide/tpls/vim/.gitignore.tpl | 1 + 2 files changed, 2 insertions(+) diff --git a/platformio/ide/tpls/emacs/.gitignore.tpl b/platformio/ide/tpls/emacs/.gitignore.tpl index b8e379fa..6f8bafd3 100644 --- a/platformio/ide/tpls/emacs/.gitignore.tpl +++ b/platformio/ide/tpls/emacs/.gitignore.tpl @@ -1,2 +1,3 @@ .pio .clang_complete +.ccls diff --git a/platformio/ide/tpls/vim/.gitignore.tpl b/platformio/ide/tpls/vim/.gitignore.tpl index bbdd36c7..1159b2d8 100644 --- a/platformio/ide/tpls/vim/.gitignore.tpl +++ b/platformio/ide/tpls/vim/.gitignore.tpl @@ -1,3 +1,4 @@ .pio .clang_complete .gcc-flags.json +.ccls From b3dabb221d8aaf4d73858d146dde057589995b17 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Jul 2020 16:07:36 +0300 Subject: [PATCH 099/223] Allow "+" in a package name --- platformio/package/pack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 1e18c55a..0e3921fa 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -55,7 +55,7 @@ class PackagePacker(object): manifest = self.load_manifest(src) filename = re.sub( - r"[^\da-zA-Z\-\._]+", + r"[^\da-zA-Z\-\._\+]+", "", "{name}{system}-{version}.tar.gz".format( name=manifest["name"], From 08a87f3a21db3168ac17933d67da51547fcc0eed Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Jul 2020 19:14:58 +0300 Subject: [PATCH 100/223] Do not allow [;.<>] chars for a package name --- platformio/package/manifest/schema.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 3502550a..e19e6f25 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -153,7 +153,9 @@ class ManifestSchema(BaseSchema): required=True, validate=[ validate.Length(min=1, max=100), - validate.Regexp(r"^[^:/]+$", error="The next chars [:/] are not allowed"), + validate.Regexp( + r"^[^:;/,@\<\>]+$", error="The next chars [:;/,@<>] are not allowed" + ), ], ) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) From ef53bcf601c02c5091bedcbe86cacd24113f6cc9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 14:17:00 +0300 Subject: [PATCH 101/223] Ignore empty fields in library.properties manifest --- platformio/package/manifest/parser.py | 2 ++ tests/package/test_manifest.py | 1 + 2 files changed, 3 insertions(+) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index cb0051e0..d49d7b9c 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -482,6 +482,8 @@ class LibraryPropertiesManifestParser(BaseManifestParser): if line.startswith("#"): continue key, value = line.split("=", 1) + if not value.strip(): + continue data[key.strip()] = value.strip() return data diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 884d6af0..53a31e9b 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -202,6 +202,7 @@ author=SomeAuthor sentence=This is Arduino library customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third +ignore_empty_field= """ raw_data = parser.LibraryPropertiesManifestParser(contents).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) From a79e933c377a459d8a5642c8d01fa914dc9e0d63 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 14:22:35 +0300 Subject: [PATCH 102/223] Ignore author's broken email in a package manifest --- platformio/package/manifest/parser.py | 2 ++ tests/package/test_manifest.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index d49d7b9c..66dee43c 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -185,6 +185,8 @@ class BaseManifestParser(object): assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) + if "@" not in author["email"]: + author["email"] = None for key in list(author.keys()): if author[key] is None: del author[key] diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 53a31e9b..fe5054d0 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -279,7 +279,7 @@ ignore_empty_field= # Author + Maintainer data = parser.LibraryPropertiesManifestParser( """ -author=Rocket Scream Electronics +author=Rocket Scream Electronics maintainer=Rocket Scream Electronics """ ).as_dict() From f97632202b70bfcc230218aee54f8d9045ce5240 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 15:57:10 +0300 Subject: [PATCH 103/223] Fix issue with KeyError --- platformio/commands/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index d08529fd..f635bff1 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -445,7 +445,7 @@ def lib_show(library, json_output): for author in lib.get("authors", []): _data = [] for key in ("name", "email", "url", "maintainer"): - if not author[key]: + if not author.get(key): continue if key == "email": _data.append("<%s>" % author[key]) From 0f8042eeb4a03762211ddfe1a1a1853ab45bf891 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 15:57:49 +0300 Subject: [PATCH 104/223] Implement PackagePacker.get_archive_name API --- platformio/package/pack.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 0e3921fa..4cbaa6c0 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -40,6 +40,16 @@ class PackagePacker(object): self.package = package self.manifest_uri = manifest_uri + @staticmethod + def get_archive_name(name, version, system=None): + return re.sub( + r"[^\da-zA-Z\-\._\+]+", + "", + "{name}{system}-{version}.tar.gz".format( + name=name, system=("-" + system) if system else "", version=version, + ), + ) + def pack(self, dst=None): tmp_dir = tempfile.mkdtemp() try: @@ -54,14 +64,10 @@ class PackagePacker(object): src = self.find_source_root(src) manifest = self.load_manifest(src) - filename = re.sub( - r"[^\da-zA-Z\-\._\+]+", - "", - "{name}{system}-{version}.tar.gz".format( - name=manifest["name"], - system="-" + manifest["system"][0] if "system" in manifest else "", - version=manifest["version"], - ), + filename = self.get_archive_name( + manifest["name"], + manifest["version"], + manifest["system"][0] if "system" in manifest else None, ) if not dst: From 8b24b0f657e7482d4b5f91ea4584beef16029329 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 23:37:28 +0300 Subject: [PATCH 105/223] Sync docs & examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 438d910c..b3e5044e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 438d910cc4d35337df90a6b23955f89a0675b52c +Subproject commit b3e5044e92390eb954811e628cddffd6dc17818b diff --git a/examples b/examples index 2fdead5d..0507e27c 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 2fdead5df5f146c6f7c70eeef63669bebab7a225 +Subproject commit 0507e27c58dbc184420658762b1037348f5f0e87 From 3c986ed681cb50625a832424bffa2c711559053e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 Jul 2020 16:28:51 +0300 Subject: [PATCH 106/223] Remove recursively .pio folders when packing a package --- platformio/package/pack.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 4cbaa6c0..0145a61e 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -28,11 +28,13 @@ from platformio.unpacker import FileUnpacker class PackagePacker(object): EXCLUDE_DEFAULT = [ "._*", + "__*", ".DS_Store", - ".git", - ".hg", - ".svn", - ".pio", + ".git/", + ".hg/", + ".svn/", + ".pio/", + "**/.pio/", ] INCLUDE_DEFAULT = ManifestFileType.items().values() From abd3f8b3b5d6d6324b0f1d728d30f1c13c9324ef Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 Jul 2020 22:53:01 +0300 Subject: [PATCH 107/223] Docs: Remove legacy library dependency syntax for github --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b3e5044e..7d24530a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b3e5044e92390eb954811e628cddffd6dc17818b +Subproject commit 7d24530a566df810d49c3a04398c94a31b7be14a From 40d6847c96211e7316edd457bd3fef08fddc5b67 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 13:46:36 +0300 Subject: [PATCH 108/223] Add option to pass a custom path where to save package archive --- platformio/commands/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index e5eb7db6..5a3092e2 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -39,9 +39,12 @@ def cli(): @cli.command("pack", short_help="Create a tarball from a package") @click.argument("package", required=True, metavar="") -def package_pack(package): +@click.option( + "-o", "--output", help="A destination path (folder or a full path to file)" +) +def package_pack(package, output): p = PackagePacker(package) - archive_path = p.pack() + archive_path = p.pack(output) click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") From 42fd28456028094919d1db5c9df12932a355f3c8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 20:21:10 +0300 Subject: [PATCH 109/223] Improve parsing "author" field of library.properties manfiest --- platformio/package/manifest/parser.py | 11 +++++++---- tests/package/test_manifest.py | 7 +++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 66dee43c..28743e54 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -198,10 +198,13 @@ class BaseManifestParser(object): return (None, None) name = raw email = None - for ldel, rdel in [("<", ">"), ("(", ")")]: - if ldel in raw and rdel in raw: - name = raw[: raw.index(ldel)] - email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + ldel = "<" + rdel = ">" + if ldel in raw and rdel in raw: + name = raw[: raw.index(ldel)] + email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + if "(" in name: + name = name.split("(")[0] return (name.strip(), email.strip() if email else None) @staticmethod diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index fe5054d0..c2436328 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -198,7 +198,7 @@ def test_library_properties_parser(): contents = """ name=TestPackage version=1.2.3 -author=SomeAuthor +author=SomeAuthor , Another Author (nickname) sentence=This is Arduino library customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third @@ -218,7 +218,10 @@ ignore_empty_field= "export": { "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, - "authors": [{"email": "info@author.com", "name": "SomeAuthor"}], + "authors": [ + {"email": "info@author.com", "name": "SomeAuthor"}, + {"name": "Another Author"}, + ], "keywords": ["uncategorized"], "customField": "Custom Value", "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", From 84132d9459a8249b1e969122a570ae2a3cec9a89 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 21:52:34 +0300 Subject: [PATCH 110/223] Fix tests --- tests/commands/test_check.py | 2 +- tests/commands/test_ci.py | 4 ++-- tests/commands/test_lib.py | 35 +++++++++++++++-------------------- tests/commands/test_update.py | 4 ++-- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 116e4a52..998d44cf 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -357,7 +357,7 @@ int main() { assert low_result.exit_code != 0 -def test_check_pvs_studio_free_license(clirunner, isolated_pio_core, tmpdir): +def test_check_pvs_studio_free_license(clirunner, tmpdir): config = """ [env:test] platform = teensy diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 0f7aceed..f3308a6a 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -114,13 +114,13 @@ def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): [ join( storage_dir, - "OneWire_ID1", + "OneWire", "examples", "DS2408_Switch", "DS2408_Switch.pde", ), "-l", - join(storage_dir, "OneWire_ID1"), + join(storage_dir, "OneWire"), "-b", "uno", ], diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 3341ef71..f51b9dc2 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -56,14 +56,14 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_cor items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ - "ArduinoJson_ID64", - "ArduinoJson_ID64@5.10.1", - "NeoPixelBus_ID547", - "AsyncMqttClient_ID346", - "ESPAsyncTCP_ID305", - "AsyncTCP_ID1826", - "Adafruit PN532_ID29", - "Adafruit BusIO_ID6214", + "ArduinoJson", + "ArduinoJson@5.10.1", + "NeoPixelBus", + "AsyncMqttClient", + "ESPAsyncTCP", + "AsyncTCP", + "Adafruit PN532", + "Adafruit BusIO", ] assert set(items1) == set(items2) @@ -94,7 +94,7 @@ def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core assert result.exit_code != 0 items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire_ID1", "ESP32WebServer"] + items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire", "ESP32WebServer"] assert set(items1) >= set(items2) @@ -135,11 +135,6 @@ def test_install_duplicates(clirunner, validate_cliresult, without_internet): validate_cliresult(result) assert "is already installed" in result.output - # by ID - result = clirunner.invoke(cmd_lib, ["-g", "install", "29"]) - validate_cliresult(result) - assert "is already installed" in result.output - # archive result = clirunner.invoke( cmd_lib, @@ -276,7 +271,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): [ "-g", "uninstall", - "1", + "OneWire", "https://github.com/bblanchon/ArduinoJson.git", "ArduinoJson@!=5.6.7", "Adafruit PN532", @@ -290,15 +285,15 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): "platformio-libmirror", "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "ESPAsyncTCP_ID305", + "ESPAsyncTCP", "ESP32WebServer", - "NeoPixelBus_ID547", + "NeoPixelBus", "PJON", - "AsyncMqttClient_ID346", - "ArduinoJson_ID64", + "AsyncMqttClient", + "ArduinoJson", "SomeLib_ID54", "PJON@src-79de467ebe19de18287becff0a1fb42d", - "AsyncTCP_ID1826", + "AsyncTCP", ] assert set(items1) == set(items2) diff --git a/tests/commands/test_update.py b/tests/commands/test_update.py index 9325e501..1817be33 100644 --- a/tests/commands/test_update.py +++ b/tests/commands/test_update.py @@ -15,8 +15,8 @@ from platformio.commands.update import cli as cmd_update -def test_update(clirunner, validate_cliresult): - matches = ("Platform Manager", "Up-to-date", "Library Manager") +def test_update(clirunner, validate_cliresult, isolated_pio_core): + matches = ("Platform Manager", "Library Manager") result = clirunner.invoke(cmd_update, ["--only-check"]) validate_cliresult(result) assert all([m in result.output for m in matches]) From a00722bef4599fa3f67f81368fe60e2651e4ff19 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 21:53:28 +0300 Subject: [PATCH 111/223] Ignore maintainer's broken email in library.properties manifest --- platformio/package/manifest/parser.py | 16 +++++++++------- tests/package/test_manifest.py | 7 ++++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 28743e54..8ba94344 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -181,7 +181,7 @@ class BaseManifestParser(object): return result @staticmethod - def normalize_author(author): + def cleanup_author(author): assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) @@ -357,7 +357,7 @@ class LibraryJsonManifestParser(BaseManifestParser): # normalize Union[dict, list] fields if not isinstance(raw, list): raw = [raw] - return [self.normalize_author(author) for author in raw] + return [self.cleanup_author(author) for author in raw] @staticmethod def _parse_platforms(raw): @@ -430,7 +430,7 @@ class ModuleJsonManifestParser(BaseManifestParser): name, email = self.parse_author_name_and_email(author) if not name: continue - result.append(self.normalize_author(dict(name=name, email=email))) + result.append(self.cleanup_author(dict(name=name, email=email))) return result @staticmethod @@ -471,7 +471,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser): ) if "author" in data: data["authors"] = self._parse_authors(data) - del data["author"] + for key in ("author", "maintainer"): + if key in data: + del data[key] if "depends" in data: data["dependencies"] = self._parse_dependencies(data["depends"]) return data @@ -544,7 +546,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): name, email = self.parse_author_name_and_email(author) if not name: continue - authors.append(self.normalize_author(dict(name=name, email=email))) + authors.append(self.cleanup_author(dict(name=name, email=email))) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) if not name: @@ -555,11 +557,11 @@ class LibraryPropertiesManifestParser(BaseManifestParser): continue found = True item["maintainer"] = True - if not item.get("email") and email: + if not item.get("email") and email and "@" in email: item["email"] = email if not found: authors.append( - self.normalize_author(dict(name=name, email=email, maintainer=True)) + self.cleanup_author(dict(name=name, email=email, maintainer=True)) ) return authors diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index c2436328..188c5d8e 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -198,7 +198,8 @@ def test_library_properties_parser(): contents = """ name=TestPackage version=1.2.3 -author=SomeAuthor , Another Author (nickname) +author=SomeAuthor , Maintainer Author (nickname) +maintainer=Maintainer Author (nickname) sentence=This is Arduino library customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third @@ -219,8 +220,8 @@ ignore_empty_field= "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, "authors": [ - {"email": "info@author.com", "name": "SomeAuthor"}, - {"name": "Another Author"}, + {"name": "SomeAuthor", "email": "info@author.com"}, + {"name": "Maintainer Author", "maintainer": True}, ], "keywords": ["uncategorized"], "customField": "Custom Value", From 940682255de05a421ec7ffce385023360f70a14a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 22:16:52 +0300 Subject: [PATCH 112/223] Lock Python's isort package to isort<5 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3db3a8ef..fbe285e2 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ passenv = * usedevelop = True deps = py36,py37,py38: black - isort + isort<5 pylint pytest pytest-xdist From f27c71a0d4ea3e07474cf0c81f9f2aa132a871e2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 22:56:14 +0300 Subject: [PATCH 113/223] Increase author name length to 100 chars for manifest --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index e19e6f25..080a7654 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -84,7 +84,7 @@ class StrictListField(fields.List): class AuthorSchema(StrictSchema): - name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) From f85cf61d68014b1bcfe543cc2b22babdadeb17fd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 23:23:14 +0300 Subject: [PATCH 114/223] Revert back max length of author name to 50 chars --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 080a7654..e19e6f25 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -84,7 +84,7 @@ class StrictListField(fields.List): class AuthorSchema(StrictSchema): - name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) From e570aadd72cc7f17877c01e1b532ae941566da76 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 Jul 2020 17:17:34 +0300 Subject: [PATCH 115/223] Docs: Sync --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7d24530a..8ea98724 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7d24530a566df810d49c3a04398c94a31b7be14a +Subproject commit 8ea98724069b8783575836e6ac2ee1d8a8414791 From a688edbdf1a2f9b50719a45d282af0d3ee70fdfd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 Jul 2020 21:53:46 +0300 Subject: [PATCH 116/223] Fix an issue with manifest parser when "new_from_archive" API is used --- platformio/package/manifest/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 8ba94344..6507849d 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -122,7 +122,9 @@ class ManifestParserFactory(object): with tarfile.open(path, mode="r:gz") as tf: for t in sorted(ManifestFileType.items().values()): try: - return ManifestParserFactory.new(tf.extractfile(t).read(), t) + return ManifestParserFactory.new( + tf.extractfile(t).read().decode(), t + ) except KeyError: pass raise UnknownManifestError("Unknown manifest file type in %s archive" % path) From 368c66727bf01b3a20fd30f9304346adeb115e34 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 12 Jul 2020 22:39:32 +0300 Subject: [PATCH 117/223] Fix issue with package packing when re-map is used and manifest is missed in "include" (copy it now) --- platformio/package/pack.py | 16 +++++++++------- tests/package/test_pack.py | 5 ++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 0145a61e..5823a4f4 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re import shutil @@ -77,12 +78,7 @@ class PackagePacker(object): elif os.path.isdir(dst): dst = os.path.join(dst, filename) - return self._create_tarball( - src, - dst, - include=manifest.get("export", {}).get("include"), - exclude=manifest.get("export", {}).get("exclude"), - ) + return self._create_tarball(src, dst, manifest) finally: shutil.rmtree(tmp_dir) @@ -114,7 +110,9 @@ class PackagePacker(object): return src - def _create_tarball(self, src, dst, include=None, exclude=None): + def _create_tarball(self, src, dst, manifest): + include = manifest.get("export", {}).get("include") + exclude = manifest.get("export", {}).get("exclude") # remap root if ( include @@ -122,6 +120,10 @@ class PackagePacker(object): and os.path.isdir(os.path.join(src, include[0])) ): src = os.path.join(src, include[0]) + with open(os.path.join(src, "library.json"), "w") as fp: + manifest_updated = manifest.copy() + del manifest_updated["export"]["include"] + json.dump(manifest_updated, fp, indent=2, ensure_ascii=False) include = None src_filters = self.compute_src_filters(include, exclude) diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index 95b43570..a964f5cd 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -61,7 +61,10 @@ def test_filters(tmpdir_factory): ) p = PackagePacker(str(pkg_dir)) with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: - assert set(tar.getnames()) == set(["util/helpers.cpp", "main.cpp"]) + assert set(tar.getnames()) == set( + ["util/helpers.cpp", "main.cpp", "library.json"] + ) + os.unlink(str(src_dir.join("library.json"))) # test include "src" and "include" pkg_dir.join("library.json").write( From cca3099d13447af6149b537a6136e3a85eec1c91 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Jul 2020 18:55:29 +0300 Subject: [PATCH 118/223] Ensure that module.json keywords are lowercased --- platformio/package/manifest/parser.py | 2 ++ tests/package/test_manifest.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 6507849d..b4a93d98 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -422,6 +422,8 @@ class ModuleJsonManifestParser(BaseManifestParser): del data["licenses"] if "dependencies" in data: data["dependencies"] = self._parse_dependencies(data["dependencies"]) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") return data def _parse_authors(self, raw): diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 188c5d8e..899f1cdf 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -172,7 +172,7 @@ def test_module_json_parser(): "name": "YottaLibrary", "description": "This is Yotta library", "homepage": "https://yottabuild.org", - "keywords": ["mbed", "Yotta"], + "keywords": ["mbed", "yotta"], "license": "Apache-2.0", "platforms": ["*"], "frameworks": ["mbed"], From 1368fa4c3b980656752e6273aeab239ed6af2e32 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Jul 2020 21:07:09 +0300 Subject: [PATCH 119/223] Implement new fields (id, ownername, url, requirements) for PackageSpec API --- platformio/commands/package.py | 6 +- platformio/package/spec.py | 134 +++++++++++++++++++++++++++------ tests/package/test_spec.py | 112 ++++++++++++++++++++++++--- 3 files changed, 215 insertions(+), 37 deletions(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 5a3092e2..b5e4ad06 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -78,7 +78,7 @@ def package_publish(package, owner, released_at, private, notify): @cli.command("unpublish", short_help="Remove a pushed package from the registry") @click.argument( - "package", required=True, metavar="[<@organization>/][@]" + "package", required=True, metavar="[/][@]" ) @click.option( "--type", @@ -96,8 +96,8 @@ def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin response = RegistryClient().unpublish_package( type=type, name=spec.name, - owner=spec.organization, - version=spec.version, + owner=spec.ownername, + version=spec.requirements, undo=undo, ) click.secho(response.get("message"), fg="green") diff --git a/platformio/package/spec.py b/platformio/package/spec.py index f031c71c..f240c843 100644 --- a/platformio/package/spec.py +++ b/platformio/package/spec.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import tarfile -from platformio.compat import get_object_members +from platformio.compat import get_object_members, string_types from platformio.package.manifest.parser import ManifestFileType @@ -55,29 +56,114 @@ class PackageType(object): class PackageSpec(object): - def __init__(self, raw=None, organization=None, name=None, version=None): - if raw is not None: - organization, name, version = self.parse(raw) - - self.organization = organization + def __init__( # pylint: disable=redefined-builtin,too-many-arguments + self, raw=None, ownername=None, id=None, name=None, requirements=None, url=None + ): + self.ownername = ownername + self.id = id self.name = name - self.version = version + self.requirements = requirements + self.url = url + + self._parse(raw) + + def __repr__(self): + return ( + "PackageSpec ".format( + ownername=self.ownername, + id=self.id, + name=self.name, + requirements=self.requirements, + url=self.url, + ) + ) + + def __eq__(self, other): + return all( + [ + self.ownername == other.ownername, + self.id == other.id, + self.name == other.name, + self.requirements == other.requirements, + self.url == other.url, + ] + ) + + def _parse(self, raw): + if raw is None: + return + if not isinstance(raw, string_types): + raw = str(raw) + raw = raw.strip() + + parsers = ( + self._parse_requirements, + self._parse_fixed_name, + self._parse_id, + self._parse_ownername, + self._parse_url, + ) + for parser in parsers: + if raw is None: + break + raw = parser(raw) + + # if name is not fixed, parse it from URL + if not self.name and self.url: + self.name = self._parse_name_from_url(self.url) + elif raw: + # the leftover is a package name + self.name = raw + + def _parse_requirements(self, raw): + if "@" not in raw: + return raw + tokens = raw.rsplit("@", 1) + if any(s in tokens[1] for s in (":", "/")): + return raw + self.requirements = tokens[1].strip() + return tokens[0].strip() + + def _parse_fixed_name(self, raw): + if "=" not in raw or raw.startswith("id="): + return raw + tokens = raw.split("=", 1) + if "/" in tokens[0]: + return raw + self.name = tokens[0].strip() + return tokens[1].strip() + + def _parse_id(self, raw): + if raw.isdigit(): + self.id = int(raw) + return None + if raw.startswith("id="): + return self._parse_id(raw[3:]) + return raw + + def _parse_ownername(self, raw): + if raw.count("/") != 1 or "@" in raw: + return raw + tokens = raw.split("/", 1) + self.ownername = tokens[0].strip() + self.name = tokens[1].strip() + return None + + def _parse_url(self, raw): + if not any(s in raw for s in ("@", ":", "/")): + return raw + self.url = raw.strip() + return None @staticmethod - def parse(raw): - organization = None - name = None - version = None - raw = raw.strip() - if raw.startswith("@") and "/" in raw: - tokens = raw[1:].split("/", 1) - organization = tokens[0].strip() - raw = tokens[1] - if "@" in raw: - name, version = raw.split("@", 1) - name = name.strip() - version = version.strip() - else: - name = raw.strip() - - return organization, name, version + def _parse_name_from_url(url): + if url.endswith("/"): + url = url[:-1] + for c in ("#", "?"): + if c in url: + url = url[: url.index(c)] + name = os.path.basename(url) + if "." in name: + return name.split(".", 1)[0].strip() + return name diff --git a/tests/package/test_spec.py b/tests/package/test_spec.py index 1886a836..dce89d7f 100644 --- a/tests/package/test_spec.py +++ b/tests/package/test_spec.py @@ -15,13 +15,105 @@ from platformio.package.spec import PackageSpec -def test_parser(): - inputs = [ - ("foo", (None, "foo", None)), - ("@org/foo", ("org", "foo", None)), - ("@org/foo @ 1.2.3", ("org", "foo", "1.2.3")), - ("bar @ 1.2.3", (None, "bar", "1.2.3")), - ("cat@^1.2", (None, "cat", "^1.2")), - ] - for raw, result in inputs: - assert PackageSpec.parse(raw) == result +def test_ownername(): + assert PackageSpec("alice/foo library") == PackageSpec( + ownername="alice", name="foo library" + ) + assert PackageSpec(" bob / bar ") == PackageSpec(ownername="bob", name="bar") + + +def test_id(): + assert PackageSpec(13) == PackageSpec(id=13) + assert PackageSpec("20") == PackageSpec(id=20) + assert PackageSpec("id=199") == PackageSpec(id=199) + + +def test_name(): + assert PackageSpec("foo") == PackageSpec(name="foo") + assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") + + +def test_requirements(): + assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") + assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") + assert PackageSpec("id=20 @ !=1.2.3,<2.0") == PackageSpec( + id=20, requirements="!=1.2.3,<2.0" + ) + + +def test_local_urls(): + assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo" + ) + assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( + url="file:///tmp/bar.zip", name="customName" + ) + assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( + url="file:///tmp/some-lib/", name="some-lib" + ) + assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" + ) + + +def test_external_urls(): + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.zip", + name="develop", + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/" + "develop.zip?param=value", + name="develop", + requirements="!=2", + ) + assert PackageSpec( + "platformio-core=" + "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", + name="platformio-core", + requirements="4.4.0", + ) + + +def test_vcs_urls(): + assert PackageSpec( + "https://github.com/platformio/platformio-core.git" + ) == PackageSpec( + name="platformio-core", url="https://github.com/platformio/platformio-core.git", + ) + assert PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) == PackageSpec( + name="wolfSSL", url="https://os.mbed.com/users/wolfSSL/code/wolfSSL/", + ) + assert PackageSpec( + "git+https://github.com/platformio/platformio-core.git#master" + ) == PackageSpec( + name="platformio-core", + url="git+https://github.com/platformio/platformio-core.git#master", + ) + assert PackageSpec( + "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" + ) == PackageSpec( + name="core", + url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", + requirements="4.4.0", + ) + assert PackageSpec("git@github.com:platformio/platformio-core.git") == PackageSpec( + name="platformio-core", url="git@github.com:platformio/platformio-core.git", + ) + assert PackageSpec( + "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" + ) == PackageSpec( + name="pkg", + url="git+git@github.com:platformio/platformio-core.git", + requirements="^1.2.3,!=5", + ) From a6f143d1ca06f27d6286648b41fb61abeab2789d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jul 2020 14:20:29 +0300 Subject: [PATCH 120/223] Dump data intended for IDE extensions/plugins using a new `platformio project idedata` command --- HISTORY.rst | 3 ++- docs | 2 +- platformio/commands/project.py | 48 ++++++++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index cea8fb1b..8885bb03 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,10 +24,11 @@ PlatformIO Core 4 - Launch command with custom options declared in `"platformio.ini" `__ - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) -* Display system-wide information using `platformio system info `__ command (`issue #3521 `_) +* Display system-wide information using a new `platformio system info `__ command (`issue #3521 `_) * List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment +* Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command * Do not generate ".travis.yml" for a new project, let the user have a choice * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) diff --git a/docs b/docs index 8ea98724..0da1c281 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8ea98724069b8783575836e6ac2ee1d8a8414791 +Subproject commit 0da1c2810b62f69179cf797f0af6762d0eb9e8f9 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 27e33455..c261a9d9 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -14,6 +14,7 @@ # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,line-too-long +import json import os import click @@ -25,7 +26,7 @@ from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError -from platformio.project.helpers import is_platformio_project +from platformio.project.helpers import is_platformio_project, load_project_ide_data @click.group(short_help="Project Manager") @@ -38,9 +39,7 @@ def cli(): "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), ) @click.option("--json-output", is_flag=True) def project_config(project_dir, json_output): @@ -54,7 +53,6 @@ def project_config(project_dir, json_output): "Computed project configuration for %s" % click.style(project_dir, fg="cyan") ) for section, options in config.as_tuple(): - click.echo() click.secho(section, fg="cyan") click.echo("-" * len(section)) click.echo( @@ -66,6 +64,46 @@ def project_config(project_dir, json_output): tablefmt="plain", ) ) + click.echo() + return None + + +@cli.command("idedata", short_help="Dump data intended for IDE extensions/plugins") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), +) +@click.option("-e", "--environment", multiple=True) +@click.option("--json-output", is_flag=True) +def project_idedata(project_dir, environment, json_output): + if not is_platformio_project(project_dir): + raise NotPlatformIOProjectError(project_dir) + with fs.cd(project_dir): + config = ProjectConfig.get_instance() + config.validate(environment) + environment = list(environment or config.envs()) + + if json_output: + return click.echo(json.dumps(load_project_ide_data(project_dir, environment))) + + for envname in environment: + click.echo("Environment: " + click.style(envname, fg="cyan", bold=True)) + click.echo("=" * (13 + len(envname))) + click.echo( + tabulate( + [ + (click.style(name, bold=True), "=", json.dumps(value, indent=2)) + for name, value in load_project_ide_data( + project_dir, envname + ).items() + ], + tablefmt="plain", + ) + ) + click.echo() + return None From ca3305863738f43d7ca93eb06a41707d627462ce Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jul 2020 23:16:46 +0300 Subject: [PATCH 121/223] New commands for the registry package management (pack, publish, unpublish) --- HISTORY.rst | 5 +++++ docs | 2 +- platformio/commands/lib.py | 19 +++---------------- platformio/commands/package.py | 14 ++++++++++++-- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8885bb03..43898869 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,11 @@ PlatformIO Core 4 - Manage organization teams - Manage resource access +* Registry Package Management + + - Publish a personal or organization package using `platformio package publish `__ command + - Remove a pushed package from the registry using `platformio package unpublish `__ command + * New `Custom Targets `__ - Pre/Post processing based on a dependent sources (other target, source file, etc.) diff --git a/docs b/docs index 0da1c281..0ed8c2da 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0da1c2810b62f69179cf797f0af6762d0eb9e8f9 +Subproject commit 0ed8c2da4db1952f027fb980045d37dc194614c4 diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index f635bff1..a1593732 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -25,8 +25,6 @@ from platformio import exception, fs, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.package.manifest.schema import ManifestSchema from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.exception import InvalidProjectConfError @@ -495,24 +493,13 @@ def lib_show(library, json_output): return True -@cli.command("register", short_help="Register a new library") +@cli.command("register", short_help="Deprecated") @click.argument("config_url") def lib_register(config_url): - if not config_url.startswith("http://") and not config_url.startswith("https://"): - raise exception.InvalidLibConfURL(config_url) - - # Validate manifest - ManifestSchema().load_manifest( - ManifestParserFactory.new_from_url(config_url).as_dict() + raise exception.UserSideException( + "This command is deprecated. Please use `pio package publish` command." ) - result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) - if "message" in result and result["message"]: - click.secho( - result["message"], - fg="green" if "successed" in result and result["successed"] else "red", - ) - @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index b5e4ad06..73eb551a 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -38,7 +38,12 @@ def cli(): @cli.command("pack", short_help="Create a tarball from a package") -@click.argument("package", required=True, metavar="") +@click.argument( + "package", + required=True, + default=os.getcwd, + metavar="", +) @click.option( "-o", "--output", help="A destination path (folder or a full path to file)" ) @@ -49,7 +54,12 @@ def package_pack(package, output): @cli.command("publish", short_help="Publish a package to the registry") -@click.argument("package", required=True, metavar="") +@click.argument( + "package", + required=True, + default=os.getcwd, + metavar="", +) @click.option( "--owner", help="PIO Account username (can be organization username). " From a2efd7f7c523e92963bbec6154c6c58ec2f1bc68 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jul 2020 23:18:07 +0300 Subject: [PATCH 122/223] Bump version to 4.4.0a5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 7ad284a1..9fc3a773 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a4") +VERSION = (4, 4, "0a5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 1ed462a29a8a303d78f423d8c161a1d5894bee6d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Jul 2020 01:00:38 +0300 Subject: [PATCH 123/223] PyLint fix --- platformio/commands/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index a1593732..5bd38aee 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -495,7 +495,7 @@ def lib_show(library, json_output): @cli.command("register", short_help="Deprecated") @click.argument("config_url") -def lib_register(config_url): +def lib_register(config_url): # pylint: disable=unused-argument raise exception.UserSideException( "This command is deprecated. Please use `pio package publish` command." ) From ea30d94324504cb0e8a3a521fddc14624cf90a3a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 Jul 2020 12:41:38 +0300 Subject: [PATCH 124/223] =?UTF-8?q?Automatically=20enable=20LDF=20dependen?= =?UTF-8?q?cy=20chain+=20mode=20(evaluates=20C/C++=20Preprocessor=20condit?= =?UTF-8?q?ional=20syntax)=20for=20Arduino=20library=20when=20=E2=80=9Clib?= =?UTF-8?q?rary.properties=E2=80=9D=20has=20=E2=80=9Cdepends=E2=80=9D=20fi?= =?UTF-8?q?eld=20//=20Resolve=20#3607?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piolib.py | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 43898869..8a2d423f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,6 +35,7 @@ PlatformIO Core 4 * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command * Do not generate ".travis.yml" for a new project, let the user have a choice +* Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) * Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) diff --git a/docs b/docs index 0ed8c2da..922f89a0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0ed8c2da4db1952f027fb980045d37dc194614c4 +Subproject commit 922f89a0d4c8e18f5f67d745826953ce2193d6b8 diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3aa6b36d..6f832382 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -507,6 +507,26 @@ class ArduinoLibBuilder(LibBuilderBase): src_filter.append("+" % (sep, ext)) return src_filter + @property + def dependencies(self): + # do not include automatically all libraries for build + # chain+ will decide later + return None + + @property + def lib_ldf_mode(self): + if not self._manifest.get("dependencies"): + return LibBuilderBase.lib_ldf_mode.fget(self) + missing = object() + global_value = self.env.GetProjectConfig().getraw( + "env:" + self.env["PIOENV"], "lib_ldf_mode", missing + ) + if global_value != missing: + return LibBuilderBase.lib_ldf_mode.fget(self) + # automatically enable C++ Preprocessing in runtime + # (Arduino IDE has this behavior) + return "chain+" + def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, ["arduino", "energia"]) From 22f1b94062191a7cfde125667071aafa174f5b5f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 Jul 2020 12:42:26 +0300 Subject: [PATCH 125/223] Bump version to 4.4.0a6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 9fc3a773..0dd66b1b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a5") +VERSION = (4, 4, "0a6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 881c5ea308ad1eafa674d664dea5b217b7264971 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jul 2020 17:37:23 +0300 Subject: [PATCH 126/223] Remove unused code --- platformio/commands/home/helpers.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index 018d5da6..aff92281 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -14,9 +14,6 @@ # pylint: disable=keyword-arg-before-vararg,arguments-differ,signature-differs -import os -import socket - import requests from twisted.internet import defer # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error @@ -52,18 +49,3 @@ def get_core_fullpath(): return where_is_program( "platformio" + (".exe" if "windows" in util.get_systype() else "") ) - - -@util.memoized(expire="10s") -def is_twitter_blocked(): - ip = "104.244.42.1" - timeout = 2 - try: - if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout) - else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) - return False - except: # pylint: disable=bare-except - pass - return True From 83110975fa967b90149b539ce7b41ae17727db74 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 23 Jul 2020 17:42:46 +0300 Subject: [PATCH 127/223] Docs: Sync --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 922f89a0..e1c26405 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 922f89a0d4c8e18f5f67d745826953ce2193d6b8 +Subproject commit e1c264053f460fcd6a1a4b7201114a737dd3ab37 From 73740aea89c537decde97a8ce89c60d7179c242b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jul 2020 17:56:41 +0300 Subject: [PATCH 128/223] Sync docs and examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 922f89a0..da10bb5e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 922f89a0d4c8e18f5f67d745826953ce2193d6b8 +Subproject commit da10bb5e00f8e711de2c9f843423db765cf8fce7 diff --git a/examples b/examples index 0507e27c..c3f6d1f1 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 0507e27c58dbc184420658762b1037348f5f0e87 +Subproject commit c3f6d1f17e6a901dc82c3bec462fcccda0cecbc9 From c193a4ceb7c795acef8534c36091db0bf24ed1f1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jul 2020 19:07:29 +0300 Subject: [PATCH 129/223] Handle proxy environment variables in lower case // Resolve #3606 --- platformio/util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index 6a664c49..c2223109 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -377,10 +377,12 @@ def _internet_on(): socket.setdefaulttimeout(timeout) for host in PING_REMOTE_HOSTS: try: - if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): + for var in ("HTTP_PROXY", "HTTPS_PROXY"): + if not os.getenv(var, var.lower()): + continue requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) - else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) + return True + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) return True except: # pylint: disable=bare-except pass From 6ace5668b8e3fe19f7636ef31de508b816920931 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jul 2020 20:57:18 +0300 Subject: [PATCH 130/223] Update the registry publish endpoints --- platformio/clients/registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index c48094ee..a1993900 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -43,7 +43,7 @@ class RegistryClient(RESTClient): with open(archive_path, "rb") as fp: response = self.send_auth_request( "post", - "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), + "/v3/packages/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={ "private": 1 if private else 0, "notify": 1 if notify else 0, @@ -67,7 +67,7 @@ class RegistryClient(RESTClient): owner = ( account.get_account_info(offline=True).get("profile").get("username") ) - path = "/v3/package/%s/%s/%s" % (owner, type, name) + path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: path = path + "/version/" + version response = self.send_auth_request( From 85f5a6a84a6680733ab2baeb9eeb8f08be110f49 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jul 2020 21:00:58 +0300 Subject: [PATCH 131/223] Bump version to 4.4.0a7 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0dd66b1b..8da7e6e0 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a6") +VERSION = (4, 4, "0a7") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 39cb23813f713fd151a7018e81069e0d7f959a8d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jul 2020 11:51:47 +0300 Subject: [PATCH 132/223] Allow ignoring "platforms" and "frameworks" fields in "library.json" and treat a library as compatible with all --- docs | 2 +- platformio/builder/tools/piolib.py | 15 +++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/docs b/docs index da10bb5e..13df46f9 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit da10bb5e00f8e711de2c9f843423db765cf8fce7 +Subproject commit 13df46f9cf4e3f822f6f642c9c9b3085a0a93193 diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 6f832382..fbd8949c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -531,10 +531,7 @@ class ArduinoLibBuilder(LibBuilderBase): return util.items_in_list(frameworks, ["arduino", "energia"]) def is_platforms_compatible(self, platforms): - items = self._manifest.get("platforms", []) - if not items: - return LibBuilderBase.is_platforms_compatible(self, platforms) - return util.items_in_list(platforms, items) + return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) class MbedLibBuilder(LibBuilderBase): @@ -768,16 +765,10 @@ class PlatformIOLibBuilder(LibBuilderBase): ) def is_platforms_compatible(self, platforms): - items = self._manifest.get("platforms") - if not items: - return LibBuilderBase.is_platforms_compatible(self, platforms) - return util.items_in_list(platforms, items) + return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) def is_frameworks_compatible(self, frameworks): - items = self._manifest.get("frameworks") - if not items: - return LibBuilderBase.is_frameworks_compatible(self, frameworks) - return util.items_in_list(frameworks, items) + return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"]) def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) From def149a29e6ea00e6898a9d68f70b358003fb2d9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jul 2020 17:13:05 +0300 Subject: [PATCH 133/223] Use updated registry API --- platformio/clients/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index a1993900..3d45584b 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -69,7 +69,7 @@ class RegistryClient(RESTClient): ) path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: - path = path + "/version/" + version + path += "/" + version response = self.send_auth_request( "delete", path, params={"undo": 1 if undo else 0}, ) From adc2d5fe7c29ac45db3421d81d108da0b9b1cd9f Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 28 Jul 2020 15:10:52 +0300 Subject: [PATCH 134/223] Update VSCode template Starting with cpptools v0.29 escaped paths in compilerArgs field don't work on Windows. --- .../tpls/vscode/.vscode/c_cpp_properties.json.tpl | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 930854d3..e6dda895 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -10,10 +10,6 @@ % return to_unix_path(text).replace('"', '\\"') % end % -% def _escape_required(flag): -% return " " in flag and systype == "windows" -% end -% % def split_args(args_string): % return click.parser.split_arg_string(to_unix_path(args_string)) % end @@ -53,10 +49,7 @@ % def _find_forced_includes(flags, inc_paths): % result = [] % include_args = ("-include", "-imacros") -% for f in flags: -% if not f.startswith(include_args): -% continue -% end +% for f in filter_args(flags, include_args): % for arg in include_args: % inc = "" % if f.startswith(arg) and f.split(arg)[1].strip(): @@ -66,6 +59,7 @@ % end % if inc: % result.append(_find_abs_path(inc, inc_paths)) +% break % end % end % end @@ -134,8 +128,7 @@ "compilerPath": "{{ cc_path }}", "compilerArgs": [ % for flag in [ -% '"%s"' % _escape(f) if _escape_required(f) else f -% for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) +% f for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) % ]: "{{ flag }}", % end From 933a09f981225f7a8e13d664fe0cf4e3098d7adb Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 28 Jul 2020 15:22:36 +0300 Subject: [PATCH 135/223] Update unit testing support for mbed framework - Take into account Mbed OS6 API changes - RawSerial is used with Mbed OS 5 since Serial doesn't support putc with baremetal profile --- examples | 2 +- platformio/commands/test/processor.py | 40 +++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/examples b/examples index c3f6d1f1..f0f4e097 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit c3f6d1f17e6a901dc82c3bec462fcccda0cecbc9 +Subproject commit f0f4e0971b0f9f7b4d77a47cb436f910bf3b4add diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index cfc0f3ca..9f1b12c8 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -25,33 +25,33 @@ TRANSPORT_OPTIONS = { "arduino": { "include": "#include ", "object": "", - "putchar": "Serial.write(c)", - "flush": "Serial.flush()", - "begin": "Serial.begin($baudrate)", - "end": "Serial.end()", + "putchar": "Serial.write(c);", + "flush": "Serial.flush();", + "begin": "Serial.begin($baudrate);", + "end": "Serial.end();", "language": "cpp", }, "mbed": { "include": "#include ", - "object": "Serial pc(USBTX, USBRX);", - "putchar": "pc.putc(c)", + "object": "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n#else\nRawSerial pc(USBTX, USBRX);\n#endif", + "putchar": "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n#else\npc.putc(c);\n#endif", "flush": "", - "begin": "pc.baud($baudrate)", + "begin": "pc.baud($baudrate);", "end": "", "language": "cpp", }, "espidf": { "include": "#include ", "object": "", - "putchar": "putchar(c)", - "flush": "fflush(stdout)", + "putchar": "putchar(c);", + "flush": "fflush(stdout);", "begin": "", "end": "", }, "zephyr": { "include": "#include ", "object": "", - "putchar": 'printk("%c", c)', + "putchar": 'printk("%c", c);', "flush": "", "begin": "", "end": "", @@ -59,18 +59,18 @@ TRANSPORT_OPTIONS = { "native": { "include": "#include ", "object": "", - "putchar": "putchar(c)", - "flush": "fflush(stdout)", + "putchar": "putchar(c);", + "flush": "fflush(stdout);", "begin": "", "end": "", }, "custom": { "include": '#include "unittest_transport.h"', "object": "", - "putchar": "unittest_uart_putchar(c)", - "flush": "unittest_uart_flush()", - "begin": "unittest_uart_begin()", - "end": "unittest_uart_end()", + "putchar": "unittest_uart_putchar(c);", + "flush": "unittest_uart_flush();", + "begin": "unittest_uart_begin();", + "end": "unittest_uart_end();", "language": "cpp", }, } @@ -174,22 +174,22 @@ class TestProcessorBase(object): "void output_start(unsigned int baudrate)", "#endif", "{", - " $begin;", + " $begin", "}", "", "void output_char(int c)", "{", - " $putchar;", + " $putchar", "}", "", "void output_flush(void)", "{", - " $flush;", + " $flush", "}", "", "void output_complete(void)", "{", - " $end;", + " $end", "}", ] ) From 2bc47f4e97b12b07be2c3a2bad5df9b7b1ef70df Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 28 Jul 2020 15:55:25 +0300 Subject: [PATCH 136/223] PyLint fix --- platformio/commands/test/processor.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 9f1b12c8..334db858 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -33,8 +33,14 @@ TRANSPORT_OPTIONS = { }, "mbed": { "include": "#include ", - "object": "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n#else\nRawSerial pc(USBTX, USBRX);\n#endif", - "putchar": "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n#else\npc.putc(c);\n#endif", + "object": ( + "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n" + "#else\nRawSerial pc(USBTX, USBRX);\n#endif" + ), + "putchar": ( + "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n" + "#else\npc.putc(c);\n#endif" + ), "flush": "", "begin": "pc.baud($baudrate);", "end": "", From abc0489ac6cf213ee1235017ecf336344825c825 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 28 Jul 2020 15:59:02 +0300 Subject: [PATCH 137/223] Update changelog --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 8a2d423f..b068fd59 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -36,6 +36,8 @@ PlatformIO Core 4 * Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command * Do not generate ".travis.yml" for a new project, let the user have a choice * Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) +* Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 +* Do not escape compiler arguments in VSCode template on Windows * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) * Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) From d329aef87627d206573bacf86df22daf80544f48 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 31 Jul 2020 15:42:26 +0300 Subject: [PATCH 138/223] Initial version of a new package manager --- platformio/app.py | 10 +- platformio/clients/account.py | 28 +- platformio/clients/{rest.py => http.py} | 26 +- platformio/clients/registry.py | 56 ++- platformio/commands/package.py | 4 +- platformio/exception.py | 33 -- platformio/managers/package.py | 8 +- .../{downloader.py => package/download.py} | 59 ++- platformio/package/exception.py | 16 +- platformio/{ => package}/lockfile.py | 0 platformio/package/manager/__init__.py | 13 + platformio/package/manager/_download.py | 95 +++++ platformio/package/manager/_install.py | 282 +++++++++++++ platformio/package/manager/_registry.py | 190 +++++++++ platformio/package/manager/base.py | 233 +++++++++++ platformio/package/manager/library.py | 64 +++ platformio/package/manager/platform.py | 30 ++ platformio/package/manager/tool.py | 29 ++ platformio/package/meta.py | 382 ++++++++++++++++++ platformio/package/pack.py | 4 +- platformio/package/spec.py | 169 -------- platformio/{unpacker.py => package/unpack.py} | 76 ++-- platformio/{ => package}/vcsclient.py | 0 platformio/util.py | 2 +- tests/package/test_manager.py | 303 ++++++++++++++ tests/package/test_meta.py | 250 ++++++++++++ tests/package/test_spec.py | 119 ------ 27 files changed, 2074 insertions(+), 407 deletions(-) rename platformio/clients/{rest.py => http.py} (73%) rename platformio/{downloader.py => package/download.py} (67%) rename platformio/{ => package}/lockfile.py (100%) create mode 100644 platformio/package/manager/__init__.py create mode 100644 platformio/package/manager/_download.py create mode 100644 platformio/package/manager/_install.py create mode 100644 platformio/package/manager/_registry.py create mode 100644 platformio/package/manager/base.py create mode 100644 platformio/package/manager/library.py create mode 100644 platformio/package/manager/platform.py create mode 100644 platformio/package/manager/tool.py create mode 100644 platformio/package/meta.py delete mode 100644 platformio/package/spec.py rename platformio/{unpacker.py => package/unpack.py} (65%) rename platformio/{ => package}/vcsclient.py (100%) create mode 100644 tests/package/test_manager.py create mode 100644 tests/package/test_meta.py delete mode 100644 tests/package/test_spec.py diff --git a/platformio/app.py b/platformio/app.py index f4ae15b9..a1a4ee73 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,8 +25,9 @@ from time import time import requests -from platformio import __version__, exception, fs, lockfile, proc +from platformio import __version__, exception, fs, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data +from platformio.package.lockfile import LockFile from platformio.project.helpers import ( get_default_projects_dir, get_project_cache_dir, @@ -125,7 +126,7 @@ class State(object): def _lock_state_file(self): if not self.lock: return - self._lockfile = lockfile.LockFile(self.path) + self._lockfile = LockFile(self.path) try: self._lockfile.acquire() except IOError: @@ -143,6 +144,9 @@ class State(object): def as_dict(self): return self._storage + def keys(self): + return self._storage.keys() + def get(self, key, default=True): return self._storage.get(key, default) @@ -187,7 +191,7 @@ class ContentCache(object): def _lock_dbindex(self): if not self.cache_dir: os.makedirs(self.cache_dir) - self._lockfile = lockfile.LockFile(self.cache_dir) + self._lockfile = LockFile(self.cache_dir) try: self._lockfile.acquire() except: # pylint: disable=bare-except diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 31e34f78..c29ef9f9 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -16,7 +16,7 @@ import os import time from platformio import __accounts_api__, app -from platformio.clients.rest import RESTClient +from platformio.clients.http import HTTPClient from platformio.exception import PlatformioException @@ -35,7 +35,7 @@ class AccountAlreadyAuthorized(AccountError): MESSAGE = "You are already authorized with {0} account." -class AccountClient(RESTClient): # pylint:disable=too-many-public-methods +class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 @@ -67,7 +67,7 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods token = self.fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.send_request(*args, **kwargs) + return self.request_json_data(*args, **kwargs) def login(self, username, password): try: @@ -79,11 +79,11 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods app.get_state_item("account", {}).get("email", "") ) - result = self.send_request( + data = self.request_json_data( "post", "/v1/login", data={"username": username, "password": password}, ) - app.set_state_item("account", result) - return result + app.set_state_item("account", data) + return data def login_with_code(self, client_id, code, redirect_uri): try: @@ -95,7 +95,7 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods app.get_state_item("account", {}).get("email", "") ) - result = self.send_request( + result = self.request_json_data( "post", "/v1/login/code", data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, @@ -107,7 +107,7 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods refresh_token = self.get_refresh_token() self.delete_local_session() try: - self.send_request( + self.request_json_data( "post", "/v1/logout", data={"refresh_token": refresh_token}, ) except AccountError: @@ -133,7 +133,7 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods app.get_state_item("account", {}).get("email", "") ) - return self.send_request( + return self.request_json_data( "post", "/v1/registration", data={ @@ -153,7 +153,9 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods ).get("auth_token") def forgot_password(self, username): - return self.send_request("post", "/v1/forgot", data={"username": username},) + return self.request_json_data( + "post", "/v1/forgot", data={"username": username}, + ) def get_profile(self): return self.send_auth_request("get", "/v1/profile",) @@ -276,15 +278,15 @@ class AccountClient(RESTClient): # pylint:disable=too-many-public-methods return auth.get("access_token") if auth.get("refresh_token"): try: - result = self.send_request( + data = self.request_json_data( "post", "/v1/login", headers={ "Authorization": "Bearer %s" % auth.get("refresh_token") }, ) - app.set_state_item("account", result) - return result.get("auth").get("access_token") + app.set_state_item("account", data) + return data.get("auth").get("access_token") except AccountError: self.delete_local_session() raise AccountNotAuthorized() diff --git a/platformio/clients/rest.py b/platformio/clients/http.py similarity index 73% rename from platformio/clients/rest.py rename to platformio/clients/http.py index 4921e2cc..e1257762 100644 --- a/platformio/clients/rest.py +++ b/platformio/clients/http.py @@ -19,11 +19,11 @@ from platformio import app, util from platformio.exception import PlatformioException -class RESTClientError(PlatformioException): +class HTTPClientError(PlatformioException): pass -class RESTClient(object): +class HTTPClient(object): def __init__(self, base_url): if base_url.endswith("/"): base_url = base_url[:-1] @@ -33,19 +33,29 @@ class RESTClient(object): retry = Retry( total=5, backoff_factor=1, - method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], - status_forcelist=[500, 502, 503, 504], + # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], + status_forcelist=[413, 429, 500, 502, 503, 504], ) adapter = requests.adapters.HTTPAdapter(max_retries=retry) self._session.mount(base_url, adapter) + def __del__(self): + if not self._session: + return + self._session.close() + self._session = None + def send_request(self, method, path, **kwargs): - # check internet before and resolve issue with 60 seconds timeout + # check Internet before and resolve issue with 60 seconds timeout + # print(self, method, path, kwargs) util.internet_on(raise_exception=True) try: - response = getattr(self._session, method)(self.base_url + path, **kwargs) + return getattr(self._session, method)(self.base_url + path, **kwargs) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - raise RESTClientError(e) + raise HTTPClientError(e) + + def request_json_data(self, *args, **kwargs): + response = self.send_request(*args, **kwargs) return self.raise_error_from_response(response) @staticmethod @@ -59,4 +69,4 @@ class RESTClient(object): message = response.json()["message"] except (KeyError, ValueError): message = response.text - raise RESTClientError(message) + raise HTTPClientError(message) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 3d45584b..b7d724b9 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -14,13 +14,18 @@ from platformio import __registry_api__, fs from platformio.clients.account import AccountClient -from platformio.clients.rest import RESTClient -from platformio.package.spec import PackageType +from platformio.clients.http import HTTPClient +from platformio.package.meta import PackageType + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote # pylint: disable=too-many-arguments -class RegistryClient(RESTClient): +class RegistryClient(HTTPClient): def __init__(self): super(RegistryClient, self).__init__(base_url=__registry_api__) @@ -30,7 +35,7 @@ class RegistryClient(RESTClient): token = AccountClient().fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.send_request(*args, **kwargs) + return self.request_json_data(*args, **kwargs) def publish_package( self, archive_path, owner=None, released_at=None, private=False, notify=True @@ -41,7 +46,7 @@ class RegistryClient(RESTClient): account.get_account_info(offline=True).get("profile").get("username") ) with open(archive_path, "rb") as fp: - response = self.send_auth_request( + return self.send_auth_request( "post", "/v3/packages/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={ @@ -57,7 +62,6 @@ class RegistryClient(RESTClient): }, data=fp, ) - return response def unpublish_package( # pylint: disable=redefined-builtin self, type, name, owner=None, version=None, undo=False @@ -70,10 +74,9 @@ class RegistryClient(RESTClient): path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: path += "/" + version - response = self.send_auth_request( + return self.send_auth_request( "delete", path, params={"undo": 1 if undo else 0}, ) - return response def update_resource(self, urn, private): return self.send_auth_request( @@ -96,3 +99,40 @@ class RegistryClient(RESTClient): return self.send_auth_request( "get", "/v3/resources", params={"owner": owner} if owner else None ) + + def list_packages(self, query=None, filters=None, page=None): + assert query or filters + search_query = [] + if filters: + valid_filters = ( + "authors", + "keywords", + "frameworks", + "platforms", + "headers", + "ids", + "names", + "owners", + "types", + ) + assert set(filters.keys()) <= set(valid_filters) + for name, values in filters.items(): + for value in set( + values if isinstance(values, (list, tuple)) else [values] + ): + search_query.append("%s:%s" % (name[:-1], value)) + if query: + search_query.append(query) + params = dict(query=quote(" ".join(search_query))) + if page: + params["page"] = int(page) + return self.request_json_data("get", "/v3/packages", params=params) + + def get_package(self, type_, owner, name, version=None): + return self.request_json_data( + "get", + "/v3/packages/{owner}/{type}/{name}".format( + type=type_, owner=owner, name=quote(name) + ), + params=dict(version=version) if version else None, + ) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 73eb551a..6ec78d38 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -18,8 +18,8 @@ from datetime import datetime import click from platformio.clients.registry import RegistryClient +from platformio.package.meta import PackageSpec, PackageType from platformio.package.pack import PackagePacker -from platformio.package.spec import PackageSpec, PackageType def validate_datetime(ctx, param, value): # pylint: disable=unused-argument @@ -106,7 +106,7 @@ def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin response = RegistryClient().unpublish_package( type=type, name=spec.name, - owner=spec.ownername, + owner=spec.owner, version=spec.requirements, undo=undo, ) diff --git a/platformio/exception.py b/platformio/exception.py index d291ad7f..c39b7957 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -119,39 +119,6 @@ class PackageInstallError(PlatformIOPackageException): ) -class ExtractArchiveItemError(PlatformIOPackageException): - - MESSAGE = ( - "Could not extract `{0}` to `{1}`. Try to disable antivirus " - "tool or check this solution -> http://bit.ly/faq-package-manager" - ) - - -class UnsupportedArchiveType(PlatformIOPackageException): - - MESSAGE = "Can not unpack file '{0}'" - - -class FDUnrecognizedStatusCode(PlatformIOPackageException): - - MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}" - - -class FDSizeMismatch(PlatformIOPackageException): - - MESSAGE = ( - "The size ({0:d} bytes) of downloaded file '{1}' " - "is not equal to remote size ({2:d} bytes)" - ) - - -class FDSHASumMismatch(PlatformIOPackageException): - - MESSAGE = ( - "The 'sha1' sum '{0}' of downloaded file '{1}' is not equal to remote '{2}'" - ) - - # # Library # diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 92ba4515..346cce59 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -26,12 +26,12 @@ import semantic_version from platformio import __version__, app, exception, fs, util from platformio.compat import hashlib_encode_data -from platformio.downloader import FileDownloader -from platformio.lockfile import LockFile +from platformio.package.download import FileDownloader from platformio.package.exception import ManifestException +from platformio.package.lockfile import LockFile from platformio.package.manifest.parser import ManifestParserFactory -from platformio.unpacker import FileUnpacker -from platformio.vcsclient import VCSClientFactory +from platformio.package.unpack import FileUnpacker +from platformio.package.vcsclient import VCSClientFactory # pylint: disable=too-many-arguments, too-many-return-statements diff --git a/platformio/downloader.py b/platformio/package/download.py similarity index 67% rename from platformio/downloader.py rename to platformio/package/download.py index ccbc5b36..3c723c4b 100644 --- a/platformio/downloader.py +++ b/platformio/package/download.py @@ -23,11 +23,7 @@ import click import requests from platformio import app, fs, util -from platformio.exception import ( - FDSHASumMismatch, - FDSizeMismatch, - FDUnrecognizedStatusCode, -) +from platformio.package.exception import PackageException class FileDownloader(object): @@ -41,7 +37,11 @@ class FileDownloader(object): verify=sys.version_info >= (2, 7, 9), ) if self._request.status_code != 200: - raise FDUnrecognizedStatusCode(self._request.status_code, url) + raise PackageException( + "Got the unrecognized status code '{0}' when downloaded {1}".format( + self._request.status_code, url + ) + ) disposition = self._request.headers.get("content-disposition") if disposition and "filename=" in disposition: @@ -74,21 +74,21 @@ class FileDownloader(object): def start(self, with_progress=True, silent=False): label = "Downloading" itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) - f = open(self._destination, "wb") + fp = open(self._destination, "wb") try: if not with_progress or self.get_size() == -1: if not silent: click.echo("%s..." % label) for chunk in itercontent: if chunk: - f.write(chunk) + fp.write(chunk) else: chunks = int(math.ceil(self.get_size() / float(io.DEFAULT_BUFFER_SIZE))) with click.progressbar(length=chunks, label=label) as pb: for _ in pb: - f.write(next(itercontent)) + fp.write(next(itercontent)) finally: - f.close() + fp.close() self._request.close() if self.get_lmtime(): @@ -96,15 +96,40 @@ class FileDownloader(object): return True - def verify(self, sha1=None): + def verify(self, checksum=None): _dlsize = getsize(self._destination) if self.get_size() != -1 and _dlsize != self.get_size(): - raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) - if not sha1: - return None - checksum = fs.calculate_file_hashsum("sha1", self._destination) - if sha1.lower() != checksum.lower(): - raise FDSHASumMismatch(checksum, self._fname, sha1) + raise PackageException( + ( + "The size ({0:d} bytes) of downloaded file '{1}' " + "is not equal to remote size ({2:d} bytes)" + ).format(_dlsize, self._fname, self.get_size()) + ) + if not checksum: + return True + + checksum_len = len(checksum) + hash_algo = None + if checksum_len == 32: + hash_algo = "md5" + elif checksum_len == 40: + hash_algo = "sha1" + elif checksum_len == 64: + hash_algo = "sha256" + + if not hash_algo: + raise PackageException( + "Could not determine checksum algorithm by %s" % checksum + ) + + dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination) + if checksum.lower() != dl_checksum.lower(): + raise PackageException( + "The checksum '{0}' of the downloaded file '{1}' " + "does not match to the remote '{2}'".format( + dl_checksum, self._fname, checksum + ) + ) return True def _preserve_filemtime(self, lmdate): diff --git a/platformio/package/exception.py b/platformio/package/exception.py index adadc088..f32c89ce 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.exception import PlatformioException +from platformio import util +from platformio.exception import PlatformioException, UserSideException class PackageException(PlatformioException): @@ -44,3 +45,16 @@ class ManifestValidationError(ManifestException): "https://docs.platformio.org/page/librarymanager/config.html" % self.messages ) + + +class MissingPackageManifestError(ManifestException): + + MESSAGE = "Could not find one of '{0}' manifest files in the package" + + +class UnknownPackageError(UserSideException): + + MESSAGE = ( + "Could not find a package with '{0}' requirements for your system '%s'" + % util.get_systype() + ) diff --git a/platformio/lockfile.py b/platformio/package/lockfile.py similarity index 100% rename from platformio/lockfile.py rename to platformio/package/lockfile.py diff --git a/platformio/package/manager/__init__.py b/platformio/package/manager/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/package/manager/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py new file mode 100644 index 00000000..a052da09 --- /dev/null +++ b/platformio/package/manager/_download.py @@ -0,0 +1,95 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import tempfile +import time + +import click + +from platformio import app, compat +from platformio.package.download import FileDownloader +from platformio.package.lockfile import LockFile + + +class PackageManagerDownloadMixin(object): + + DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month + + def compute_download_path(self, *args): + request_hash = hashlib.new("sha256") + for arg in args: + request_hash.update(compat.hashlib_encode_data(arg)) + dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest()) + return dl_path + + def get_download_usagedb_path(self): + return os.path.join(self.get_download_dir(), "usage.db") + + def set_download_utime(self, path, utime=None): + with app.State(self.get_download_usagedb_path(), lock=True) as state: + state[os.path.basename(path)] = int(time.time() if not utime else utime) + + def cleanup_expired_downloads(self): + with app.State(self.get_download_usagedb_path(), lock=True) as state: + # remove outdated + for fname in list(state.keys()): + if state[fname] > (time.time() - self.DOWNLOAD_CACHE_EXPIRE): + continue + del state[fname] + dl_path = os.path.join(self.get_download_dir(), fname) + if os.path.isfile(dl_path): + os.remove(dl_path) + + def download(self, url, checksum=None, silent=False): + dl_path = self.compute_download_path(url, checksum or "") + if os.path.isfile(dl_path): + self.set_download_utime(dl_path) + return dl_path + + with_progress = not silent and not app.is_disabled_progressbar() + tmp_path = tempfile.mkstemp(dir=self.get_download_dir())[1] + try: + with LockFile(dl_path): + try: + fd = FileDownloader(url) + fd.set_destination(tmp_path) + fd.start(with_progress=with_progress, silent=silent) + except IOError as e: + raise_error = not with_progress + if with_progress: + try: + fd = FileDownloader(url) + fd.set_destination(tmp_path) + fd.start(with_progress=False, silent=silent) + except IOError: + raise_error = True + if raise_error: + click.secho( + "Error: Please read http://bit.ly/package-manager-ioerror", + fg="red", + err=True, + ) + raise e + if checksum: + fd.verify(checksum) + os.rename(tmp_path, dl_path) + finally: + if os.path.isfile(tmp_path): + os.remove(tmp_path) + + assert os.path.isfile(dl_path) + self.set_download_utime(dl_path) + return dl_path diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py new file mode 100644 index 00000000..a5aa1c38 --- /dev/null +++ b/platformio/package/manager/_install.py @@ -0,0 +1,282 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import shutil +import tempfile + +import click + +from platformio import app, compat, fs, util +from platformio.package.exception import PackageException, UnknownPackageError +from platformio.package.lockfile import LockFile +from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.unpack import FileUnpacker +from platformio.package.vcsclient import VCSClientFactory + + +class PackageManagerInstallMixin(object): + + INSTALL_HISTORY = None # avoid circle dependencies + + @staticmethod + def unpack(src, dst): + with_progress = not app.is_disabled_progressbar() + try: + with FileUnpacker(src) as fu: + return fu.unpack(dst, with_progress=with_progress) + except IOError as e: + if not with_progress: + raise e + with FileUnpacker(src) as fu: + return fu.unpack(dst, with_progress=False) + + def install(self, spec, silent=False): + with LockFile(self.package_dir): + pkg = self._install(spec, silent=silent) + self.memcache_reset() + self.cleanup_expired_downloads() + return pkg + + def _install(self, spec, search_filters=None, silent=False): + spec = self.ensure_spec(spec) + + # avoid circle dependencies + if not self.INSTALL_HISTORY: + self.INSTALL_HISTORY = [] + if spec in self.INSTALL_HISTORY: + return None + self.INSTALL_HISTORY.append(spec) + + # check if package is already installed + pkg = self.get_package(spec) + if pkg: + if not silent: + click.secho( + "{name} @ {version} is already installed".format( + **pkg.metadata.as_dict() + ), + fg="yellow", + ) + return pkg + + if not silent: + msg = "Installing %s" % click.style(spec.humanize(), fg="cyan") + self.print_message(msg) + + if spec.url: + pkg = self.install_from_url(spec.url, spec, silent=silent) + else: + pkg = self.install_from_registry(spec, search_filters, silent=silent) + + if not pkg or not pkg.metadata: + raise PackageException( + "Could not install package '%s' for '%s' system" + % (spec.humanize(), util.get_systype()) + ) + + if not silent: + self.print_message( + click.style( + "{name} @ {version} has been successfully installed!".format( + **pkg.metadata.as_dict() + ), + fg="green", + ) + ) + + self.memcache_reset() + self.install_dependencies(pkg, silent) + return pkg + + def install_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageSourceItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message(click.style("Installing dependencies...", fg="yellow")) + for dependency in manifest.get("dependencies"): + if not self.install_dependency(dependency, silent) and not silent: + click.secho( + "Warning! Could not install dependency %s for package '%s'" + % (dependency, pkg.metadata.name), + fg="yellow", + ) + + def install_dependency(self, dependency, silent=False): + spec = PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + search_filters = { + key: value + for key, value in dependency.items() + if key in ("authors", "platforms", "frameworks") + } + return self._install(spec, search_filters=search_filters or None, silent=silent) + + def install_from_url(self, url, spec, checksum=None, silent=False): + spec = self.ensure_spec(spec) + tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir()) + vcs = None + try: + if url.startswith("file://"): + _url = url[7:] + if os.path.isfile(_url): + self.unpack(_url, tmp_dir) + else: + fs.rmtree(tmp_dir) + shutil.copytree(_url, tmp_dir, symlinks=True) + elif url.startswith(("http://", "https://")): + dl_path = self.download(url, checksum, silent=silent) + assert os.path.isfile(dl_path) + self.unpack(dl_path, tmp_dir) + else: + vcs = VCSClientFactory.newClient(tmp_dir, url) + assert vcs.export() + + root_dir = self.find_pkg_root(tmp_dir, spec) + pkg_item = PackageSourceItem( + root_dir, + self.build_metadata( + root_dir, spec, vcs.get_current_revision() if vcs else None + ), + ) + pkg_item.dump_meta() + return self._install_tmp_pkg(pkg_item) + finally: + if os.path.isdir(tmp_dir): + fs.rmtree(tmp_dir) + + def _install_tmp_pkg(self, tmp_pkg): + assert isinstance(tmp_pkg, PackageSourceItem) + # validate package version and declared requirements + if ( + tmp_pkg.metadata.spec.requirements + and tmp_pkg.metadata.version not in tmp_pkg.metadata.spec.requirements + ): + raise PackageException( + "Package version %s doesn't satisfy requirements %s based on %s" + % ( + tmp_pkg.metadata.version, + tmp_pkg.metadata.spec.requirements, + tmp_pkg.metadata, + ) + ) + dst_pkg = PackageSourceItem( + os.path.join(self.package_dir, tmp_pkg.get_safe_dirname()) + ) + + # what to do with existing package? + action = "overwrite" + if dst_pkg.metadata and dst_pkg.metadata.spec.url: + if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: + action = "detach-existing" + elif tmp_pkg.metadata.spec.url: + action = "detach-new" + elif dst_pkg.metadata and dst_pkg.metadata.version != tmp_pkg.metadata.version: + action = ( + "detach-existing" + if tmp_pkg.metadata.version > dst_pkg.metadata.version + else "detach-new" + ) + + def _cleanup_dir(path): + if os.path.isdir(path): + fs.rmtree(path) + + if action == "detach-existing": + target_dirname = "%s@%s" % ( + tmp_pkg.get_safe_dirname(), + dst_pkg.metadata.version, + ) + if dst_pkg.metadata.spec.url: + target_dirname = "%s@src-%s" % ( + tmp_pkg.get_safe_dirname(), + hashlib.md5( + compat.hashlib_encode_data(dst_pkg.metadata.spec.url) + ).hexdigest(), + ) + # move existing into the new place + pkg_dir = os.path.join(self.package_dir, target_dirname) + _cleanup_dir(pkg_dir) + shutil.move(dst_pkg.path, pkg_dir) + # move new source to the destination location + _cleanup_dir(dst_pkg.path) + shutil.move(tmp_pkg.path, dst_pkg.path) + return PackageSourceItem(dst_pkg.path) + + if action == "detach-new": + target_dirname = "%s@%s" % ( + tmp_pkg.get_safe_dirname(), + tmp_pkg.metadata.version, + ) + if tmp_pkg.metadata.spec.url: + target_dirname = "%s@src-%s" % ( + tmp_pkg.get_safe_dirname(), + hashlib.md5( + compat.hashlib_encode_data(tmp_pkg.metadata.spec.url) + ).hexdigest(), + ) + pkg_dir = os.path.join(self.package_dir, target_dirname) + _cleanup_dir(pkg_dir) + shutil.move(tmp_pkg.path, pkg_dir) + return PackageSourceItem(pkg_dir) + + # otherwise, overwrite existing + _cleanup_dir(dst_pkg.path) + shutil.move(tmp_pkg.path, dst_pkg.path) + return PackageSourceItem(dst_pkg.path) + + def uninstall(self, path_or_spec, silent=False): + with LockFile(self.package_dir): + pkg = ( + PackageSourceItem(path_or_spec) + if os.path.isdir(path_or_spec) + else self.get_package(path_or_spec) + ) + if not pkg or not pkg.metadata: + raise UnknownPackageError(path_or_spec) + + if not silent: + self.print_message( + "Uninstalling %s @ %s: \t" + % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), + nl=False, + ) + if os.path.islink(pkg.path): + os.unlink(pkg.path) + else: + fs.rmtree(pkg.path) + self.memcache_reset() + + # unfix detached-package with the same name + detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) + if ( + detached_pkg + and "@" in detached_pkg.path + and not os.path.isdir( + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) + ) + ): + shutil.move( + detached_pkg.path, + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), + ) + self.memcache_reset() + + if not silent: + click.echo("[%s]" % click.style("OK", fg="green")) + return True diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py new file mode 100644 index 00000000..3f3ae813 --- /dev/null +++ b/platformio/package/manager/_registry.py @@ -0,0 +1,190 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import click + +from platformio.clients.http import HTTPClient +from platformio.clients.registry import RegistryClient +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageMetaData, PackageSpec + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class RegistryFileMirrorsIterator(object): + + HTTP_CLIENT_INSTANCES = {} + + def __init__(self, download_url): + self.download_url = download_url + self._url_parts = urlparse(download_url) + self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) + self._visited_mirrors = [] + + def __iter__(self): + return self + + def __next__(self): + http = self.get_http_client() + response = http.send_request( + "head", + self._url_parts.path, + allow_redirects=False, + params=dict(bypass=",".join(self._visited_mirrors)) + if self._visited_mirrors + else None, + ) + stop_conditions = [ + response.status_code not in (302, 307), + not response.headers.get("Location"), + not response.headers.get("X-PIO-Mirror"), + response.headers.get("X-PIO-Mirror") in self._visited_mirrors, + ] + if any(stop_conditions): + raise StopIteration + self._visited_mirrors.append(response.headers.get("X-PIO-Mirror")) + return ( + response.headers.get("Location"), + response.headers.get("X-PIO-Content-SHA256"), + ) + + def get_http_client(self): + if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES: + RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[ + self._base_url + ] = HTTPClient(self._base_url) + return RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[self._base_url] + + +class PackageManageRegistryMixin(object): + def install_from_registry(self, spec, search_filters=None, silent=False): + packages = self.search_registry_packages(spec, search_filters) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + package, version = self.find_best_registry_version(packages, spec) + pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None + if not pkgfile: + raise UnknownPackageError(spec.humanize()) + + for url, checksum in RegistryFileMirrorsIterator(pkgfile["download_url"]): + try: + return self.install_from_url( + url, + PackageSpec( + owner=package["owner"]["username"], + id=package["id"], + name=package["name"], + ), + checksum or pkgfile["checksum"]["sha256"], + silent=silent, + ) + except Exception as e: # pylint: disable=broad-except + click.secho("Warning! Package Mirror: %s" % e, fg="yellow") + click.secho("Looking for another mirror...", fg="yellow") + + return None + + def get_registry_client_instance(self): + if not self._registry_client: + self._registry_client = RegistryClient() + return self._registry_client + + def search_registry_packages(self, spec, filters=None): + filters = filters or {} + if spec.id: + filters["ids"] = str(spec.id) + else: + filters["types"] = self.pkg_type + filters["names"] = '"%s"' % spec.name.lower() + if spec.owner: + filters["owners"] = spec.owner.lower() + return self.get_registry_client_instance().list_packages(filters=filters)[ + "items" + ] + + def fetch_registry_package_versions(self, owner, name): + return self.get_registry_client_instance().get_package( + self.pkg_type, owner, name + )["versions"] + + @staticmethod + def print_multi_package_issue(packages, spec): + click.secho( + "Warning! More than one package has been found by ", fg="yellow", nl=False + ) + click.secho(spec.humanize(), fg="cyan", nl=False) + click.secho(" requirements:", fg="yellow") + for item in packages: + click.echo( + " - {owner}/{name} @ {version}".format( + owner=click.style(item["owner"]["username"], fg="cyan"), + name=item["name"], + version=item["version"]["name"], + ) + ) + click.secho( + "Please specify detailed REQUIREMENTS using package owner and version " + "(showed above) to avoid project compatibility issues.", + fg="yellow", + ) + + def find_best_registry_version(self, packages, spec): + # find compatible version within the latest package versions + for package in packages: + version = self._pick_best_pkg_version([package["version"]], spec) + if version: + return (package, version) + + if not spec.requirements: + return None + + # if the custom version requirements, check ALL package versions + for package in packages: + version = self._pick_best_pkg_version( + self.fetch_registry_package_versions( + package["owner"]["username"], package["name"] + ), + spec, + ) + if version: + return (package, version) + time.sleep(1) + return None + + def _pick_best_pkg_version(self, versions, spec): + best = None + for version in versions: + semver = PackageMetaData.to_semver(version["name"]) + if spec.requirements and semver not in spec.requirements: + continue + if not any( + self.is_system_compatible(f.get("system")) for f in version["files"] + ): + continue + if not best or (semver > PackageMetaData.to_semver(best["name"])): + best = version + return best + + def _pick_compatible_pkg_file(self, version_files): + for item in version_files: + if self.is_system_compatible(item.get("system")): + return item + return None diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py new file mode 100644 index 00000000..e48fc250 --- /dev/null +++ b/platformio/package/manager/base.py @@ -0,0 +1,233 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +import click +import semantic_version + +from platformio import fs, util +from platformio.commands import PlatformioCLI +from platformio.package.exception import ManifestException, MissingPackageManifestError +from platformio.package.manager._download import PackageManagerDownloadMixin +from platformio.package.manager._install import PackageManagerInstallMixin +from platformio.package.manager._registry import PackageManageRegistryMixin +from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.meta import ( + PackageMetaData, + PackageSourceItem, + PackageSpec, + PackageType, +) +from platformio.project.helpers import get_project_cache_dir + + +class BasePackageManager( + PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin +): + MEMORY_CACHE = {} + + def __init__(self, pkg_type, package_dir): + self.pkg_type = pkg_type + self.package_dir = self.ensure_dir_exists(package_dir) + self.MEMORY_CACHE = {} + self._download_dir = None + self._tmp_dir = None + self._registry_client = None + + def memcache_get(self, key, default=None): + return self.MEMORY_CACHE.get(key, default) + + def memcache_set(self, key, value): + self.MEMORY_CACHE[key] = value + + def memcache_reset(self): + self.MEMORY_CACHE.clear() + + @staticmethod + def is_system_compatible(value): + if not value or "*" in value: + return True + return util.items_in_list(value, util.get_systype()) + + @staticmethod + def generate_rand_version(): + return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") + + @staticmethod + def ensure_dir_exists(path): + if not os.path.isdir(path): + os.makedirs(path) + assert os.path.isdir(path) + return path + + @staticmethod + def ensure_spec(spec): + return spec if isinstance(spec, PackageSpec) else PackageSpec(spec) + + @property + def manifest_names(self): + raise NotImplementedError + + def print_message(self, message, nl=True): + click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) + + def get_download_dir(self): + if not self._download_dir: + self._download_dir = self.ensure_dir_exists( + os.path.join(get_project_cache_dir(), "downloads") + ) + return self._download_dir + + def get_tmp_dir(self): + if not self._tmp_dir: + self._tmp_dir = self.ensure_dir_exists( + os.path.join(get_project_cache_dir(), "tmp") + ) + return self._tmp_dir + + def find_pkg_root(self, path, spec): # pylint: disable=unused-argument + if self.manifest_exists(path): + return path + for root, _, _ in os.walk(path): + if self.manifest_exists(root): + return root + raise MissingPackageManifestError(", ".join(self.manifest_names)) + + def get_manifest_path(self, pkg_dir): + if not os.path.isdir(pkg_dir): + return None + for name in self.manifest_names: + manifest_path = os.path.join(pkg_dir, name) + if os.path.isfile(manifest_path): + return manifest_path + return None + + def manifest_exists(self, pkg_dir): + return self.get_manifest_path(pkg_dir) + + def load_manifest(self, src): + path = src.path if isinstance(src, PackageSourceItem) else src + cache_key = "load_manifest-%s" % path + result = self.memcache_get(cache_key) + if result: + return result + candidates = ( + [os.path.join(path, name) for name in self.manifest_names] + if os.path.isdir(path) + else [path] + ) + for item in candidates: + if not os.path.isfile(item): + continue + try: + result = ManifestParserFactory.new_from_file(item).as_dict() + self.memcache_set(cache_key, result) + return result + except ManifestException as e: + if not PlatformioCLI.in_silence(): + click.secho(str(e), fg="yellow") + raise MissingPackageManifestError(", ".join(self.manifest_names)) + + def build_legacy_spec(self, pkg_dir): + # find src manifest + src_manifest_name = ".piopkgmanager.json" + src_manifest_path = None + for name in os.listdir(pkg_dir): + if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): + continue + src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) + break + + if src_manifest_path: + src_manifest = fs.load_json(src_manifest_path) + return PackageSpec( + name=src_manifest.get("name"), + url=src_manifest.get("url"), + requirements=src_manifest.get("requirements"), + ) + + # fall back to a package manifest + manifest = self.load_manifest(pkg_dir) + return PackageSpec(name=manifest.get("name")) + + def build_metadata(self, pkg_dir, spec, vcs_revision=None): + manifest = self.load_manifest(pkg_dir) + metadata = PackageMetaData( + type=self.pkg_type, + name=manifest.get("name"), + version=manifest.get("version"), + spec=spec, + ) + if not metadata.name or spec.is_custom_name(): + metadata.name = spec.name + if vcs_revision: + metadata.version = "%s+sha.%s" % ( + metadata.version if metadata.version else "0.0.0", + vcs_revision, + ) + if not metadata.version: + metadata.version = self.generate_rand_version() + return metadata + + def get_installed(self): + result = [] + for name in os.listdir(self.package_dir): + pkg_dir = os.path.join(self.package_dir, name) + if not os.path.isdir(pkg_dir): + continue + pkg = PackageSourceItem(pkg_dir) + if not pkg.metadata: + try: + spec = self.build_legacy_spec(pkg_dir) + pkg.metadata = self.build_metadata(pkg_dir, spec) + except MissingPackageManifestError: + pass + if pkg.metadata: + result.append(pkg) + return result + + def get_package(self, spec): + def _ci_strings_are_equal(a, b): + if a == b: + return True + if not a or not b: + return False + return a.strip().lower() == b.strip().lower() + + spec = self.ensure_spec(spec) + best = None + for pkg in self.get_installed(): + skip_conditions = [ + spec.owner + and not _ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner), + spec.url and spec.url != pkg.metadata.spec.url, + spec.id and spec.id != pkg.metadata.spec.id, + not spec.id + and not spec.url + and not _ci_strings_are_equal(spec.name, pkg.metadata.name), + ] + if any(skip_conditions): + continue + if self.pkg_type == PackageType.TOOL: + # TODO: check "system" for pkg + pass + + assert isinstance(pkg.metadata.version, semantic_version.Version) + if spec.requirements and pkg.metadata.version not in spec.requirements: + continue + if not best or (pkg.metadata.version > best.metadata.version): + best = pkg + return best diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py new file mode 100644 index 00000000..9fe924b9 --- /dev/null +++ b/platformio/package/manager/library.py @@ -0,0 +1,64 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from platformio.package.exception import MissingPackageManifestError +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageSpec, PackageType +from platformio.project.helpers import get_project_global_lib_dir + + +class LibraryPackageManager(BasePackageManager): + def __init__(self, package_dir=None): + super(LibraryPackageManager, self).__init__( + PackageType.LIBRARY, package_dir or get_project_global_lib_dir() + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.LIBRARY] + + def find_pkg_root(self, path, spec): + try: + return super(LibraryPackageManager, self).find_pkg_root(path, spec) + except MissingPackageManifestError: + pass + assert isinstance(spec, PackageSpec) + + root_dir = self.find_library_root(path) + + # automatically generate library manifest + with open(os.path.join(root_dir, "library.json"), "w") as fp: + json.dump( + dict(name=spec.name, version=self.generate_rand_version(),), + fp, + indent=2, + ) + + return root_dir + + @staticmethod + def find_library_root(path): + for root, dirs, files in os.walk(path): + if not files and len(dirs) == 1: + continue + for fname in files: + if not fname.endswith((".c", ".cpp", ".h", ".S")): + continue + if os.path.isdir(os.path.join(os.path.dirname(root), "src")): + return os.path.dirname(root) + return root + return path diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py new file mode 100644 index 00000000..627bad47 --- /dev/null +++ b/platformio/package/manager/platform.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageType +from platformio.project.config import ProjectConfig + + +class PlatformPackageManager(BasePackageManager): + def __init__(self, package_dir=None): + self.config = ProjectConfig.get_instance() + super(PlatformPackageManager, self).__init__( + PackageType.PLATFORM, + package_dir or self.config.get_optional_dir("platforms"), + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.PLATFORM] diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py new file mode 100644 index 00000000..db660303 --- /dev/null +++ b/platformio/package/manager/tool.py @@ -0,0 +1,29 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageType +from platformio.project.config import ProjectConfig + + +class ToolPackageManager(BasePackageManager): + def __init__(self, package_dir=None): + self.config = ProjectConfig.get_instance() + super(ToolPackageManager, self).__init__( + PackageType.TOOL, package_dir or self.config.get_optional_dir("packages"), + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.TOOL] diff --git a/platformio/package/meta.py b/platformio/package/meta.py new file mode 100644 index 00000000..0f1214e1 --- /dev/null +++ b/platformio/package/meta.py @@ -0,0 +1,382 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import tarfile + +import semantic_version + +from platformio.compat import get_object_members, string_types +from platformio.package.manifest.parser import ManifestFileType + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class PackageType(object): + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r:gz") as tf: + for t in sorted(cls.items().values()): + for manifest in manifest_map[t]: + try: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + + +class PackageSpec(object): + def __init__( # pylint: disable=redefined-builtin,too-many-arguments + self, raw=None, owner=None, id=None, name=None, requirements=None, url=None + ): + self.owner = owner + self.id = id + self.name = name + self._requirements = None + self.url = url + if requirements: + self.requirements = requirements + self._name_is_custom = False + self._parse(raw) + + def __eq__(self, other): + return all( + [ + self.owner == other.owner, + self.id == other.id, + self.name == other.name, + self.requirements == other.requirements, + self.url == other.url, + ] + ) + + def __repr__(self): + return ( + "PackageSpec ".format(**self.as_dict()) + ) + + @property + def requirements(self): + return self._requirements + + @requirements.setter + def requirements(self, value): + if not value: + self._requirements = None + return + self._requirements = ( + value + if isinstance(value, semantic_version.SimpleSpec) + else semantic_version.SimpleSpec(value) + ) + + def humanize(self): + if self.url: + result = self.url + elif self.id: + result = "id:%d" % self.id + else: + result = "" + if self.owner: + result = self.owner + "/" + result += self.name + if self.requirements: + result += " @ " + str(self.requirements) + return result + + def is_custom_name(self): + return self._name_is_custom + + def as_dict(self): + return dict( + owner=self.owner, + id=self.id, + name=self.name, + requirements=str(self.requirements) if self.requirements else None, + url=self.url, + ) + + def _parse(self, raw): + if raw is None: + return + if not isinstance(raw, string_types): + raw = str(raw) + raw = raw.strip() + + parsers = ( + self._parse_requirements, + self._parse_custom_name, + self._parse_id, + self._parse_owner, + self._parse_url, + ) + for parser in parsers: + if raw is None: + break + raw = parser(raw) + + # if name is not custom, parse it from URL + if not self.name and self.url: + self.name = self._parse_name_from_url(self.url) + elif raw: + # the leftover is a package name + self.name = raw + + def _parse_requirements(self, raw): + if "@" not in raw: + return raw + tokens = raw.rsplit("@", 1) + if any(s in tokens[1] for s in (":", "/")): + return raw + self.requirements = tokens[1].strip() + return tokens[0].strip() + + def _parse_custom_name(self, raw): + if "=" not in raw or raw.startswith("id="): + return raw + tokens = raw.split("=", 1) + if "/" in tokens[0]: + return raw + self.name = tokens[0].strip() + self._name_is_custom = True + return tokens[1].strip() + + def _parse_id(self, raw): + if raw.isdigit(): + self.id = int(raw) + return None + if raw.startswith("id="): + return self._parse_id(raw[3:]) + return raw + + def _parse_owner(self, raw): + if raw.count("/") != 1 or "@" in raw: + return raw + tokens = raw.split("/", 1) + self.owner = tokens[0].strip() + self.name = tokens[1].strip() + return None + + def _parse_url(self, raw): + if not any(s in raw for s in ("@", ":", "/")): + return raw + self.url = raw.strip() + parts = urlparse(self.url) + + # if local file or valid URL with scheme vcs+protocol:// + if parts.scheme == "file" or "+" in parts.scheme or self.url.startswith("git+"): + return None + + # parse VCS + git_conditions = [ + parts.path.endswith(".git"), + # Handle GitHub URL (https://github.com/user/package) + parts.netloc in ("github.com", "gitlab.com", "bitbucket.com") + and not parts.path.endswith((".zip", ".tar.gz")), + ] + hg_conditions = [ + # Handle Developer Mbed URL + # (https://developer.mbed.org/users/user/code/package/) + # (https://os.mbed.com/users/user/code/package/) + parts.netloc + in ("mbed.com", "os.mbed.com", "developer.mbed.org") + ] + if any(git_conditions): + self.url = "git+" + self.url + elif any(hg_conditions): + self.url = "hg+" + self.url + + return None + + @staticmethod + def _parse_name_from_url(url): + if url.endswith("/"): + url = url[:-1] + for c in ("#", "?"): + if c in url: + url = url[: url.index(c)] + + # parse real repository name from Github + parts = urlparse(url) + if parts.netloc == "github.com" and parts.path.count("/") > 2: + return parts.path.split("/")[2] + + name = os.path.basename(url) + if "." in name: + return name.split(".", 1)[0].strip() + return name + + +class PackageMetaData(object): + def __init__( # pylint: disable=redefined-builtin + self, type, name, version, spec=None + ): + assert type in PackageType.items().values() + if spec: + assert isinstance(spec, PackageSpec) + self.type = type + self.name = name + self._version = None + self.version = version + self.spec = spec + + def __repr__(self): + return ( + "PackageMetaData -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import tarfile - -from platformio.compat import get_object_members, string_types -from platformio.package.manifest.parser import ManifestFileType - - -class PackageType(object): - LIBRARY = "library" - PLATFORM = "platform" - TOOL = "tool" - - @classmethod - def items(cls): - return get_object_members(cls) - - @classmethod - def get_manifest_map(cls): - return { - cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), - cls.LIBRARY: ( - ManifestFileType.LIBRARY_JSON, - ManifestFileType.LIBRARY_PROPERTIES, - ManifestFileType.MODULE_JSON, - ), - cls.TOOL: (ManifestFileType.PACKAGE_JSON,), - } - - @classmethod - def from_archive(cls, path): - assert path.endswith("tar.gz") - manifest_map = cls.get_manifest_map() - with tarfile.open(path, mode="r:gz") as tf: - for t in sorted(cls.items().values()): - for manifest in manifest_map[t]: - try: - if tf.getmember(manifest): - return t - except KeyError: - pass - return None - - -class PackageSpec(object): - def __init__( # pylint: disable=redefined-builtin,too-many-arguments - self, raw=None, ownername=None, id=None, name=None, requirements=None, url=None - ): - self.ownername = ownername - self.id = id - self.name = name - self.requirements = requirements - self.url = url - - self._parse(raw) - - def __repr__(self): - return ( - "PackageSpec ".format( - ownername=self.ownername, - id=self.id, - name=self.name, - requirements=self.requirements, - url=self.url, - ) - ) - - def __eq__(self, other): - return all( - [ - self.ownername == other.ownername, - self.id == other.id, - self.name == other.name, - self.requirements == other.requirements, - self.url == other.url, - ] - ) - - def _parse(self, raw): - if raw is None: - return - if not isinstance(raw, string_types): - raw = str(raw) - raw = raw.strip() - - parsers = ( - self._parse_requirements, - self._parse_fixed_name, - self._parse_id, - self._parse_ownername, - self._parse_url, - ) - for parser in parsers: - if raw is None: - break - raw = parser(raw) - - # if name is not fixed, parse it from URL - if not self.name and self.url: - self.name = self._parse_name_from_url(self.url) - elif raw: - # the leftover is a package name - self.name = raw - - def _parse_requirements(self, raw): - if "@" not in raw: - return raw - tokens = raw.rsplit("@", 1) - if any(s in tokens[1] for s in (":", "/")): - return raw - self.requirements = tokens[1].strip() - return tokens[0].strip() - - def _parse_fixed_name(self, raw): - if "=" not in raw or raw.startswith("id="): - return raw - tokens = raw.split("=", 1) - if "/" in tokens[0]: - return raw - self.name = tokens[0].strip() - return tokens[1].strip() - - def _parse_id(self, raw): - if raw.isdigit(): - self.id = int(raw) - return None - if raw.startswith("id="): - return self._parse_id(raw[3:]) - return raw - - def _parse_ownername(self, raw): - if raw.count("/") != 1 or "@" in raw: - return raw - tokens = raw.split("/", 1) - self.ownername = tokens[0].strip() - self.name = tokens[1].strip() - return None - - def _parse_url(self, raw): - if not any(s in raw for s in ("@", ":", "/")): - return raw - self.url = raw.strip() - return None - - @staticmethod - def _parse_name_from_url(url): - if url.endswith("/"): - url = url[:-1] - for c in ("#", "?"): - if c in url: - url = url[: url.index(c)] - name = os.path.basename(url) - if "." in name: - return name.split(".", 1)[0].strip() - return name diff --git a/platformio/unpacker.py b/platformio/package/unpack.py similarity index 65% rename from platformio/unpacker.py rename to platformio/package/unpack.py index 7fce466d..a00873cd 100644 --- a/platformio/unpacker.py +++ b/platformio/package/unpack.py @@ -19,10 +19,19 @@ from zipfile import ZipFile import click -from platformio import exception, util +from platformio import util +from platformio.package.exception import PackageException -class ArchiveBase(object): +class ExtractArchiveItemError(PackageException): + + MESSAGE = ( + "Could not extract `{0}` to `{1}`. Try to disable antivirus " + "tool or check this solution -> http://bit.ly/faq-package-manager" + ) + + +class BaseArchiver(object): def __init__(self, arhfileobj): self._afo = arhfileobj @@ -46,9 +55,9 @@ class ArchiveBase(object): self._afo.close() -class TARArchive(ArchiveBase): +class TARArchiver(BaseArchiver): def __init__(self, archpath): - super(TARArchive, self).__init__(tarfile_open(archpath)) + super(TARArchiver, self).__init__(tarfile_open(archpath)) def get_items(self): return self._afo.getmembers() @@ -79,7 +88,7 @@ class TARArchive(ArchiveBase): self.is_link(item) and self.is_bad_link(item, dest_dir), ] if not any(bad_conds): - super(TARArchive, self).extract_item(item, dest_dir) + super(TARArchiver, self).extract_item(item, dest_dir) else: click.secho( "Blocked insecure item `%s` from TAR archive" % item.name, @@ -88,9 +97,9 @@ class TARArchive(ArchiveBase): ) -class ZIPArchive(ArchiveBase): +class ZIPArchiver(BaseArchiver): def __init__(self, archpath): - super(ZIPArchive, self).__init__(ZipFile(archpath)) + super(ZIPArchiver, self).__init__(ZipFile(archpath)) @staticmethod def preserve_permissions(item, dest_dir): @@ -121,48 +130,59 @@ class ZIPArchive(ArchiveBase): class FileUnpacker(object): - def __init__(self, archpath): - self.archpath = archpath - self._unpacker = None + def __init__(self, path): + self.path = path + self._archiver = None + + def _init_archiver(self): + magic_map = { + b"\x1f\x8b\x08": TARArchiver, + b"\x42\x5a\x68": TARArchiver, + b"\x50\x4b\x03\x04": ZIPArchiver, + } + magic_len = max(len(k) for k in magic_map) + with open(self.path, "rb") as fp: + data = fp.read(magic_len) + for magic, archiver in magic_map.items(): + if data.startswith(magic): + return archiver(self.path) + raise PackageException("Unknown archive type '%s'" % self.path) def __enter__(self): - if self.archpath.lower().endswith((".gz", ".bz2", ".tar")): - self._unpacker = TARArchive(self.archpath) - elif self.archpath.lower().endswith(".zip"): - self._unpacker = ZIPArchive(self.archpath) - if not self._unpacker: - raise exception.UnsupportedArchiveType(self.archpath) + self._archiver = self._init_archiver() return self def __exit__(self, *args): - if self._unpacker: - self._unpacker.close() + if self._archiver: + self._archiver.close() def unpack( - self, dest_dir=".", with_progress=True, check_unpacked=True, silent=False + self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False ): - assert self._unpacker + assert self._archiver + if not dest_dir: + dest_dir = os.getcwd() if not with_progress or silent: if not silent: click.echo("Unpacking...") - for item in self._unpacker.get_items(): - self._unpacker.extract_item(item, dest_dir) + for item in self._archiver.get_items(): + self._archiver.extract_item(item, dest_dir) else: - items = self._unpacker.get_items() + items = self._archiver.get_items() with click.progressbar(items, label="Unpacking") as pb: for item in pb: - self._unpacker.extract_item(item, dest_dir) + self._archiver.extract_item(item, dest_dir) if not check_unpacked: return True # check on disk - for item in self._unpacker.get_items(): - filename = self._unpacker.get_item_filename(item) + for item in self._archiver.get_items(): + filename = self._archiver.get_item_filename(item) item_path = os.path.join(dest_dir, filename) try: - if not self._unpacker.is_link(item) and not os.path.exists(item_path): - raise exception.ExtractArchiveItemError(filename, dest_dir) + if not self._archiver.is_link(item) and not os.path.exists(item_path): + raise ExtractArchiveItemError(filename, dest_dir) except NotImplementedError: pass return True diff --git a/platformio/vcsclient.py b/platformio/package/vcsclient.py similarity index 100% rename from platformio/vcsclient.py rename to platformio/package/vcsclient.py diff --git a/platformio/util.py b/platformio/util.py index c2223109..36254586 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -371,7 +371,7 @@ PING_REMOTE_HOSTS = [ ] -@memoized(expire="5s") +@memoized(expire="10s") def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py new file mode 100644 index 00000000..2f7f4a62 --- /dev/null +++ b/tests/package/test_manager.py @@ -0,0 +1,303 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pytest + +from platformio import fs, util +from platformio.package.exception import MissingPackageManifestError +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.package.pack import PackagePacker + + +def test_download(isolated_pio_core): + url = "https://github.com/platformio/platformio-core/archive/v4.3.4.zip" + checksum = "69d59642cb91e64344f2cdc1d3b98c5cd57679b5f6db7accc7707bd4c5d9664a" + lm = LibraryPackageManager() + archive_path = lm.download(url, checksum, silent=True) + assert fs.calculate_file_hashsum("sha256", archive_path) == checksum + lm.cleanup_expired_downloads() + assert os.path.isfile(archive_path) + # test outdated downloads + lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1) + lm.cleanup_expired_downloads() + assert not os.path.isfile(archive_path) + # check that key is deleted from DB + with open(lm.get_download_usagedb_path()) as fp: + assert os.path.basename(archive_path) not in fp.read() + + +def test_find_pkg_root(isolated_pio_core, tmpdir_factory): + # has manifest + pkg_dir = tmpdir_factory.mktemp("package-has-manifest") + root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() + root_dir.join("platform.json").write("") + pm = PlatformPackageManager() + found_dir = pm.find_pkg_root(str(pkg_dir), spec=None) + assert os.path.realpath(str(root_dir)) == os.path.realpath(found_dir) + + # does not have manifest + pkg_dir = tmpdir_factory.mktemp("package-does-not-have-manifest") + pkg_dir.join("nested").mkdir().join("folder").mkdir().join("readme.txt").write("") + pm = PlatformPackageManager() + with pytest.raises(MissingPackageManifestError): + pm.find_pkg_root(str(pkg_dir), spec=None) + + # library package without manifest, should find source root + pkg_dir = tmpdir_factory.mktemp("library-package-without-manifest") + root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() + root_dir.join("src").mkdir().join("main.cpp").write("") + root_dir.join("include").mkdir().join("main.h").write("") + assert os.path.realpath(str(root_dir)) == os.path.realpath( + LibraryPackageManager.find_library_root(str(pkg_dir)) + ) + + # library manager should create "library.json" + lm = LibraryPackageManager() + spec = PackageSpec("custom-name@1.0.0") + pkg_root = lm.find_pkg_root(pkg_dir, spec) + manifest_path = os.path.join(pkg_root, "library.json") + assert os.path.realpath(str(root_dir)) == os.path.realpath(pkg_root) + assert os.path.isfile(manifest_path) + manifest = lm.load_manifest(pkg_root) + assert manifest["name"] == "custom-name" + assert "0.0.0" in str(manifest["version"]) + + +def test_build_legacy_spec(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + pm = PlatformPackageManager(str(storage_dir)) + # test src manifest + pkg1_dir = storage_dir.join("pkg-1").mkdir() + pkg1_dir.join(".pio").mkdir().join(".piopkgmanager.json").write( + """ +{ + "name": "StreamSpy-0.0.1.tar", + "url": "https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + "requirements": null +} +""" + ) + assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec( + name="StreamSpy-0.0.1.tar", + url="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + ) + + # without src manifest + pkg2_dir = storage_dir.join("pkg-2").mkdir() + pkg2_dir.join("main.cpp").write("") + with pytest.raises(MissingPackageManifestError): + pm.build_legacy_spec(str(pkg2_dir)) + + # with package manifest + pkg3_dir = storage_dir.join("pkg-3").mkdir() + pkg3_dir.join("platform.json").write('{"name": "pkg3", "version": "1.2.0"}') + assert pm.build_legacy_spec(str(pkg3_dir)) == PackageSpec(name="pkg3") + + +def test_build_metadata(isolated_pio_core, tmpdir_factory): + pm = PlatformPackageManager() + vcs_revision = "a2ebfd7c0f" + pkg_dir = tmpdir_factory.mktemp("package") + + # test package without manifest + with pytest.raises(MissingPackageManifestError): + pm.load_manifest(str(pkg_dir)) + with pytest.raises(MissingPackageManifestError): + pm.build_metadata(str(pkg_dir), PackageSpec("MyLib")) + + # with manifest + pkg_dir.join("platform.json").write( + '{"name": "Dev-Platform", "version": "1.2.3-alpha.1"}' + ) + metadata = pm.build_metadata(str(pkg_dir), PackageSpec("owner/platform-name")) + assert metadata.name == "Dev-Platform" + assert str(metadata.version) == "1.2.3-alpha.1" + + # with vcs + metadata = pm.build_metadata( + str(pkg_dir), PackageSpec("owner/platform-name"), vcs_revision + ) + assert str(metadata.version) == ("1.2.3-alpha.1+sha." + vcs_revision) + assert metadata.version.build[1] == vcs_revision + + +def test_install_from_url(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # install from local directory + src_dir = tmp_dir.join("local-lib-dir").mkdir() + src_dir.join("main.cpp").write("") + spec = PackageSpec("file://%s" % src_dir) + pkg = lm.install(spec, silent=True) + assert os.path.isfile(os.path.join(pkg.path, "main.cpp")) + manifest = lm.load_manifest(pkg) + assert manifest["name"] == "local-lib-dir" + assert manifest["version"].startswith("0.0.0+") + assert spec == pkg.metadata.spec + + # install from local archive + src_dir = tmp_dir.join("archive-src").mkdir() + root_dir = src_dir.mkdir("root") + root_dir.mkdir("src").join("main.cpp").write("#include ") + root_dir.join("library.json").write( + '{"name": "manifest-lib-name", "version": "2.0.0"}' + ) + tarball_path = PackagePacker(str(src_dir)).pack(str(tmp_dir)) + spec = PackageSpec("file://%s" % tarball_path) + pkg = lm.install(spec, silent=True) + assert os.path.isfile(os.path.join(pkg.path, "src", "main.cpp")) + assert pkg == lm.get_package(spec) + assert spec == pkg.metadata.spec + + # install from registry + src_dir = tmp_dir.join("registry-1").mkdir() + src_dir.join("library.properties").write( + """ +name = wifilib +version = 5.2.7 +""" + ) + spec = PackageSpec("company/wifilib @ ^5") + pkg = lm.install_from_url("file://%s" % src_dir, spec) + assert str(pkg.metadata.version) == "5.2.7" + + +def test_install_from_registry(isolated_pio_core, tmpdir_factory): + # Libraries + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) + # library with dependencies + lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert len(lm.get_installed()) == 3 + pkg = lm.get_package("AsyncTCP-esphome") + assert pkg.metadata.spec.owner == "ottowinter" + assert not lm.get_package("non-existing-package") + # mbed library + assert lm.install("wolfSSL", silent=True) + assert len(lm.get_installed()) == 4 + + # Tools + tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) + pkg = tm.install("tool-stlink @ ~1.10400.0", silent=True) + manifest = tm.load_manifest(pkg) + assert tm.is_system_compatible(manifest.get("system")) + assert util.get_systype() in manifest.get("system", []) + + +def test_get_installed(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # VCS package + ( + storage_dir.join("pkg-vcs") + .mkdir() + .join(".git") + .mkdir() + .join(".piopm") + .write( + """ +{ + "name": "pkg-via-vcs", + "spec": { + "id": null, + "name": "pkg-via-vcs", + "owner": null, + "requirements": null, + "url": "git+https://github.com/username/repo.git" + }, + "type": "library", + "version": "0.0.0+sha.1ea4d5e" +} +""" + ) + ) + + # package without metadata file + ( + storage_dir.join("foo@3.4.5") + .mkdir() + .join("library.json") + .write('{"name": "foo", "version": "3.4.5"}') + ) + + # package with metadata file + foo_dir = storage_dir.join("foo").mkdir() + foo_dir.join("library.json").write('{"name": "foo", "version": "3.6.0"}') + foo_dir.join(".piopm").write( + """ +{ + "name": "foo", + "spec": { + "name": "foo", + "owner": null, + "requirements": "^3" + }, + "type": "library", + "version": "3.6.0" +} +""" + ) + + # invalid package + storage_dir.join("invalid-package").mkdir().join("package.json").write( + '{"name": "tool-scons", "version": "4.0.0"}' + ) + + installed = lm.get_installed() + assert len(installed) == 3 + assert set(["pkg-via-vcs", "foo"]) == set(p.metadata.name for p in installed) + assert str(lm.get_package("foo").metadata.version) == "3.6.0" + + +def test_uninstall(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # foo @ 1.0.0 + pkg_dir = tmp_dir.join("foo").mkdir() + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + lm.install_from_url("file://%s" % pkg_dir, "foo") + # foo @ 1.3.0 + pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') + lm.install_from_url("file://%s" % pkg_dir, "foo") + # bar + pkg_dir = tmp_dir.join("bar").mkdir() + pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') + lm.install("file://%s" % pkg_dir, silent=True) + + assert len(lm.get_installed()) == 3 + assert os.path.isdir(os.path.join(str(storage_dir), "foo")) + assert os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) + + # check detaching + assert lm.uninstall("FOO", silent=True) + assert len(lm.get_installed()) == 2 + assert os.path.isdir(os.path.join(str(storage_dir), "foo")) + assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) + + # uninstall the rest + assert lm.uninstall("foo", silent=True) + assert lm.uninstall("bar", silent=True) + + assert len(lm.get_installed()) == 0 diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py new file mode 100644 index 00000000..d9d205c7 --- /dev/null +++ b/tests/package/test_meta.py @@ -0,0 +1,250 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import jsondiff +import semantic_version + +from platformio.package.meta import PackageMetaData, PackageSpec, PackageType + + +def test_spec_owner(): + assert PackageSpec("alice/foo library") == PackageSpec( + owner="alice", name="foo library" + ) + spec = PackageSpec(" Bob / BarUpper ") + assert spec != PackageSpec(owner="BOB", name="BARUPPER") + assert spec.owner == "Bob" + assert spec.name == "BarUpper" + + +def test_spec_id(): + assert PackageSpec(13) == PackageSpec(id=13) + assert PackageSpec("20") == PackageSpec(id=20) + spec = PackageSpec("id=199") + assert spec == PackageSpec(id=199) + assert isinstance(spec.id, int) + + +def test_spec_name(): + assert PackageSpec("foo") == PackageSpec(name="foo") + assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") + + +def test_spec_requirements(): + assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") + assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") + spec = PackageSpec("id=20 @ !=1.2.3,<2.0") + assert isinstance(spec.requirements, semantic_version.SimpleSpec) + assert semantic_version.Version("1.3.0-beta.1") in spec.requirements + assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") + + +def test_spec_local_urls(): + assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo" + ) + assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( + url="file:///tmp/bar.zip", name="customName" + ) + assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( + url="file:///tmp/some-lib/", name="some-lib" + ) + assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" + ) + + +def test_spec_external_urls(): + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.zip", + name="platformio-core", + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/" + "develop.zip?param=value", + name="platformio-core", + requirements="!=2", + ) + spec = PackageSpec( + "Custom-Name=" + "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" + ) + assert spec.is_custom_name() + assert spec.name == "Custom-Name" + assert spec == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", + name="Custom-Name", + requirements="4.4.0", + ) + + +def test_spec_vcs_urls(): + assert PackageSpec("https://github.com/platformio/platformio-core") == PackageSpec( + name="platformio-core", url="git+https://github.com/platformio/platformio-core" + ) + assert PackageSpec("https://gitlab.com/username/reponame") == PackageSpec( + name="reponame", url="git+https://gitlab.com/username/reponame" + ) + assert PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) == PackageSpec( + name="wolfSSL", url="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core.git#master" + ) == PackageSpec( + name="platformio-core", + url="git+https://github.com/platformio/platformio-core.git#master", + ) + assert PackageSpec( + "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" + ) == PackageSpec( + name="core", + url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", + requirements="4.4.0", + ) + assert PackageSpec( + "username@github.com:platformio/platformio-core.git" + ) == PackageSpec( + name="platformio-core", + url="git+username@github.com:platformio/platformio-core.git", + ) + assert PackageSpec( + "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" + ) == PackageSpec( + name="pkg", + url="git+git@github.com:platformio/platformio-core.git", + requirements="^1.2.3,!=5", + ) + + +def test_spec_as_dict(): + assert not jsondiff.diff( + PackageSpec("bob/foo@1.2.3").as_dict(), + { + "owner": "bob", + "id": None, + "name": "foo", + "requirements": "1.2.3", + "url": None, + }, + ) + assert not jsondiff.diff( + PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ).as_dict(), + { + "owner": None, + "id": None, + "name": "platformio-core", + "requirements": "!=2", + "url": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value", + }, + ) + + +def test_metadata_as_dict(): + metadata = PackageMetaData(PackageType.LIBRARY, "foo", "1.2.3") + # test setter + metadata.version = "0.1.2+12345" + assert metadata.version == semantic_version.Version("0.1.2+12345") + assert not jsondiff.diff( + metadata.as_dict(), + { + "type": PackageType.LIBRARY, + "name": "foo", + "version": "0.1.2+12345", + "spec": None, + }, + ) + + assert not jsondiff.diff( + PackageMetaData( + PackageType.TOOL, + "toolchain", + "2.0.5", + PackageSpec("platformio/toolchain@~2.0.0"), + ).as_dict(), + { + "type": PackageType.TOOL, + "name": "toolchain", + "version": "2.0.5", + "spec": { + "owner": "platformio", + "id": None, + "name": "toolchain", + "requirements": "~2.0.0", + "url": None, + }, + }, + ) + + +def test_metadata_dump(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + metadata = PackageMetaData( + PackageType.TOOL, + "toolchain", + "2.0.5", + PackageSpec("platformio/toolchain@~2.0.0"), + ) + + dst = pkg_dir.join(".piopm") + metadata.dump(str(dst)) + assert os.path.isfile(str(dst)) + contents = dst.read() + assert all(s in contents for s in ("null", '"~2.0.0"')) + + +def test_metadata_load(tmpdir_factory): + contents = """ +{ + "name": "foo", + "spec": { + "name": "foo", + "owner": "username", + "requirements": "!=3.4.5" + }, + "type": "platform", + "version": "0.1.3" +} +""" + pkg_dir = tmpdir_factory.mktemp("package") + dst = pkg_dir.join(".piopm") + dst.write(contents) + metadata = PackageMetaData.load(str(dst)) + assert metadata.version == semantic_version.Version("0.1.3") + assert metadata == PackageMetaData( + PackageType.PLATFORM, + "foo", + "0.1.3", + spec=PackageSpec(owner="username", name="foo", requirements="!=3.4.5"), + ) + + piopm_path = pkg_dir.join(".piopm") + metadata = PackageMetaData( + PackageType.LIBRARY, "mylib", version="1.2.3", spec=PackageSpec("mylib") + ) + metadata.dump(str(piopm_path)) + restored_metadata = PackageMetaData.load(str(piopm_path)) + assert metadata == restored_metadata diff --git a/tests/package/test_spec.py b/tests/package/test_spec.py deleted file mode 100644 index dce89d7f..00000000 --- a/tests/package/test_spec.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from platformio.package.spec import PackageSpec - - -def test_ownername(): - assert PackageSpec("alice/foo library") == PackageSpec( - ownername="alice", name="foo library" - ) - assert PackageSpec(" bob / bar ") == PackageSpec(ownername="bob", name="bar") - - -def test_id(): - assert PackageSpec(13) == PackageSpec(id=13) - assert PackageSpec("20") == PackageSpec(id=20) - assert PackageSpec("id=199") == PackageSpec(id=199) - - -def test_name(): - assert PackageSpec("foo") == PackageSpec(name="foo") - assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") - - -def test_requirements(): - assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") - assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") - assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") - assert PackageSpec("id=20 @ !=1.2.3,<2.0") == PackageSpec( - id=20, requirements="!=1.2.3,<2.0" - ) - - -def test_local_urls(): - assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( - url="file:///tmp/foo.tar.gz", name="foo" - ) - assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( - url="file:///tmp/bar.zip", name="customName" - ) - assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( - url="file:///tmp/some-lib/", name="some-lib" - ) - assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( - url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" - ) - - -def test_external_urls(): - assert PackageSpec( - "https://github.com/platformio/platformio-core/archive/develop.zip" - ) == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/develop.zip", - name="develop", - ) - assert PackageSpec( - "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" - " @ !=2" - ) == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/" - "develop.zip?param=value", - name="develop", - requirements="!=2", - ) - assert PackageSpec( - "platformio-core=" - "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" - ) == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", - name="platformio-core", - requirements="4.4.0", - ) - - -def test_vcs_urls(): - assert PackageSpec( - "https://github.com/platformio/platformio-core.git" - ) == PackageSpec( - name="platformio-core", url="https://github.com/platformio/platformio-core.git", - ) - assert PackageSpec( - "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" - ) == PackageSpec( - name="wolfSSL", url="https://os.mbed.com/users/wolfSSL/code/wolfSSL/", - ) - assert PackageSpec( - "git+https://github.com/platformio/platformio-core.git#master" - ) == PackageSpec( - name="platformio-core", - url="git+https://github.com/platformio/platformio-core.git#master", - ) - assert PackageSpec( - "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" - ) == PackageSpec( - name="core", - url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", - requirements="4.4.0", - ) - assert PackageSpec("git@github.com:platformio/platformio-core.git") == PackageSpec( - name="platformio-core", url="git@github.com:platformio/platformio-core.git", - ) - assert PackageSpec( - "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" - ) == PackageSpec( - name="pkg", - url="git+git@github.com:platformio/platformio-core.git", - requirements="^1.2.3,!=5", - ) From a1970bbfe316693fedbc4d569fbe8c82794570a7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 14:38:28 +0300 Subject: [PATCH 139/223] Allow a forced package installation with removing existing package --- platformio/package/manager/_download.py | 3 +- platformio/package/manager/_install.py | 50 ++++++++++++++++--------- platformio/package/manager/_registry.py | 2 +- platformio/package/manager/base.py | 19 +++++++++- platformio/package/meta.py | 11 +++++- tests/package/test_manager.py | 23 ++++++++++-- 6 files changed, 82 insertions(+), 26 deletions(-) diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index a052da09..2ab8c143 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -14,6 +14,7 @@ import hashlib import os +import shutil import tempfile import time @@ -85,7 +86,7 @@ class PackageManagerDownloadMixin(object): raise e if checksum: fd.verify(checksum) - os.rename(tmp_path, dl_path) + shutil.copyfile(tmp_path, dl_path) finally: if os.path.isfile(tmp_path): os.remove(tmp_path) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index a5aa1c38..58b1de1e 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,6 @@ import click from platformio import app, compat, fs, util from platformio.package.exception import PackageException, UnknownPackageError -from platformio.package.lockfile import LockFile from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -43,25 +42,33 @@ class PackageManagerInstallMixin(object): with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=False) - def install(self, spec, silent=False): - with LockFile(self.package_dir): - pkg = self._install(spec, silent=silent) + def install(self, spec, silent=False, force=False): + try: + self.lock() + pkg = self._install(spec, silent=silent, force=force) self.memcache_reset() self.cleanup_expired_downloads() return pkg + finally: + self.unlock() - def _install(self, spec, search_filters=None, silent=False): + def _install(self, spec, search_filters=None, silent=False, force=False): spec = self.ensure_spec(spec) # avoid circle dependencies if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] + self.INSTALL_HISTORY = {} if spec in self.INSTALL_HISTORY: - return None - self.INSTALL_HISTORY.append(spec) + return self.INSTALL_HISTORY[spec] # check if package is already installed pkg = self.get_package(spec) + + # if a forced installation + if pkg and force: + self.uninstall(pkg, silent=silent) + pkg = None + if pkg: if not silent: click.secho( @@ -99,6 +106,7 @@ class PackageManagerInstallMixin(object): self.memcache_reset() self.install_dependencies(pkg, silent) + self.INSTALL_HISTORY[spec] = pkg return pkg def install_dependencies(self, pkg, silent=False): @@ -240,15 +248,18 @@ class PackageManagerInstallMixin(object): shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) - def uninstall(self, path_or_spec, silent=False): - with LockFile(self.package_dir): - pkg = ( - PackageSourceItem(path_or_spec) - if os.path.isdir(path_or_spec) - else self.get_package(path_or_spec) - ) + def uninstall(self, pkg, silent=False): + try: + self.lock() + + if not isinstance(pkg, PackageSourceItem): + pkg = ( + PackageSourceItem(pkg) + if os.path.isdir(pkg) + else self.get_package(pkg) + ) if not pkg or not pkg.metadata: - raise UnknownPackageError(path_or_spec) + raise UnknownPackageError(pkg) if not silent: self.print_message( @@ -276,7 +287,10 @@ class PackageManagerInstallMixin(object): os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), ) self.memcache_reset() + finally: + self.unlock() + + if not silent: + click.echo("[%s]" % click.style("OK", fg="green")) - if not silent: - click.echo("[%s]" % click.style("OK", fg="green")) return True diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 3f3ae813..cdd46cd0 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -37,7 +37,7 @@ class RegistryFileMirrorsIterator(object): self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) self._visited_mirrors = [] - def __iter__(self): + def __iter__(self): # pylint: disable=non-iterator-returned return self def __next__(self): diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index e48fc250..93599200 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -21,6 +21,7 @@ import semantic_version from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.package.exception import ManifestException, MissingPackageManifestError +from platformio.package.lockfile import LockFile from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager._registry import PackageManageRegistryMixin @@ -34,7 +35,7 @@ from platformio.package.meta import ( from platformio.project.helpers import get_project_cache_dir -class BasePackageManager( +class BasePackageManager( # pylint: disable=too-many-public-methods PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin ): MEMORY_CACHE = {} @@ -43,10 +44,26 @@ class BasePackageManager( self.pkg_type = pkg_type self.package_dir = self.ensure_dir_exists(package_dir) self.MEMORY_CACHE = {} + + self._lockfile = None self._download_dir = None self._tmp_dir = None self._registry_client = None + def lock(self): + if self._lockfile: + return + self._lockfile = LockFile(self.package_dir) + self._lockfile.acquire() + + def unlock(self): + if hasattr(self, "_lockfile") and self._lockfile: + self._lockfile.release() + self._lockfile = None + + def __del__(self): + self.unlock() + def memcache_get(self, key, default=None): return self.MEMORY_CACHE.get(key, default) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 0f1214e1..6cd2904b 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -16,10 +16,11 @@ import json import os import re import tarfile +from binascii import crc32 import semantic_version -from platformio.compat import get_object_members, string_types +from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType try: @@ -89,6 +90,14 @@ class PackageSpec(object): ] ) + def __hash__(self): + return crc32( + hashlib_encode_data( + "%s-%s-%s-%s-%s" + % (self.owner, self.id, self.name, self.requirements, self.url) + ) + ) + def __repr__(self): return ( "PackageSpec 5 + + def test_get_installed(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) @@ -276,7 +291,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): # foo @ 1.0.0 pkg_dir = tmp_dir.join("foo").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') - lm.install_from_url("file://%s" % pkg_dir, "foo") + foo_1_0_0_pkg = lm.install_from_url("file://%s" % pkg_dir, "foo") # foo @ 1.3.0 pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') @@ -284,7 +299,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): # bar pkg_dir = tmp_dir.join("bar").mkdir() pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') - lm.install("file://%s" % pkg_dir, silent=True) + bar_pkg = lm.install("file://%s" % pkg_dir, silent=True) assert len(lm.get_installed()) == 3 assert os.path.isdir(os.path.join(str(storage_dir), "foo")) @@ -297,7 +312,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) # uninstall the rest - assert lm.uninstall("foo", silent=True) - assert lm.uninstall("bar", silent=True) + assert lm.uninstall(foo_1_0_0_pkg.path, silent=True) + assert lm.uninstall(bar_pkg, silent=True) assert len(lm.get_installed()) == 0 From 41c2d64ef0786b7dc511a21140fc98272f318f2e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 15:36:28 +0300 Subject: [PATCH 140/223] Fix "PermissionError: [WinError 32] The process cannot access the file" on Windows --- platformio/package/manager/_download.py | 6 ++++-- platformio/package/manager/_registry.py | 4 ++++ tests/package/test_manager.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 2ab8c143..01be36aa 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -61,7 +61,7 @@ class PackageManagerDownloadMixin(object): return dl_path with_progress = not silent and not app.is_disabled_progressbar() - tmp_path = tempfile.mkstemp(dir=self.get_download_dir())[1] + tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir()) try: with LockFile(dl_path): try: @@ -86,9 +86,11 @@ class PackageManagerDownloadMixin(object): raise e if checksum: fd.verify(checksum) - shutil.copyfile(tmp_path, dl_path) + os.close(tmp_fd) + os.rename(tmp_path, dl_path) finally: if os.path.isfile(tmp_path): + os.close(tmp_fd) os.remove(tmp_path) assert os.path.isfile(dl_path) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index cdd46cd0..a14bde98 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -64,6 +64,10 @@ class RegistryFileMirrorsIterator(object): response.headers.get("X-PIO-Content-SHA256"), ) + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + def get_http_client(self): if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES: RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[ diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 7d3cb070..2d8a7b14 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -71,7 +71,7 @@ def test_find_pkg_root(isolated_pio_core, tmpdir_factory): # library manager should create "library.json" lm = LibraryPackageManager() spec = PackageSpec("custom-name@1.0.0") - pkg_root = lm.find_pkg_root(pkg_dir, spec) + pkg_root = lm.find_pkg_root(str(pkg_dir), spec) manifest_path = os.path.join(pkg_root, "library.json") assert os.path.realpath(str(root_dir)) == os.path.realpath(pkg_root) assert os.path.isfile(manifest_path) From 6ac538fba4e75a1be5d44028459ff168c7ab866a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 15:49:10 +0300 Subject: [PATCH 141/223] Remove unused import --- platformio/package/manager/_download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 01be36aa..83de9f37 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -14,7 +14,6 @@ import hashlib import os -import shutil import tempfile import time From a01b3a247361c3af01e9201db492a08f140bcc2f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 19:58:59 +0300 Subject: [PATCH 142/223] Do not raise exception when package is not found (404), return None --- platformio/clients/http.py | 4 +++- platformio/clients/registry.py | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index e1257762..3070f749 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -69,4 +69,6 @@ class HTTPClient(object): message = response.json()["message"] except (KeyError, ValueError): message = response.text - raise HTTPClientError(message) + exc = HTTPClientError(message) + exc.response = response + raise exc diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index b7d724b9..f8130c60 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -14,7 +14,7 @@ from platformio import __registry_api__, fs from platformio.clients.account import AccountClient -from platformio.clients.http import HTTPClient +from platformio.clients.http import HTTPClient, HTTPClientError from platformio.package.meta import PackageType try: @@ -120,7 +120,7 @@ class RegistryClient(HTTPClient): for value in set( values if isinstance(values, (list, tuple)) else [values] ): - search_query.append("%s:%s" % (name[:-1], value)) + search_query.append('%s:"%s"' % (name[:-1], value)) if query: search_query.append(query) params = dict(query=quote(" ".join(search_query))) @@ -129,10 +129,15 @@ class RegistryClient(HTTPClient): return self.request_json_data("get", "/v3/packages", params=params) def get_package(self, type_, owner, name, version=None): - return self.request_json_data( - "get", - "/v3/packages/{owner}/{type}/{name}".format( - type=type_, owner=owner, name=quote(name) - ), - params=dict(version=version) if version else None, - ) + try: + return self.request_json_data( + "get", + "/v3/packages/{owner}/{type}/{name}".format( + type=type_, owner=owner.lower(), name=quote(name.lower()) + ), + params=dict(version=version) if version else None, + ) + except HTTPClientError as e: + if e.response.status_code == 404: + return None + raise e From 2dd69e21c00f7e843521b7a998063d60ae803d43 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 20:17:07 +0300 Subject: [PATCH 143/223] Implement package removing with dependencies --- platformio/clients/http.py | 14 ++-- platformio/package/manager/_install.py | 69 +++--------------- platformio/package/manager/_registry.py | 29 +++++--- platformio/package/manager/_uninstall.py | 93 ++++++++++++++++++++++++ platformio/package/manager/base.py | 16 ++-- tests/package/test_manager.py | 28 ++++++- 6 files changed, 167 insertions(+), 82 deletions(-) create mode 100644 platformio/package/manager/_uninstall.py diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 3070f749..47f6c162 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -20,7 +20,13 @@ from platformio.exception import PlatformioException class HTTPClientError(PlatformioException): - pass + def __init__(self, message, response=None): + super(HTTPClientError, self).__init__() + self.message = message + self.response = response + + def __str__(self): # pragma: no cover + return self.message class HTTPClient(object): @@ -52,7 +58,7 @@ class HTTPClient(object): try: return getattr(self._session, method)(self.base_url + path, **kwargs) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - raise HTTPClientError(e) + raise HTTPClientError(str(e)) def request_json_data(self, *args, **kwargs): response = self.send_request(*args, **kwargs) @@ -69,6 +75,4 @@ class HTTPClient(object): message = response.json()["message"] except (KeyError, ValueError): message = response.text - exc = HTTPClientError(message) - exc.response = response - raise exc + raise HTTPClientError(message, response) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 58b1de1e..ea409f2e 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -20,7 +20,7 @@ import tempfile import click from platformio import app, compat, fs, util -from platformio.package.exception import PackageException, UnknownPackageError +from platformio.package.exception import PackageException from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -28,7 +28,7 @@ from platformio.package.vcsclient import VCSClientFactory class PackageManagerInstallMixin(object): - INSTALL_HISTORY = None # avoid circle dependencies + _INSTALL_HISTORY = None # avoid circle dependencies @staticmethod def unpack(src, dst): @@ -56,10 +56,10 @@ class PackageManagerInstallMixin(object): spec = self.ensure_spec(spec) # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = {} - if spec in self.INSTALL_HISTORY: - return self.INSTALL_HISTORY[spec] + if not self._INSTALL_HISTORY: + self._INSTALL_HISTORY = {} + if spec in self._INSTALL_HISTORY: + return self._INSTALL_HISTORY[spec] # check if package is already installed pkg = self.get_package(spec) @@ -105,11 +105,11 @@ class PackageManagerInstallMixin(object): ) self.memcache_reset() - self.install_dependencies(pkg, silent) - self.INSTALL_HISTORY[spec] = pkg + self._install_dependencies(pkg, silent) + self._INSTALL_HISTORY[spec] = pkg return pkg - def install_dependencies(self, pkg, silent=False): + def _install_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageSourceItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): @@ -117,14 +117,14 @@ class PackageManagerInstallMixin(object): if not silent: self.print_message(click.style("Installing dependencies...", fg="yellow")) for dependency in manifest.get("dependencies"): - if not self.install_dependency(dependency, silent) and not silent: + if not self._install_dependency(dependency, silent) and not silent: click.secho( "Warning! Could not install dependency %s for package '%s'" % (dependency, pkg.metadata.name), fg="yellow", ) - def install_dependency(self, dependency, silent=False): + def _install_dependency(self, dependency, silent=False): spec = PackageSpec( name=dependency.get("name"), requirements=dependency.get("version") ) @@ -247,50 +247,3 @@ class PackageManagerInstallMixin(object): _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) - - def uninstall(self, pkg, silent=False): - try: - self.lock() - - if not isinstance(pkg, PackageSourceItem): - pkg = ( - PackageSourceItem(pkg) - if os.path.isdir(pkg) - else self.get_package(pkg) - ) - if not pkg or not pkg.metadata: - raise UnknownPackageError(pkg) - - if not silent: - self.print_message( - "Uninstalling %s @ %s: \t" - % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), - nl=False, - ) - if os.path.islink(pkg.path): - os.unlink(pkg.path) - else: - fs.rmtree(pkg.path) - self.memcache_reset() - - # unfix detached-package with the same name - detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) - if ( - detached_pkg - and "@" in detached_pkg.path - and not os.path.isdir( - os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) - ) - ): - shutil.move( - detached_pkg.path, - os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), - ) - self.memcache_reset() - finally: - self.unlock() - - if not silent: - click.echo("[%s]" % click.style("OK", fg="green")) - - return True diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index a14bde98..d5c9ddad 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -78,12 +78,19 @@ class RegistryFileMirrorsIterator(object): class PackageManageRegistryMixin(object): def install_from_registry(self, spec, search_filters=None, silent=False): - packages = self.search_registry_packages(spec, search_filters) - if not packages: - raise UnknownPackageError(spec.humanize()) - if len(packages) > 1 and not silent: - self.print_multi_package_issue(packages, spec) - package, version = self.find_best_registry_version(packages, spec) + if spec.owner and spec.name and not search_filters: + package = self.fetch_registry_package(spec.owner, spec.name) + if not package: + raise UnknownPackageError(spec.humanize()) + version = self._pick_best_pkg_version(package["versions"], spec) + else: + packages = self.search_registry_packages(spec, search_filters) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + package, version = self.find_best_registry_version(packages, spec) + pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None if not pkgfile: raise UnknownPackageError(spec.humanize()) @@ -117,17 +124,17 @@ class PackageManageRegistryMixin(object): filters["ids"] = str(spec.id) else: filters["types"] = self.pkg_type - filters["names"] = '"%s"' % spec.name.lower() + filters["names"] = spec.name.lower() if spec.owner: filters["owners"] = spec.owner.lower() return self.get_registry_client_instance().list_packages(filters=filters)[ "items" ] - def fetch_registry_package_versions(self, owner, name): + def fetch_registry_package(self, owner, name): return self.get_registry_client_instance().get_package( self.pkg_type, owner, name - )["versions"] + ) @staticmethod def print_multi_package_issue(packages, spec): @@ -163,9 +170,9 @@ class PackageManageRegistryMixin(object): # if the custom version requirements, check ALL package versions for package in packages: version = self._pick_best_pkg_version( - self.fetch_registry_package_versions( + self.fetch_registry_package( package["owner"]["username"], package["name"] - ), + ).get("versions"), spec, ) if version: diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py new file mode 100644 index 00000000..e754eab2 --- /dev/null +++ b/platformio/package/manager/_uninstall.py @@ -0,0 +1,93 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil + +import click + +from platformio import fs +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSourceItem, PackageSpec + + +class PackageManagerUninstallMixin(object): + def uninstall(self, pkg, silent=False, skip_dependencies=False): + try: + self.lock() + return self._uninstall(pkg, silent, skip_dependencies) + finally: + self.unlock() + + def _uninstall(self, pkg, silent=False, skip_dependencies=False): + if not isinstance(pkg, PackageSourceItem): + pkg = ( + PackageSourceItem(pkg) if os.path.isdir(pkg) else self.get_package(pkg) + ) + if not pkg or not pkg.metadata: + raise UnknownPackageError(pkg) + + if not silent: + self.print_message( + "Removing %s @ %s: \t" + % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), + nl=False, + ) + + # firstly, remove dependencies + if not skip_dependencies: + self._uninstall_dependencies(pkg, silent) + + if os.path.islink(pkg.path): + os.unlink(pkg.path) + else: + fs.rmtree(pkg.path) + self.memcache_reset() + + # unfix detached-package with the same name + detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) + if ( + detached_pkg + and "@" in detached_pkg.path + and not os.path.isdir( + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) + ) + ): + shutil.move( + detached_pkg.path, + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), + ) + self.memcache_reset() + + if not silent: + click.echo("[%s]" % click.style("OK", fg="green")) + + return True + + def _uninstall_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageSourceItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message(click.style("Removing dependencies...", fg="yellow")) + for dependency in manifest.get("dependencies"): + pkg = self.get_package( + PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + ) + if not pkg: + continue + self._uninstall(pkg, silent=silent) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 93599200..ca065833 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -25,6 +25,7 @@ from platformio.package.lockfile import LockFile from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager._registry import PackageManageRegistryMixin +from platformio.package.manager._uninstall import PackageManagerUninstallMixin from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import ( PackageMetaData, @@ -36,14 +37,17 @@ from platformio.project.helpers import get_project_cache_dir class BasePackageManager( # pylint: disable=too-many-public-methods - PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin + PackageManagerDownloadMixin, + PackageManageRegistryMixin, + PackageManagerInstallMixin, + PackageManagerUninstallMixin, ): - MEMORY_CACHE = {} + _MEMORY_CACHE = {} def __init__(self, pkg_type, package_dir): self.pkg_type = pkg_type self.package_dir = self.ensure_dir_exists(package_dir) - self.MEMORY_CACHE = {} + self._MEMORY_CACHE = {} self._lockfile = None self._download_dir = None @@ -65,13 +69,13 @@ class BasePackageManager( # pylint: disable=too-many-public-methods self.unlock() def memcache_get(self, key, default=None): - return self.MEMORY_CACHE.get(key, default) + return self._MEMORY_CACHE.get(key, default) def memcache_set(self, key, value): - self.MEMORY_CACHE[key] = value + self._MEMORY_CACHE[key] = value def memcache_reset(self): - self.MEMORY_CACHE.clear() + self._MEMORY_CACHE.clear() @staticmethod def is_system_compatible(value): diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 2d8a7b14..5898ae2b 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -18,7 +18,10 @@ import time import pytest from platformio import fs, util -from platformio.package.exception import MissingPackageManifestError +from platformio.package.exception import ( + MissingPackageManifestError, + UnknownPackageError, +) from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager @@ -193,14 +196,24 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): # mbed library assert lm.install("wolfSSL", silent=True) assert len(lm.get_installed()) == 4 + # case sensitive author name + assert lm.install("DallasTemperature", silent=True) + assert lm.get_package("OneWire").metadata.version.major >= 2 + assert len(lm.get_installed()) == 6 # Tools tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) - pkg = tm.install("tool-stlink @ ~1.10400.0", silent=True) + pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True) manifest = tm.load_manifest(pkg) assert tm.is_system_compatible(manifest.get("system")) assert util.get_systype() in manifest.get("system", []) + # Test unknown + with pytest.raises(UnknownPackageError): + tm.install("unknown-package-tool @ 9.1.1", silent=True) + with pytest.raises(UnknownPackageError): + tm.install("owner/unknown-package-tool", silent=True) + def test_install_force(isolated_pio_core, tmpdir_factory): lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) @@ -316,3 +329,14 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert lm.uninstall(bar_pkg, silent=True) assert len(lm.get_installed()) == 0 + + # test uninstall dependencies + assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert len(lm.get_installed()) == 3 + assert lm.uninstall("AsyncMqttClient-esphome", silent=True, skip_dependencies=True) + assert len(lm.get_installed()) == 2 + + lm = LibraryPackageManager(str(storage_dir)) + assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert lm.uninstall("AsyncMqttClient-esphome", silent=True) + assert len(lm.get_installed()) == 0 From 893ca1b328523650a9832ba2dbfaf14994d61584 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Aug 2020 13:27:05 +0300 Subject: [PATCH 144/223] Switch library manager to the new package manager --- platformio/builder/tools/piolib.py | 58 ++- platformio/clients/http.py | 5 +- platformio/commands/lib/__init__.py | 13 + .../commands/{lib.py => lib/command.py} | 178 ++++--- platformio/commands/lib/helpers.py | 90 ++++ platformio/commands/system/command.py | 4 +- platformio/commands/update.py | 8 +- platformio/compat.py | 8 + platformio/exception.py | 9 - platformio/maintenance.py | 41 +- platformio/managers/lib.py | 374 --------------- platformio/managers/package.py | 8 +- platformio/package/manager/_install.py | 39 +- platformio/package/manager/_legacy.py | 57 +++ platformio/package/manager/_registry.py | 50 +- platformio/package/manager/_uninstall.py | 7 +- platformio/package/manager/_update.py | 166 +++++++ platformio/package/manager/base.py | 76 +-- platformio/package/manager/library.py | 2 +- platformio/package/manager/platform.py | 2 +- platformio/package/manager/tool.py | 2 +- platformio/package/manifest/schema.py | 2 +- platformio/package/meta.py | 69 ++- platformio/package/vcsclient.py | 34 +- tests/commands/test_ci.py | 2 +- tests/commands/test_lib.py | 442 ++++++------------ tests/commands/test_lib_complex.py | 348 ++++++++++++++ tests/package/test_manager.py | 85 ++++ tests/package/test_meta.py | 50 +- tests/test_maintenance.py | 4 +- 30 files changed, 1318 insertions(+), 915 deletions(-) create mode 100644 platformio/commands/lib/__init__.py rename platformio/commands/{lib.py => lib/command.py} (81%) create mode 100644 platformio/commands/lib/helpers.py delete mode 100644 platformio/managers/lib.py create mode 100644 platformio/package/manager/_legacy.py create mode 100644 platformio/package/manager/_update.py create mode 100644 tests/commands/test_lib_complex.py diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index fbd8949c..24229b1c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -34,11 +34,13 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool from platformio.compat import WINDOWS, hashlib_encode_data, string_types -from platformio.managers.lib import LibraryManager +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.library import LibraryPackageManager from platformio.package.manifest.parser import ( ManifestParserError, ManifestParserFactory, ) +from platformio.package.meta import PackageSourceItem from platformio.project.options import ProjectOptions @@ -851,34 +853,36 @@ class ProjectAsLibBuilder(LibBuilderBase): pass def install_dependencies(self): - def _is_builtin(uri): + def _is_builtin(spec): for lb in self.env.GetLibBuilders(): - if lb.name == uri: + if lb.name == spec: return True return False - not_found_uri = [] - for uri in self.dependencies: + not_found_specs = [] + for spec in self.dependencies: # check if built-in library - if _is_builtin(uri): + if _is_builtin(spec): continue found = False for storage_dir in self.env.GetLibSourceDirs(): - lm = LibraryManager(storage_dir) - if lm.get_package_dir(*lm.parse_pkg_uri(uri)): + lm = LibraryPackageManager(storage_dir) + if lm.get_package(spec): found = True break if not found: - not_found_uri.append(uri) + not_found_specs.append(spec) did_install = False - lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))) - for uri in not_found_uri: + lm = LibraryPackageManager( + self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) + ) + for spec in not_found_specs: try: - lm.install(uri) + lm.install(spec) did_install = True - except (exception.LibNotFound, exception.InternetIsOffline) as e: + except (UnknownPackageError, exception.InternetIsOffline) as e: click.secho("Warning! %s" % e, fg="yellow") # reset cache @@ -886,17 +890,17 @@ class ProjectAsLibBuilder(LibBuilderBase): DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) def process_dependencies(self): # pylint: disable=too-many-branches - for uri in self.dependencies: + for spec in self.dependencies: found = False for storage_dir in self.env.GetLibSourceDirs(): if found: break - lm = LibraryManager(storage_dir) - lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri)) - if not lib_dir: + lm = LibraryPackageManager(storage_dir) + pkg = lm.get_package(spec) + if not pkg: continue for lb in self.env.GetLibBuilders(): - if lib_dir != lb.path: + if pkg.path != lb.path: continue if lb not in self.depbuilders: self.depend_recursive(lb) @@ -908,7 +912,7 @@ class ProjectAsLibBuilder(LibBuilderBase): # look for built-in libraries by a name # which don't have package manifest for lb in self.env.GetLibBuilders(): - if lb.name != uri: + if lb.name != spec: continue if lb not in self.depbuilders: self.depend_recursive(lb) @@ -1000,10 +1004,6 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches def ConfigureProjectLibBuilder(env): - def _get_vcs_info(lb): - path = LibraryManager.get_src_manifest_path(lb.path) - return fs.load_json(path) if path else None - def _correct_found_libs(lib_builders): # build full dependency graph found_lbs = [lb for lb in lib_builders if lb.dependent] @@ -1019,15 +1019,13 @@ def ConfigureProjectLibBuilder(env): margin = "| " * (level) for lb in root.depbuilders: title = "<%s>" % lb.name - vcs_info = _get_vcs_info(lb) - if lb.version: - title += " %s" % lb.version - if vcs_info and vcs_info.get("version"): - title += " #%s" % vcs_info.get("version") + pkg = PackageSourceItem(lb.path) + if pkg.metadata: + title += " %s" % pkg.metadata.version click.echo("%s|-- %s" % (margin, title), nl=False) if int(ARGUMENTS.get("PIOVERBOSE", 0)): - if vcs_info: - click.echo(" [%s]" % vcs_info.get("url"), nl=False) + if pkg.metadata and pkg.metadata.spec.external: + click.echo(" [%s]" % pkg.metadata.spec.url, nl=False) click.echo(" (", nl=False) click.echo(lb.path, nl=False) click.echo(")", nl=False) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 47f6c162..974017b7 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -30,7 +30,9 @@ class HTTPClientError(PlatformioException): class HTTPClient(object): - def __init__(self, base_url): + def __init__( + self, base_url, + ): if base_url.endswith("/"): base_url = base_url[:-1] self.base_url = base_url @@ -51,6 +53,7 @@ class HTTPClient(object): self._session.close() self._session = None + @util.throttle(500) def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout # print(self, method, path, kwargs) diff --git a/platformio/commands/lib/__init__.py b/platformio/commands/lib/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/lib/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/lib.py b/platformio/commands/lib/command.py similarity index 81% rename from platformio/commands/lib.py rename to platformio/commands/lib/command.py index 5bd38aee..33249f3e 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib/command.py @@ -18,16 +18,21 @@ import os import time import click -import semantic_version from tabulate import tabulate from platformio import exception, fs, util from platformio.commands import PlatformioCLI +from platformio.commands.lib.helpers import ( + get_builtin_libs, + is_builtin_lib, + save_project_libdeps, +) from platformio.compat import dump_json_to_unicode -from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.proc import is_ci from platformio.project.config import ProjectConfig -from platformio.project.exception import InvalidProjectConfError from platformio.project.helpers import get_project_dir, is_platformio_project try: @@ -124,89 +129,106 @@ def cli(ctx, **options): @cli.command("install", short_help="Install library") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") @click.option( - "--save", + "--save/--no-save", is_flag=True, - help="Save installed libraries into the `platformio.ini` dependency list", + default=True, + help="Save installed libraries into the `platformio.ini` dependency list" + " (enabled by default)", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( - "--interactive", is_flag=True, help="Allow to make a choice for all prompts" + "--interactive", + is_flag=True, + help="Deprecated! Please use a strict dependency specification (owner/libname)", ) @click.option( "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists" ) @click.pass_context -def lib_install( # pylint: disable=too-many-arguments +def lib_install( # pylint: disable=too-many-arguments,unused-argument ctx, libraries, save, silent, interactive, force ): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) - installed_manifests = {} + installed_pkgs = {} for storage_dir in storage_dirs: if not silent and (libraries or storage_dir in storage_libdeps): print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) + lm = LibraryPackageManager(storage_dir) + if libraries: - for library in libraries: - pkg_dir = lm.install( - library, silent=silent, interactive=interactive, force=force - ) - installed_manifests[library] = lm.load_manifest(pkg_dir) + installed_pkgs = { + library: lm.install(library, silent=silent, force=force) + for library in libraries + } + elif storage_dir in storage_libdeps: builtin_lib_storages = None for library in storage_libdeps[storage_dir]: try: - pkg_dir = lm.install( - library, silent=silent, interactive=interactive, force=force - ) - installed_manifests[library] = lm.load_manifest(pkg_dir) - except exception.LibNotFound as e: + lm.install(library, silent=silent, force=force) + except UnknownPackageError as e: if builtin_lib_storages is None: builtin_lib_storages = get_builtin_libs() if not silent or not is_builtin_lib(builtin_lib_storages, library): click.secho("Warning! %s" % e, fg="yellow") - if not save or not libraries: - return + if save and installed_pkgs: + _save_deps(ctx, installed_pkgs) + + +def _save_deps(ctx, pkgs, action="add"): + specs = [] + for library, pkg in pkgs.items(): + spec = PackageSpec(library) + if spec.external: + specs.append(spec) + else: + specs.append( + PackageSpec( + owner=pkg.metadata.spec.owner, + name=pkg.metadata.spec.name, + requirements=spec.requirements + or ( + ("^%s" % pkg.metadata.version) + if not pkg.metadata.version.build + else pkg.metadata.version + ), + ) + ) input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] for input_dir in input_dirs: - config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini")) - config.validate(project_environments) - for env in config.envs(): - if project_environments and env not in project_environments: - continue - config.expand_interpolations = False - try: - lib_deps = config.get("env:" + env, "lib_deps") - except InvalidProjectConfError: - lib_deps = [] - for library in libraries: - if library in lib_deps: - continue - manifest = installed_manifests[library] - try: - assert library.lower() == manifest["name"].lower() - assert semantic_version.Version(manifest["version"]) - lib_deps.append("{name}@^{version}".format(**manifest)) - except (AssertionError, ValueError): - lib_deps.append(library) - config.set("env:" + env, "lib_deps", lib_deps) - config.save() + if not is_platformio_project(input_dir): + continue + save_project_libdeps(input_dir, specs, project_environments, action=action) -@cli.command("uninstall", short_help="Uninstall libraries") +@cli.command("uninstall", short_help="Remove libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") +@click.option( + "--save/--no-save", + is_flag=True, + default=True, + help="Remove libraries from the `platformio.ini` dependency list and save changes" + " (enabled by default)", +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.pass_context -def lib_uninstall(ctx, libraries): +def lib_uninstall(ctx, libraries, save, silent): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + uninstalled_pkgs = {} for storage_dir in storage_dirs: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - for library in libraries: - lm.uninstall(library) + lm = LibraryPackageManager(storage_dir) + uninstalled_pkgs = { + library: lm.uninstall(library, silent=silent) for library in libraries + } + + if save and uninstalled_pkgs: + _save_deps(ctx, uninstalled_pkgs, action="remove") @cli.command("update", short_help="Update installed libraries") @@ -220,42 +242,53 @@ def lib_uninstall(ctx, libraries): @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) @click.pass_context -def lib_update(ctx, libraries, only_check, dry_run, json_output): +def lib_update( # pylint: disable=too-many-arguments + ctx, libraries, only_check, dry_run, silent, json_output +): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] only_check = dry_run or only_check json_result = {} for storage_dir in storage_dirs: if not json_output: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - - _libraries = libraries - if not _libraries: - _libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()] + lm = LibraryPackageManager(storage_dir) + _libraries = libraries or lm.get_installed() if only_check and json_output: result = [] for library in _libraries: - pkg_dir = library if os.path.isdir(library) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = lm.parse_pkg_uri(library) - pkg_dir = lm.get_package_dir(name, requirements, url) - if not pkg_dir: + spec = None + pkg = None + if isinstance(library, PackageSourceItem): + pkg = library + else: + spec = PackageSpec(library) + pkg = lm.get_package(spec) + if not pkg: continue - latest = lm.outdated(pkg_dir, requirements) - if not latest: + outdated = lm.outdated(pkg, spec) + if not outdated.is_outdated(allow_incompatible=True): continue - manifest = lm.load_manifest(pkg_dir) - manifest["versionLatest"] = latest + manifest = lm.legacy_load_manifest(pkg) + manifest["versionWanted"] = ( + str(outdated.wanted) if outdated.wanted else None + ) + manifest["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) result.append(manifest) json_result[storage_dir] = result else: for library in _libraries: - lm.update(library, only_check=only_check) + spec = ( + None + if isinstance(library, PackageSourceItem) + else PackageSpec(library) + ) + lm.update(library, spec=spec, only_check=only_check, silent=silent) if json_output: return click.echo( @@ -276,8 +309,8 @@ def lib_list(ctx, json_output): for storage_dir in storage_dirs: if not json_output: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - items = lm.get_installed() + lm = LibraryPackageManager(storage_dir) + items = lm.legacy_get_installed() if json_output: json_result[storage_dir] = items elif items: @@ -301,6 +334,7 @@ def lib_list(ctx, json_output): @click.option("--json-output", is_flag=True) @click.option("--page", type=click.INT, default=1) @click.option("--id", multiple=True) +@click.option("-o", "--owner", multiple=True) @click.option("-n", "--name", multiple=True) @click.option("-a", "--author", multiple=True) @click.option("-k", "--keyword", multiple=True) @@ -404,12 +438,8 @@ def lib_builtin(storage, json_output): @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) def lib_show(library, json_output): - lm = LibraryManager() - name, requirements, _ = lm.parse_pkg_uri(library) - lib_id = lm.search_lib_id( - {"name": name, "requirements": requirements}, - silent=json_output, - interactive=not json_output, + lib_id = LibraryPackageManager().reveal_registry_package_id( + library, silent=json_output ) lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py new file mode 100644 index 00000000..a5cc07e3 --- /dev/null +++ b/platformio/commands/lib/helpers.py @@ -0,0 +1,90 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio.compat import ci_strings_are_equal +from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.meta import PackageSpec +from platformio.project.config import ProjectConfig +from platformio.project.exception import InvalidProjectConfError + + +def get_builtin_libs(storage_names=None): + # pylint: disable=import-outside-toplevel + from platformio.package.manager.library import LibraryPackageManager + + items = [] + storage_names = storage_names or [] + pm = PlatformManager() + for manifest in pm.get_installed(): + p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + for storage in p.get_lib_storages(): + if storage_names and storage["name"] not in storage_names: + continue + lm = LibraryPackageManager(storage["path"]) + items.append( + { + "name": storage["name"], + "path": storage["path"], + "items": lm.legacy_get_installed(), + } + ) + return items + + +def is_builtin_lib(storages, name): + for storage in storages or []: + if any(lib.get("name") == name for lib in storage["items"]): + return True + return False + + +def ignore_deps_by_specs(deps, specs): + result = [] + for dep in deps: + depspec = PackageSpec(dep) + if depspec.external: + result.append(dep) + continue + ignore_conditions = [] + for spec in specs: + if depspec.owner: + ignore_conditions.append( + ci_strings_are_equal(depspec.owner, spec.owner) + and ci_strings_are_equal(depspec.name, spec.name) + ) + else: + ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name)) + if not any(ignore_conditions): + result.append(dep) + return result + + +def save_project_libdeps(project_dir, specs, environments=None, action="add"): + config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + config.expand_interpolations = False + lib_deps = [] + try: + lib_deps = ignore_deps_by_specs(config.get("env:" + env, "lib_deps"), specs) + except InvalidProjectConfError: + pass + if action == "add": + lib_deps.extend(spec.as_dependency() for spec in specs) + config.set("env:" + env, "lib_deps", lib_deps) + config.save() diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index a684e14a..af4071e0 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -26,9 +26,9 @@ from platformio.commands.system.completion import ( install_completion_code, uninstall_completion_code, ) -from platformio.managers.lib import LibraryManager from platformio.managers.package import PackageManager from platformio.managers.platform import PlatformManager +from platformio.package.manager.library import LibraryPackageManager from platformio.project.config import ProjectConfig @@ -73,7 +73,7 @@ def system_info(json_output): } data["global_lib_nums"] = { "title": "Global Libraries", - "value": len(LibraryManager().get_installed()), + "value": len(LibraryPackageManager().get_installed()), } data["dev_platform_nums"] = { "title": "Development Platforms", diff --git a/platformio/commands/update.py b/platformio/commands/update.py index 1bac4f77..bf829165 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -15,11 +15,11 @@ import click from platformio import app -from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY -from platformio.commands.lib import lib_update as cmd_lib_update +from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY +from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.managers.core import update_core_packages -from platformio.managers.lib import LibraryManager +from platformio.package.manager.library import LibraryPackageManager @click.command( @@ -55,5 +55,5 @@ def cli(ctx, core_packages, only_check, dry_run): click.echo() click.echo("Library Manager") click.echo("===============") - ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir] + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryPackageManager().package_dir] ctx.invoke(cmd_lib_update, only_check=only_check) diff --git a/platformio/compat.py b/platformio/compat.py index 7f749fc9..59362d01 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -50,6 +50,14 @@ def get_object_members(obj, ignore_private=True): } +def ci_strings_are_equal(a, b): + if a == b: + return True + if not a or not b: + return False + return a.strip().lower() == b.strip().lower() + + if PY2: import imp diff --git a/platformio/exception.py b/platformio/exception.py index c39b7957..9ab0e4d8 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -124,15 +124,6 @@ class PackageInstallError(PlatformIOPackageException): # -class LibNotFound(PlatformioException): - - MESSAGE = ( - "Library `{0}` has not been found in PlatformIO Registry.\n" - "You can ignore this message, if `{0}` is a built-in library " - "(included in framework, SDK). E.g., SPI, Wire, etc." - ) - - class NotGlobalLibDir(UserSideException): MESSAGE = ( diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 0c8ee2df..b0e64f52 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -21,13 +21,13 @@ import semantic_version from platformio import __version__, app, exception, fs, telemetry, util from platformio.commands import PlatformioCLI -from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY -from platformio.commands.lib import lib_update as cmd_lib_update +from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY +from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version from platformio.managers.core import update_core_packages -from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.library import LibraryPackageManager from platformio.proc import is_container @@ -240,7 +240,7 @@ def check_platformio_upgrade(): click.echo("") -def check_internal_updates(ctx, what): +def check_internal_updates(ctx, what): # pylint: disable=too-many-branches last_check = app.get_state_item("last_check", {}) interval = int(app.get_setting("check_%s_interval" % what)) * 3600 * 24 if (time() - interval) < last_check.get(what + "_update", 0): @@ -251,20 +251,27 @@ def check_internal_updates(ctx, what): util.internet_on(raise_exception=True) - pm = PlatformManager() if what == "platforms" else LibraryManager() outdated_items = [] - for manifest in pm.get_installed(): - if manifest["name"] in outdated_items: - continue - conds = [ - pm.outdated(manifest["__pkg_dir"]), - what == "platforms" - and PlatformFactory.newPlatform( - manifest["__pkg_dir"] - ).are_outdated_packages(), - ] - if any(conds): - outdated_items.append(manifest["name"]) + pm = PlatformManager() if what == "platforms" else LibraryPackageManager() + if isinstance(pm, PlatformManager): + for manifest in pm.get_installed(): + if manifest["name"] in outdated_items: + continue + conds = [ + pm.outdated(manifest["__pkg_dir"]), + what == "platforms" + and PlatformFactory.newPlatform( + manifest["__pkg_dir"] + ).are_outdated_packages(), + ] + if any(conds): + outdated_items.append(manifest["name"]) + else: + for pkg in pm.get_installed(): + if pkg.metadata.name in outdated_items: + continue + if pm.outdated(pkg).is_outdated(): + outdated_items.append(pkg.metadata.name) if not outdated_items: return diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py deleted file mode 100644 index 6e6b1b7d..00000000 --- a/platformio/managers/lib.py +++ /dev/null @@ -1,374 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=too-many-arguments, too-many-locals, too-many-branches -# pylint: disable=too-many-return-statements - -import json -from glob import glob -from os.path import isdir, join - -import click -import semantic_version - -from platformio import app, exception, util -from platformio.compat import glob_escape -from platformio.managers.package import BasePkgManager -from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.package.exception import ManifestException -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.project.config import ProjectConfig - - -class LibraryManager(BasePkgManager): - - FILE_CACHE_VALID = "30d" # 1 month - - def __init__(self, package_dir=None): - self.config = ProjectConfig.get_instance() - super(LibraryManager, self).__init__( - package_dir or self.config.get_optional_dir("globallib") - ) - - @property - def manifest_names(self): - return [".library.json", "library.json", "library.properties", "module.json"] - - def get_manifest_path(self, pkg_dir): - path = BasePkgManager.get_manifest_path(self, pkg_dir) - if path: - return path - - # if library without manifest, returns first source file - src_dir = join(glob_escape(pkg_dir)) - if isdir(join(pkg_dir, "src")): - src_dir = join(src_dir, "src") - chs_files = glob(join(src_dir, "*.[chS]")) - if chs_files: - return chs_files[0] - cpp_files = glob(join(src_dir, "*.cpp")) - if cpp_files: - return cpp_files[0] - - return None - - def max_satisfying_repo_version(self, versions, requirements=None): - def _cmp_dates(datestr1, datestr2): - date1 = util.parse_date(datestr1) - date2 = util.parse_date(datestr2) - if date1 == date2: - return 0 - return -1 if date1 < date2 else 1 - - semver_spec = None - try: - semver_spec = ( - semantic_version.SimpleSpec(requirements) if requirements else None - ) - except ValueError: - pass - - item = {} - - for v in versions: - semver_new = self.parse_semver_version(v["name"]) - if semver_spec: - # pylint: disable=unsupported-membership-test - if not semver_new or semver_new not in semver_spec: - continue - if not item or self.parse_semver_version(item["name"]) < semver_new: - item = v - elif requirements: - if requirements == v["name"]: - return v - - else: - if not item or _cmp_dates(item["released"], v["released"]) == -1: - item = v - return item - - def get_latest_repo_version(self, name, requirements, silent=False): - item = self.max_satisfying_repo_version( - util.get_api_result( - "/lib/info/%d" - % self.search_lib_id( - {"name": name, "requirements": requirements}, silent=silent - ), - cache_valid="1h", - )["versions"], - requirements, - ) - return item["name"] if item else None - - def _install_from_piorepo(self, name, requirements): - assert name.startswith("id="), name - version = self.get_latest_repo_version(name, requirements) - if not version: - raise exception.UndefinedPackageVersion( - requirements or "latest", util.get_systype() - ) - dl_data = util.get_api_result( - "/lib/download/" + str(name[3:]), dict(version=version), cache_valid="30d" - ) - assert dl_data - - return self._install_from_url( - name, - dl_data["url"].replace("http://", "https://") - if app.get_setting("strict_ssl") - else dl_data["url"], - requirements, - ) - - def search_lib_id( # pylint: disable=too-many-branches - self, filters, silent=False, interactive=False - ): - assert isinstance(filters, dict) - assert "name" in filters - - # try to find ID within installed packages - lib_id = self._get_lib_id_from_installed(filters) - if lib_id: - return lib_id - - # looking in PIO Library Registry - if not silent: - click.echo( - "Looking for %s library in registry" - % click.style(filters["name"], fg="cyan") - ) - query = [] - for key in filters: - if key not in ("name", "authors", "frameworks", "platforms"): - continue - values = filters[key] - if not isinstance(values, list): - values = [v.strip() for v in values.split(",") if v] - for value in values: - query.append( - '%s:"%s"' % (key[:-1] if key.endswith("s") else key, value) - ) - - lib_info = None - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h" - ) - if result["total"] == 1: - lib_info = result["items"][0] - elif result["total"] > 1: - if silent and not interactive: - lib_info = result["items"][0] - else: - click.secho( - "Conflict: More than one library has been found " - "by request %s:" % json.dumps(filters), - fg="yellow", - err=True, - ) - # pylint: disable=import-outside-toplevel - from platformio.commands.lib import print_lib_item - - for item in result["items"]: - print_lib_item(item) - - if not interactive: - click.secho( - "Automatically chose the first available library " - "(use `--interactive` option to make a choice)", - fg="yellow", - err=True, - ) - lib_info = result["items"][0] - else: - deplib_id = click.prompt( - "Please choose library ID", - type=click.Choice([str(i["id"]) for i in result["items"]]), - ) - for item in result["items"]: - if item["id"] == int(deplib_id): - lib_info = item - break - - if not lib_info: - if list(filters) == ["name"]: - raise exception.LibNotFound(filters["name"]) - raise exception.LibNotFound(str(filters)) - if not silent: - click.echo( - "Found: %s" - % click.style( - "https://platformio.org/lib/show/{id}/{name}".format(**lib_info), - fg="blue", - ) - ) - return int(lib_info["id"]) - - def _get_lib_id_from_installed(self, filters): - if filters["name"].startswith("id="): - return int(filters["name"][3:]) - package_dir = self.get_package_dir( - filters["name"], filters.get("requirements", filters.get("version")) - ) - if not package_dir: - return None - manifest = self.load_manifest(package_dir) - if "id" not in manifest: - return None - - for key in ("frameworks", "platforms"): - if key not in filters: - continue - if key not in manifest: - return None - if not util.items_in_list( - util.items_to_list(filters[key]), util.items_to_list(manifest[key]) - ): - return None - - if "authors" in filters: - if "authors" not in manifest: - return None - manifest_authors = manifest["authors"] - if not isinstance(manifest_authors, list): - manifest_authors = [manifest_authors] - manifest_authors = [ - a["name"] - for a in manifest_authors - if isinstance(a, dict) and "name" in a - ] - filter_authors = filters["authors"] - if not isinstance(filter_authors, list): - filter_authors = [filter_authors] - if not set(filter_authors) <= set(manifest_authors): - return None - - return int(manifest["id"]) - - def install( # pylint: disable=arguments-differ - self, - name, - requirements=None, - silent=False, - after_update=False, - interactive=False, - force=False, - ): - _name, _requirements, _url = self.parse_pkg_uri(name, requirements) - if not _url: - name = "id=%d" % self.search_lib_id( - {"name": _name, "requirements": _requirements}, - silent=silent, - interactive=interactive, - ) - requirements = _requirements - pkg_dir = BasePkgManager.install( - self, - name, - requirements, - silent=silent, - after_update=after_update, - force=force, - ) - - if not pkg_dir: - return None - - manifest = None - try: - manifest = ManifestParserFactory.new_from_dir(pkg_dir).as_dict() - except ManifestException: - pass - if not manifest or not manifest.get("dependencies"): - return pkg_dir - - if not silent: - click.secho("Installing dependencies", fg="yellow") - - builtin_lib_storages = None - for filters in manifest["dependencies"]: - assert "name" in filters - - # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] - history_key = str(filters) - if history_key in self.INSTALL_HISTORY: - continue - self.INSTALL_HISTORY.append(history_key) - - if any(s in filters.get("version", "") for s in ("\\", "/")): - self.install( - "{name}={version}".format(**filters), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - else: - try: - lib_id = self.search_lib_id(filters, silent, interactive) - except exception.LibNotFound as e: - if builtin_lib_storages is None: - builtin_lib_storages = get_builtin_libs() - if not silent or is_builtin_lib( - builtin_lib_storages, filters["name"] - ): - click.secho("Warning! %s" % e, fg="yellow") - continue - - if filters.get("version"): - self.install( - lib_id, - filters.get("version"), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - else: - self.install( - lib_id, - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - return pkg_dir - - -def get_builtin_libs(storage_names=None): - items = [] - storage_names = storage_names or [] - pm = PlatformManager() - for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) - for storage in p.get_lib_storages(): - if storage_names and storage["name"] not in storage_names: - continue - lm = LibraryManager(storage["path"]) - items.append( - { - "name": storage["name"], - "path": storage["path"], - "items": lm.get_installed(), - } - ) - return items - - -def is_builtin_lib(storages, name): - for storage in storages or []: - if any(l.get("name") == name for l in storage["items"]): - return True - return False diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 346cce59..071d6788 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -482,7 +482,7 @@ class PkgInstallerMixin(object): self.unpack(dlpath, tmp_dir) os.remove(dlpath) else: - vcs = VCSClientFactory.newClient(tmp_dir, url) + vcs = VCSClientFactory.new(tmp_dir, url) assert vcs.export() src_manifest_dir = vcs.storage_dir src_manifest["version"] = vcs.get_current_revision() @@ -628,9 +628,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if "__src_url" in manifest: try: - vcs = VCSClientFactory.newClient( - pkg_dir, manifest["__src_url"], silent=True - ) + vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"], silent=True) except (AttributeError, exception.PlatformioException): return None if not vcs.can_be_updated: @@ -800,7 +798,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): return True if "__src_url" in manifest: - vcs = VCSClientFactory.newClient(pkg_dir, manifest["__src_url"]) + vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"]) assert vcs.update() self._update_src_manifest( dict(version=vcs.get_current_revision()), vcs.storage_dir diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index ea409f2e..cb565f10 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -20,7 +20,7 @@ import tempfile import click from platformio import app, compat, fs, util -from platformio.package.exception import PackageException +from platformio.package.exception import MissingPackageManifestError, PackageException from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -83,7 +83,7 @@ class PackageManagerInstallMixin(object): msg = "Installing %s" % click.style(spec.humanize(), fg="cyan") self.print_message(msg) - if spec.url: + if spec.external: pkg = self.install_from_url(spec.url, spec, silent=silent) else: pkg = self.install_from_registry(spec, search_filters, silent=silent) @@ -152,7 +152,7 @@ class PackageManagerInstallMixin(object): assert os.path.isfile(dl_path) self.unpack(dl_path, tmp_dir) else: - vcs = VCSClientFactory.newClient(tmp_dir, url) + vcs = VCSClientFactory.new(tmp_dir, url) assert vcs.export() root_dir = self.find_pkg_root(tmp_dir, spec) @@ -189,12 +189,20 @@ class PackageManagerInstallMixin(object): # what to do with existing package? action = "overwrite" - if dst_pkg.metadata and dst_pkg.metadata.spec.url: + if tmp_pkg.metadata.spec.has_custom_name(): + action = "overwrite" + dst_pkg = PackageSourceItem( + os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) + ) + elif dst_pkg.metadata and dst_pkg.metadata.spec.external: if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: action = "detach-existing" - elif tmp_pkg.metadata.spec.url: + elif tmp_pkg.metadata.spec.external: action = "detach-new" - elif dst_pkg.metadata and dst_pkg.metadata.version != tmp_pkg.metadata.version: + elif dst_pkg.metadata and ( + dst_pkg.metadata.version != tmp_pkg.metadata.version + or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner + ): action = ( "detach-existing" if tmp_pkg.metadata.version > dst_pkg.metadata.version @@ -231,7 +239,7 @@ class PackageManagerInstallMixin(object): tmp_pkg.get_safe_dirname(), tmp_pkg.metadata.version, ) - if tmp_pkg.metadata.spec.url: + if tmp_pkg.metadata.spec.external: target_dirname = "%s@src-%s" % ( tmp_pkg.get_safe_dirname(), hashlib.md5( @@ -247,3 +255,20 @@ class PackageManagerInstallMixin(object): _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) + + def get_installed(self): + result = [] + for name in os.listdir(self.package_dir): + pkg_dir = os.path.join(self.package_dir, name) + if not os.path.isdir(pkg_dir): + continue + pkg = PackageSourceItem(pkg_dir) + if not pkg.metadata: + try: + spec = self.build_legacy_spec(pkg_dir) + pkg.metadata = self.build_metadata(pkg_dir, spec) + except MissingPackageManifestError: + pass + if pkg.metadata: + result.append(pkg) + return result diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py new file mode 100644 index 00000000..22478eff --- /dev/null +++ b/platformio/package/manager/_legacy.py @@ -0,0 +1,57 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs +from platformio.package.meta import PackageSourceItem, PackageSpec + + +class PackageManagerLegacyMixin(object): + def build_legacy_spec(self, pkg_dir): + # find src manifest + src_manifest_name = ".piopkgmanager.json" + src_manifest_path = None + for name in os.listdir(pkg_dir): + if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): + continue + src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) + break + + if src_manifest_path: + src_manifest = fs.load_json(src_manifest_path) + return PackageSpec( + name=src_manifest.get("name"), + url=src_manifest.get("url"), + requirements=src_manifest.get("requirements"), + ) + + # fall back to a package manifest + manifest = self.load_manifest(pkg_dir) + return PackageSpec(name=manifest.get("name")) + + def legacy_load_manifest(self, pkg): + assert isinstance(pkg, PackageSourceItem) + manifest = self.load_manifest(pkg) + manifest["__pkg_dir"] = pkg.path + for key in ("name", "version"): + if not manifest.get(key): + manifest[key] = str(getattr(pkg.metadata, key)) + if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external: + manifest["__src_url"] = pkg.metadata.spec.url + manifest["version"] = str(pkg.metadata.version) + return manifest + + def legacy_get_installed(self): + return [self.legacy_load_manifest(pkg) for pkg in self.get_installed()] diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index d5c9ddad..41ca58b3 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -79,10 +79,10 @@ class RegistryFileMirrorsIterator(object): class PackageManageRegistryMixin(object): def install_from_registry(self, spec, search_filters=None, silent=False): if spec.owner and spec.name and not search_filters: - package = self.fetch_registry_package(spec.owner, spec.name) + package = self.fetch_registry_package(spec) if not package: raise UnknownPackageError(spec.humanize()) - version = self._pick_best_pkg_version(package["versions"], spec) + version = self.pick_best_registry_version(package["versions"], spec) else: packages = self.search_registry_packages(spec, search_filters) if not packages: @@ -131,10 +131,33 @@ class PackageManageRegistryMixin(object): "items" ] - def fetch_registry_package(self, owner, name): - return self.get_registry_client_instance().get_package( - self.pkg_type, owner, name - ) + def fetch_registry_package(self, spec): + result = None + if spec.owner and spec.name: + result = self.get_registry_client_instance().get_package( + self.pkg_type, spec.owner, spec.name + ) + if not result and (spec.id or (spec.name and not spec.owner)): + packages = self.search_registry_packages(spec) + if packages: + result = self.get_registry_client_instance().get_package( + self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"] + ) + if not result: + raise UnknownPackageError(spec.humanize()) + return result + + def reveal_registry_package_id(self, spec, silent=False): + spec = self.ensure_spec(spec) + if spec.id: + return spec.id + packages = self.search_registry_packages(spec) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + click.echo("") + return packages[0]["id"] @staticmethod def print_multi_package_issue(packages, spec): @@ -160,7 +183,7 @@ class PackageManageRegistryMixin(object): def find_best_registry_version(self, packages, spec): # find compatible version within the latest package versions for package in packages: - version = self._pick_best_pkg_version([package["version"]], spec) + version = self.pick_best_registry_version([package["version"]], spec) if version: return (package, version) @@ -169,9 +192,13 @@ class PackageManageRegistryMixin(object): # if the custom version requirements, check ALL package versions for package in packages: - version = self._pick_best_pkg_version( + version = self.pick_best_registry_version( self.fetch_registry_package( - package["owner"]["username"], package["name"] + PackageSpec( + id=package["id"], + owner=package["owner"]["username"], + name=package["name"], + ) ).get("versions"), spec, ) @@ -180,11 +207,12 @@ class PackageManageRegistryMixin(object): time.sleep(1) return None - def _pick_best_pkg_version(self, versions, spec): + def pick_best_registry_version(self, versions, spec=None): + assert not spec or isinstance(spec, PackageSpec) best = None for version in versions: semver = PackageMetaData.to_semver(version["name"]) - if spec.requirements and semver not in spec.requirements: + if spec and spec.requirements and semver not in spec.requirements: continue if not any( self.is_system_compatible(f.get("system")) for f in version["files"] diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index e754eab2..813ada6d 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -31,10 +31,7 @@ class PackageManagerUninstallMixin(object): self.unlock() def _uninstall(self, pkg, silent=False, skip_dependencies=False): - if not isinstance(pkg, PackageSourceItem): - pkg = ( - PackageSourceItem(pkg) if os.path.isdir(pkg) else self.get_package(pkg) - ) + pkg = self.get_package(pkg) if not pkg or not pkg.metadata: raise UnknownPackageError(pkg) @@ -73,7 +70,7 @@ class PackageManagerUninstallMixin(object): if not silent: click.echo("[%s]" % click.style("OK", fg="green")) - return True + return pkg def _uninstall_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageSourceItem) diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py new file mode 100644 index 00000000..87d5e7f4 --- /dev/null +++ b/platformio/package/manager/_update.py @@ -0,0 +1,166 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click + +from platformio import util +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import ( + PackageOutdatedResult, + PackageSourceItem, + PackageSpec, +) +from platformio.package.vcsclient import VCSBaseException, VCSClientFactory + + +class PackageManagerUpdateMixin(object): + def outdated(self, pkg, spec=None): + assert isinstance(pkg, PackageSourceItem) + assert not spec or isinstance(spec, PackageSpec) + assert os.path.isdir(pkg.path) and pkg.metadata + + # skip detached package to a specific version + detached_conditions = [ + "@" in pkg.path, + pkg.metadata.spec and not pkg.metadata.spec.external, + not spec, + ] + if all(detached_conditions): + return PackageOutdatedResult(current=pkg.metadata.version, detached=True) + + latest = None + wanted = None + if pkg.metadata.spec.external: + latest = self._fetch_vcs_latest_version(pkg) + else: + try: + reg_pkg = self.fetch_registry_package(pkg.metadata.spec) + latest = ( + self.pick_best_registry_version(reg_pkg["versions"]) or {} + ).get("name") + if spec: + wanted = ( + self.pick_best_registry_version(reg_pkg["versions"], spec) or {} + ).get("name") + if not wanted: # wrong library + latest = None + except UnknownPackageError: + pass + + return PackageOutdatedResult( + current=pkg.metadata.version, latest=latest, wanted=wanted + ) + + def _fetch_vcs_latest_version(self, pkg): + vcs = None + try: + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url, silent=True) + except VCSBaseException: + return None + if not vcs.can_be_updated: + return None + return str( + self.build_metadata( + pkg.path, pkg.metadata.spec, vcs_revision=vcs.get_latest_revision() + ).version + ) + + def update(self, pkg, spec=None, only_check=False, silent=False): + pkg = self.get_package(pkg) + if not pkg or not pkg.metadata: + raise UnknownPackageError(pkg) + + if not silent: + click.echo( + "{} {:<45} {:<30}".format( + "Checking" if only_check else "Updating", + click.style(pkg.metadata.spec.humanize(), fg="cyan"), + "%s (%s)" % (pkg.metadata.version, spec.requirements) + if spec and spec.requirements + else str(pkg.metadata.version), + ), + nl=False, + ) + if not util.internet_on(): + if not silent: + click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) + return pkg + + outdated = self.outdated(pkg, spec) + if not silent: + self.print_outdated_state(outdated) + + up_to_date = any( + [ + outdated.detached, + not outdated.latest, + outdated.latest and outdated.current == outdated.latest, + outdated.wanted and outdated.current == outdated.wanted, + ] + ) + if only_check or up_to_date: + return pkg + + try: + self.lock() + return self._update(pkg, outdated, silent=silent) + finally: + self.unlock() + + @staticmethod + def print_outdated_state(outdated): + if outdated.detached: + return click.echo("[%s]" % (click.style("Detached", fg="yellow"))) + if not outdated.latest or outdated.current == outdated.latest: + return click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) + if outdated.wanted and outdated.current == outdated.wanted: + return click.echo( + "[%s]" + % (click.style("Incompatible (%s)" % outdated.latest, fg="yellow")) + ) + return click.echo( + "[%s]" % (click.style(str(outdated.wanted or outdated.latest), fg="red")) + ) + + def _update(self, pkg, outdated, silent=False): + if pkg.metadata.spec.external: + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url) + assert vcs.update() + pkg.metadata.version = self._fetch_vcs_latest_version(pkg) + pkg.dump_meta() + return pkg + + new_pkg = self.install( + PackageSpec( + id=pkg.metadata.spec.id, + owner=pkg.metadata.spec.owner, + name=pkg.metadata.spec.name, + requirements=outdated.wanted or outdated.latest, + ), + silent=silent, + ) + if new_pkg: + old_pkg = self.get_package( + PackageSpec( + id=pkg.metadata.spec.id, + owner=pkg.metadata.spec.owner, + name=pkg.metadata.name, + requirements=pkg.metadata.version, + ) + ) + if old_pkg: + self.uninstall(old_pkg, silent=silent, skip_dependencies=True) + return new_pkg diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index ca065833..ee2928c5 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -18,14 +18,17 @@ from datetime import datetime import click import semantic_version -from platformio import fs, util +from platformio import util from platformio.commands import PlatformioCLI +from platformio.compat import ci_strings_are_equal from platformio.package.exception import ManifestException, MissingPackageManifestError from platformio.package.lockfile import LockFile from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin +from platformio.package.manager._legacy import PackageManagerLegacyMixin from platformio.package.manager._registry import PackageManageRegistryMixin from platformio.package.manager._uninstall import PackageManagerUninstallMixin +from platformio.package.manager._update import PackageManagerUpdateMixin from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import ( PackageMetaData, @@ -41,6 +44,8 @@ class BasePackageManager( # pylint: disable=too-many-public-methods PackageManageRegistryMixin, PackageManagerInstallMixin, PackageManagerUninstallMixin, + PackageManagerUpdateMixin, + PackageManagerLegacyMixin, ): _MEMORY_CACHE = {} @@ -83,10 +88,6 @@ class BasePackageManager( # pylint: disable=too-many-public-methods return True return util.items_in_list(value, util.get_systype()) - @staticmethod - def generate_rand_version(): - return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") - @staticmethod def ensure_dir_exists(path): if not os.path.isdir(path): @@ -162,27 +163,9 @@ class BasePackageManager( # pylint: disable=too-many-public-methods click.secho(str(e), fg="yellow") raise MissingPackageManifestError(", ".join(self.manifest_names)) - def build_legacy_spec(self, pkg_dir): - # find src manifest - src_manifest_name = ".piopkgmanager.json" - src_manifest_path = None - for name in os.listdir(pkg_dir): - if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): - continue - src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) - break - - if src_manifest_path: - src_manifest = fs.load_json(src_manifest_path) - return PackageSpec( - name=src_manifest.get("name"), - url=src_manifest.get("url"), - requirements=src_manifest.get("requirements"), - ) - - # fall back to a package manifest - manifest = self.load_manifest(pkg_dir) - return PackageSpec(name=manifest.get("name")) + @staticmethod + def generate_rand_version(): + return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") def build_metadata(self, pkg_dir, spec, vcs_revision=None): manifest = self.load_manifest(pkg_dir) @@ -192,7 +175,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods version=manifest.get("version"), spec=spec, ) - if not metadata.name or spec.is_custom_name(): + if not metadata.name or spec.has_custom_name(): metadata.name = spec.name if vcs_revision: metadata.version = "%s+sha.%s" % ( @@ -203,42 +186,27 @@ class BasePackageManager( # pylint: disable=too-many-public-methods metadata.version = self.generate_rand_version() return metadata - def get_installed(self): - result = [] - for name in os.listdir(self.package_dir): - pkg_dir = os.path.join(self.package_dir, name) - if not os.path.isdir(pkg_dir): - continue - pkg = PackageSourceItem(pkg_dir) - if not pkg.metadata: - try: - spec = self.build_legacy_spec(pkg_dir) - pkg.metadata = self.build_metadata(pkg_dir, spec) - except MissingPackageManifestError: - pass - if pkg.metadata: - result.append(pkg) - return result - def get_package(self, spec): - def _ci_strings_are_equal(a, b): - if a == b: - return True - if not a or not b: - return False - return a.strip().lower() == b.strip().lower() + if isinstance(spec, PackageSourceItem): + return spec + + if not isinstance(spec, PackageSpec) and os.path.isdir(spec): + for pkg in self.get_installed(): + if spec == pkg.path: + return pkg + return None spec = self.ensure_spec(spec) best = None for pkg in self.get_installed(): skip_conditions = [ spec.owner - and not _ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner), - spec.url and spec.url != pkg.metadata.spec.url, + and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner), + spec.external and spec.url != pkg.metadata.spec.url, spec.id and spec.id != pkg.metadata.spec.id, not spec.id - and not spec.url - and not _ci_strings_are_equal(spec.name, pkg.metadata.name), + and not spec.external + and not ci_strings_are_equal(spec.name, pkg.metadata.name), ] if any(skip_conditions): continue diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 9fe924b9..1375e84e 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -21,7 +21,7 @@ from platformio.package.meta import PackageSpec, PackageType from platformio.project.helpers import get_project_global_lib_dir -class LibraryPackageManager(BasePackageManager): +class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): super(LibraryPackageManager, self).__init__( PackageType.LIBRARY, package_dir or get_project_global_lib_dir() diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 627bad47..c79e7d10 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -17,7 +17,7 @@ from platformio.package.meta import PackageType from platformio.project.config import ProjectConfig -class PlatformPackageManager(BasePackageManager): +class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): self.config = ProjectConfig.get_instance() super(PlatformPackageManager, self).__init__( diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py index db660303..ae111798 100644 --- a/platformio/package/manager/tool.py +++ b/platformio/package/manager/tool.py @@ -17,7 +17,7 @@ from platformio.package.meta import PackageType from platformio.project.config import ProjectConfig -class ToolPackageManager(BasePackageManager): +class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): self.config = ProjectConfig.get_instance() super(ToolPackageManager, self).__init__( diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index e19e6f25..b886fb5a 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -250,7 +250,7 @@ class ManifestSchema(BaseSchema): def load_spdx_licenses(): r = requests.get( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.9/json/licenses.json" + "/v3.10/json/licenses.json" ) r.raise_for_status() return r.json() diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 6cd2904b..af1e0baa 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -65,7 +65,44 @@ class PackageType(object): return None -class PackageSpec(object): +class PackageOutdatedResult(object): + def __init__(self, current, latest=None, wanted=None, detached=False): + self.current = current + self.latest = latest + self.wanted = wanted + self.detached = detached + + def __repr__(self): + return ( + "PackageOutdatedResult ".format( + current=self.current, + latest=self.latest, + wanted=self.wanted, + detached=self.detached, + ) + ) + + def __setattr__(self, name, value): + if ( + value + and name in ("current", "latest", "wanted") + and not isinstance(value, semantic_version.Version) + ): + value = semantic_version.Version(str(value)) + return super(PackageOutdatedResult, self).__setattr__(name, value) + + def is_outdated(self, allow_incompatible=False): + if self.detached or not self.latest or self.current == self.latest: + return False + if allow_incompatible: + return self.current != self.latest + if self.wanted: + return self.current != self.wanted + return True + + +class PackageSpec(object): # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=redefined-builtin,too-many-arguments self, raw=None, owner=None, id=None, name=None, requirements=None, url=None ): @@ -74,6 +111,7 @@ class PackageSpec(object): self.name = name self._requirements = None self.url = url + self.raw = raw if requirements: self.requirements = requirements self._name_is_custom = False @@ -104,6 +142,10 @@ class PackageSpec(object): "requirements={requirements} url={url}>".format(**self.as_dict()) ) + @property + def external(self): + return bool(self.url) + @property def requirements(self): return self._requirements @@ -116,24 +158,24 @@ class PackageSpec(object): self._requirements = ( value if isinstance(value, semantic_version.SimpleSpec) - else semantic_version.SimpleSpec(value) + else semantic_version.SimpleSpec(str(value)) ) def humanize(self): + result = "" if self.url: result = self.url - elif self.id: - result = "id:%d" % self.id - else: - result = "" + elif self.name: if self.owner: result = self.owner + "/" result += self.name + elif self.id: + result = "id:%d" % self.id if self.requirements: result += " @ " + str(self.requirements) return result - def is_custom_name(self): + def has_custom_name(self): return self._name_is_custom def as_dict(self): @@ -145,6 +187,19 @@ class PackageSpec(object): url=self.url, ) + def as_dependency(self): + if self.url: + return self.raw or self.url + result = "" + if self.name: + result = "%s/%s" % (self.owner, self.name) if self.owner else self.name + elif self.id: + result = str(self.id) + assert result + if self.requirements: + result = "%s@%s" % (result, self.requirements) + return result + def _parse(self, raw): if raw is None: return diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index 56291966..2e9bb238 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -17,7 +17,11 @@ from os.path import join from subprocess import CalledProcessError, check_call from sys import modules -from platformio.exception import PlatformioException, UserSideException +from platformio.package.exception import ( + PackageException, + PlatformioException, + UserSideException, +) from platformio.proc import exec_command try: @@ -26,9 +30,13 @@ except ImportError: from urlparse import urlparse +class VCSBaseException(PackageException): + pass + + class VCSClientFactory(object): @staticmethod - def newClient(src_dir, remote_url, silent=False): + def new(src_dir, remote_url, silent=False): result = urlparse(remote_url) type_ = result.scheme tag = None @@ -41,12 +49,15 @@ class VCSClientFactory(object): if "#" in remote_url: remote_url, tag = remote_url.rsplit("#", 1) if not type_: - raise PlatformioException("VCS: Unknown repository type %s" % remote_url) - obj = getattr(modules[__name__], "%sClient" % type_.title())( - src_dir, remote_url, tag, silent - ) - assert isinstance(obj, VCSClientBase) - return obj + raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) + try: + obj = getattr(modules[__name__], "%sClient" % type_.title())( + src_dir, remote_url, tag, silent + ) + assert isinstance(obj, VCSClientBase) + return obj + except (AttributeError, AssertionError): + raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) class VCSClientBase(object): @@ -101,7 +112,7 @@ class VCSClientBase(object): check_call(args, **kwargs) return True except CalledProcessError as e: - raise PlatformioException("VCS: Could not process command %s" % e.cmd) + raise VCSBaseException("VCS: Could not process command %s" % e.cmd) def get_cmd_output(self, args, **kwargs): args = [self.command] + args @@ -110,7 +121,7 @@ class VCSClientBase(object): result = exec_command(args, **kwargs) if result["returncode"] == 0: return result["out"].strip() - raise PlatformioException( + raise VCSBaseException( "VCS: Could not receive an output from `%s` command (%s)" % (args, result) ) @@ -227,7 +238,6 @@ class SvnClient(VCSClientBase): return self.run_cmd(args) def update(self): - args = ["update"] return self.run_cmd(args) @@ -239,4 +249,4 @@ class SvnClient(VCSClientBase): line = line.strip() if line.startswith("Revision:"): return line.split(":", 1)[1].strip() - raise PlatformioException("Could not detect current SVN revision") + raise VCSBaseException("Could not detect current SVN revision") diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index f3308a6a..0ea22dd6 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -15,7 +15,7 @@ from os.path import isfile, join from platformio.commands.ci import cli as cmd_ci -from platformio.commands.lib import cli as cmd_lib +from platformio.commands.lib.command import cli as cmd_lib def test_ci_empty(clirunner): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index f51b9dc2..1880d671 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -13,332 +13,184 @@ # limitations under the License. import json -import re +import os -from platformio import exception -from platformio.commands import PlatformioCLI -from platformio.commands.lib import cli as cmd_lib +import semantic_version -PlatformioCLI.leftover_args = ["--json-output"] # hook for click +from platformio.clients.registry import RegistryClient +from platformio.commands.lib.command import cli as cmd_lib +from platformio.package.meta import PackageType +from platformio.package.vcsclient import VCSClientFactory +from platformio.project.config import ProjectConfig -def test_search(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) +def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): + regclient = RegistryClient() + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write( + """ +[env] +lib_deps = ArduinoJson + +[env:one] +board = devkit + +[env:two] +framework = foo +lib_deps = + CustomLib + ArduinoJson @ 5.10.1 +""" + ) + result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "install", "64"]) validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) - assert int(match.group(1)) > 2 + aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert config.get("env:one", "lib_deps") == [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"] + ] + assert config.get("env:two", "lib_deps") == [ + "CustomLib", + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + ] - result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) + # ensure "build" version without NPM spec + result = clirunner.invoke( + cmd_lib, + ["-d", str(project_dir), "-e", "one", "install", "mbed-sam-grove/LinkedList"], + ) validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) - assert int(match.group(1)) > 1 + ll_pkg_data = regclient.get_package( + PackageType.LIBRARY, "mbed-sam-grove", "LinkedList" + ) + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert config.get("env:one", "lib_deps") == [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + ] - -def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): + # check external package via Git repo result = clirunner.invoke( cmd_lib, [ - "-g", + "-d", + str(project_dir), + "-e", + "one", "install", - "64", - "ArduinoJson@~5.10.0", - "547@2.2.4", - "AsyncMqttClient@<=0.8.2", - "Adafruit PN532@1.2.0", + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ], ) validate_cliresult(result) + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert len(config.get("env:one", "lib_deps")) == 3 + assert config.get("env:one", "lib_deps")[2] == ( + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3" + ) - # install unknown library - result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.LibNotFound) - - items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = [ - "ArduinoJson", - "ArduinoJson@5.10.1", - "NeoPixelBus", - "AsyncMqttClient", - "ESPAsyncTCP", - "AsyncTCP", - "Adafruit PN532", - "Adafruit BusIO", + # test uninstalling + result = clirunner.invoke( + cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"] + ) + validate_cliresult(result) + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert len(config.get("env:one", "lib_deps")) == 2 + assert len(config.get("env:two", "lib_deps")) == 1 + assert config.get("env:one", "lib_deps") == [ + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ] - assert set(items1) == set(items2) + + # test list + result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) + validate_cliresult(result) + assert "Version: 0.8.3+sha." in result.stdout + assert ( + "Source: git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" + in result.stdout + ) + result = clirunner.invoke( + cmd_lib, ["-d", str(project_dir), "list", "--json-output"] + ) + validate_cliresult(result) + data = {} + for key, value in json.loads(result.stdout).items(): + data[os.path.basename(key)] = value + ame_lib = next( + item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome" + ) + ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"]) + assert data["two"] == [] + assert "__pkg_dir" in data["one"][0] + assert ( + ame_lib["__src_url"] + == "git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" + ) + assert ame_lib["version"] == ("0.8.3+sha.%s" % ame_vcs.get_current_revision()) -def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): +def test_update(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("test-updates") + result = clirunner.invoke( + cmd_lib, + ["-d", str(storage_dir), "install", "ArduinoJson @ 5.10.1", "Blynk @ ~0.5.0"], + ) + validate_cliresult(result) + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "--json-output"] + ) + validate_cliresult(result) + outdated = json.loads(result.stdout) + assert len(outdated) == 2 + # ArduinoJson + assert outdated[0]["version"] == "5.10.1" + assert outdated[0]["versionWanted"] is None + assert semantic_version.Version( + outdated[0]["versionLatest"] + ) > semantic_version.Version("6.16.0") + # Blynk + assert outdated[1]["version"] == "0.5.4" + assert outdated[1]["versionWanted"] is None + assert semantic_version.Version( + outdated[1]["versionLatest"] + ) > semantic_version.Version("0.6.0") + + # check with spec result = clirunner.invoke( cmd_lib, [ - "-g", - "install", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", - "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz", - "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "-d", + str(storage_dir), + "update", + "--dry-run", + "--json-output", + "ArduinoJson @ ^5", ], ) validate_cliresult(result) - - # incorrect requirements + outdated = json.loads(result.stdout) + assert outdated[0]["version"] == "5.10.1" + assert outdated[0]["versionWanted"] == "5.13.4" + assert semantic_version.Version( + outdated[0]["versionLatest"] + ) > semantic_version.Version("6.16.0") + # update with spec result = clirunner.invoke( - cmd_lib, - [ - "-g", - "install", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", - ], + cmd_lib, ["-d", str(storage_dir), "update", "--silent", "ArduinoJson @ ^5.10.1"] ) - assert result.exit_code != 0 - - items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire", "ESP32WebServer"] - assert set(items1) >= set(items2) - - -def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): + validate_cliresult(result) result = clirunner.invoke( - cmd_lib, - [ - "-g", - "install", - "https://github.com/gioblu/PJON.git#3.0", - "https://github.com/gioblu/PJON.git#6.2", - "https://github.com/bblanchon/ArduinoJson.git", - "https://gitlab.com/ivankravets/rs485-nodeproto.git", - "https://github.com/platformio/platformio-libmirror.git", - # "https://developer.mbed.org/users/simon/code/TextLCD/", - "knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", - ], + cmd_lib, ["-d", str(storage_dir), "list", "--json-output"] ) validate_cliresult(result) - items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = [ - "PJON", - "PJON@src-79de467ebe19de18287becff0a1fb42d", - "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "rs485-nodeproto", - "platformio-libmirror", - "PubSubClient", - ] - assert set(items1) >= set(items2) + items = json.loads(result.stdout) + assert len(items) == 2 + assert items[0]["version"] == "5.13.4" + assert items[1]["version"] == "0.5.4" - -def test_install_duplicates(clirunner, validate_cliresult, without_internet): - # registry + # Check incompatible result = clirunner.invoke( - cmd_lib, - ["-g", "install", "http://dl.platformio.org/libraries/archives/0/9540.tar.gz"], + cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "ArduinoJson @ ^5"] ) validate_cliresult(result) - assert "is already installed" in result.output - - # archive - result = clirunner.invoke( - cmd_lib, - [ - "-g", - "install", - "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - ], - ) - validate_cliresult(result) - assert "is already installed" in result.output - - # repository - result = clirunner.invoke( - cmd_lib, - ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], - ) - validate_cliresult(result) - assert "is already installed" in result.output - - -def test_global_lib_list(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["-g", "list"]) - validate_cliresult(result) - assert all( - [ - n in result.output - for n in ( - "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - "Version: 5.10.1", - "Source: git+https://github.com/gioblu/PJON.git#3.0", - "Version: 1fb26fd", - ) - ] - ) - - result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) - assert all( - [ - n in result.output - for n in ( - "__pkg_dir", - '"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"', - '"version": "5.10.1"', - ) - ] - ) - items1 = [i["name"] for i in json.loads(result.output)] - items2 = [ - "ESP32WebServer", - "ArduinoJson", - "ArduinoJson", - "ArduinoJson", - "ArduinoJson", - "AsyncMqttClient", - "AsyncTCP", - "SomeLib", - "ESPAsyncTCP", - "NeoPixelBus", - "OneWire", - "PJON", - "PJON", - "PubSubClient", - "Adafruit PN532", - "Adafruit BusIO", - "platformio-libmirror", - "rs485-nodeproto", - ] - assert sorted(items1) == sorted(items2) - - versions1 = [ - "{name}@{version}".format(**item) for item in json.loads(result.output) - ] - versions2 = [ - "ArduinoJson@5.8.2", - "ArduinoJson@5.10.1", - "AsyncMqttClient@0.8.2", - "NeoPixelBus@2.2.4", - "PJON@07fe9aa", - "PJON@1fb26fd", - "PubSubClient@bef5814", - "Adafruit PN532@1.2.0", - ] - assert set(versions1) >= set(versions2) - - -def test_global_lib_update_check(clirunner, validate_cliresult): - result = clirunner.invoke( - cmd_lib, ["-g", "update", "--only-check", "--json-output"] - ) - validate_cliresult(result) - output = json.loads(result.output) - assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([l["name"] for l in output]) - - -def test_global_lib_update(clirunner, validate_cliresult): - # update library using package directory - result = clirunner.invoke( - cmd_lib, ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"] - ) - validate_cliresult(result) - oudated = json.loads(result.output) - assert len(oudated) == 1 - assert "__pkg_dir" in oudated[0] - result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) - validate_cliresult(result) - assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output - - # update rest libraries - result = clirunner.invoke(cmd_lib, ["-g", "update"]) - validate_cliresult(result) - assert result.output.count("[Detached]") == 5 - assert result.output.count("[Up-to-date]") == 12 - - # update unknown library - result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) - - -def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): - # uninstall using package directory - result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) - validate_cliresult(result) - items = json.loads(result.output) - items = sorted(items, key=lambda item: item["__pkg_dir"]) - result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) - validate_cliresult(result) - assert ("Uninstalling %s" % items[0]["name"]) in result.output - - # uninstall the rest libraries - result = clirunner.invoke( - cmd_lib, - [ - "-g", - "uninstall", - "OneWire", - "https://github.com/bblanchon/ArduinoJson.git", - "ArduinoJson@!=5.6.7", - "Adafruit PN532", - ], - ) - validate_cliresult(result) - - items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = [ - "rs485-nodeproto", - "platformio-libmirror", - "PubSubClient", - "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "ESPAsyncTCP", - "ESP32WebServer", - "NeoPixelBus", - "PJON", - "AsyncMqttClient", - "ArduinoJson", - "SomeLib_ID54", - "PJON@src-79de467ebe19de18287becff0a1fb42d", - "AsyncTCP", - ] - assert set(items1) == set(items2) - - # uninstall unknown library - result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) - - -def test_lib_show(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["show", "64"]) - validate_cliresult(result) - assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")]) - result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) - validate_cliresult(result) - assert "OneWire" in result.output - - -def test_lib_builtin(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["builtin"]) - validate_cliresult(result) - result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) - validate_cliresult(result) - - -def test_lib_stats(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["stats"]) - validate_cliresult(result) - assert all( - [ - s in result.output - for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") - ] - ) - - result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) - validate_cliresult(result) - assert set( - [ - "dlweek", - "added", - "updated", - "topkeywords", - "dlmonth", - "dlday", - "lastkeywords", - ] - ) == set(json.loads(result.output).keys()) + assert "Incompatible" in result.stdout diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py new file mode 100644 index 00000000..3f0f3725 --- /dev/null +++ b/tests/commands/test_lib_complex.py @@ -0,0 +1,348 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re + +from platformio import exception +from platformio.commands import PlatformioCLI +from platformio.commands.lib.command import cli as cmd_lib +from platformio.package.exception import UnknownPackageError + +PlatformioCLI.leftover_args = ["--json-output"] # hook for click + + +def test_search(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) + validate_cliresult(result) + match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + assert int(match.group(1)) > 2 + + result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) + validate_cliresult(result) + match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + assert int(match.group(1)) > 1 + + +def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "64", + "ArduinoJson@~5.10.0", + "547@2.2.4", + "AsyncMqttClient@<=0.8.2", + "Adafruit PN532@1.2.0", + ], + ) + validate_cliresult(result) + + # install unknown library + result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@5.10.1", + "NeoPixelBus", + "AsyncMqttClient", + "ESPAsyncTCP", + "AsyncTCP", + "Adafruit PN532", + "Adafruit BusIO", + ] + assert set(items1) == set(items2) + + +def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", + "SomeLib=https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) + validate_cliresult(result) + + # incorrect requirements + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", + ], + ) + assert result.exit_code != 0 + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "SomeLib", + "OneWire", + "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8", + ] + assert set(items1) >= set(items2) + + +def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/gioblu/PJON.git#3.0", + "https://github.com/gioblu/PJON.git#6.2", + "https://github.com/bblanchon/ArduinoJson.git", + "https://github.com/platformio/platformio-libmirror.git", + # "https://developer.mbed.org/users/simon/code/TextLCD/", + "https://github.com/knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", + ], + ) + validate_cliresult(result) + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377", + "PubSubClient@src-98ec699a461a31615982e5adaaefadda", + ] + assert set(items1) >= set(items2) + + +def test_install_duplicates(clirunner, validate_cliresult, without_internet): + # registry + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz", + ], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + # archive + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + # repository + result = clirunner.invoke( + cmd_lib, + ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + +def test_global_lib_list(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["-g", "list"]) + validate_cliresult(result) + assert all( + [ + n in result.output + for n in ( + "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "Version: 5.10.1", + "Source: git+https://github.com/gioblu/PJON.git#3.0", + "Version: 3.0.0+sha.1fb26fd", + ) + ] + ) + + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + assert all( + [ + n in result.output + for n in ( + "__pkg_dir", + '"__src_url": "git+https://github.com/gioblu/PJON.git#6.2"', + '"version": "5.10.1"', + ) + ] + ) + items1 = [i["name"] for i in json.loads(result.output)] + items2 = [ + "Adafruit BusIO", + "Adafruit PN532", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "AsyncMqttClient", + "AsyncTCP", + "DallasTemperature", + "ESP32WebServer", + "ESPAsyncTCP", + "NeoPixelBus", + "OneWire", + "PJON", + "PJON", + "platformio-libmirror", + "PubSubClient", + ] + assert sorted(items1) == sorted(items2) + + versions1 = [ + "{name}@{version}".format(**item) for item in json.loads(result.output) + ] + versions2 = [ + "ArduinoJson@5.8.2", + "ArduinoJson@5.10.1", + "AsyncMqttClient@0.8.2", + "NeoPixelBus@2.2.4", + "PJON@6.2.0+sha.07fe9aa", + "PJON@3.0.0+sha.1fb26fd", + "PubSubClient@2.6.0+sha.bef5814", + "Adafruit PN532@1.2.0", + ] + assert set(versions1) >= set(versions2) + + +def test_global_lib_update_check(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"]) + validate_cliresult(result) + output = json.loads(result.output) + assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([lib["name"] for lib in output]) + + +def test_global_lib_update(clirunner, validate_cliresult): + # update library using package directory + result = clirunner.invoke( + cmd_lib, ["-g", "update", "NeoPixelBus", "--dry-run", "--json-output"] + ) + validate_cliresult(result) + oudated = json.loads(result.output) + assert len(oudated) == 1 + assert "__pkg_dir" in oudated[0] + result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) + validate_cliresult(result) + assert "Removing NeoPixelBus @ 2.2.4" in result.output + + # update rest libraries + result = clirunner.invoke(cmd_lib, ["-g", "update"]) + validate_cliresult(result) + assert result.output.count("[Detached]") == 1 + assert result.output.count("[Up-to-date]") == 15 + + # update unknown library + result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): + # uninstall using package directory + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + validate_cliresult(result) + items = json.loads(result.output) + items = sorted(items, key=lambda item: item["__pkg_dir"]) + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) + validate_cliresult(result) + assert ("Removing %s" % items[0]["name"]) in result.output + + # uninstall the rest libraries + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "uninstall", + "OneWire", + "https://github.com/bblanchon/ArduinoJson.git", + "ArduinoJson@!=5.6.7", + "Adafruit PN532", + ], + ) + validate_cliresult(result) + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "AsyncMqttClient", + "AsyncTCP", + "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8", + "ESPAsyncTCP", + "NeoPixelBus", + "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377", + "PubSubClient@src-98ec699a461a31615982e5adaaefadda", + "SomeLib", + ] + assert set(items1) == set(items2) + + # uninstall unknown library + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_lib_show(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["show", "64"]) + validate_cliresult(result) + assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")]) + result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) + validate_cliresult(result) + assert "OneWire" in result.output + + +def test_lib_builtin(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["builtin"]) + validate_cliresult(result) + result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) + validate_cliresult(result) + + +def test_lib_stats(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["stats"]) + validate_cliresult(result) + assert all( + [ + s in result.output + for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") + ] + ) + + result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) + validate_cliresult(result) + assert set( + [ + "dlweek", + "added", + "updated", + "topkeywords", + "dlmonth", + "dlday", + "lastkeywords", + ] + ) == set(json.loads(result.output).keys()) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 5898ae2b..131346af 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -16,6 +16,7 @@ import os import time import pytest +import semantic_version from platformio import fs, util from platformio.package.exception import ( @@ -201,6 +202,12 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): assert lm.get_package("OneWire").metadata.version.major >= 2 assert len(lm.get_installed()) == 6 + # test conflicted names + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage"))) + lm.install("4@2.6.1", silent=True) + lm.install("5357@2.6.1", silent=True) + assert len(lm.get_installed()) == 2 + # Tools tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True) @@ -340,3 +347,81 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) assert lm.uninstall("AsyncMqttClient-esphome", silent=True) assert len(lm.get_installed()) == 0 + + +def test_registry(isolated_pio_core): + lm = LibraryPackageManager() + + # reveal ID + assert lm.reveal_registry_package_id(PackageSpec(id=13)) == 13 + assert lm.reveal_registry_package_id(PackageSpec(name="OneWire"), silent=True) == 1 + with pytest.raises(UnknownPackageError): + lm.reveal_registry_package_id(PackageSpec(name="/non-existing-package/")) + + # fetch package data + assert lm.fetch_registry_package(PackageSpec(id=1))["name"] == "OneWire" + assert lm.fetch_registry_package(PackageSpec(name="ArduinoJson"))["id"] == 64 + assert ( + lm.fetch_registry_package( + PackageSpec(id=13, owner="adafruit", name="Renamed library") + )["name"] + == "Adafruit GFX Library" + ) + with pytest.raises(UnknownPackageError): + lm.fetch_registry_package( + PackageSpec(owner="unknown<>owner", name="/non-existing-package/") + ) + with pytest.raises(UnknownPackageError): + lm.fetch_registry_package(PackageSpec(name="/non-existing-package/")) + + +def test_update_with_metadata(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.install("ArduinoJson @ 5.10.1", silent=True) + + # tesy latest + outdated = lm.outdated(pkg) + assert str(outdated.current) == "5.10.1" + assert outdated.wanted is None + assert outdated.latest > outdated.current + assert outdated.latest > semantic_version.Version("5.99.99") + + # test wanted + outdated = lm.outdated(pkg, PackageSpec("ArduinoJson@~5")) + assert str(outdated.current) == "5.10.1" + assert str(outdated.wanted) == "5.13.4" + assert outdated.latest > semantic_version.Version("6.16.0") + + # update to the wanted 5.x + new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5"), silent=True) + assert str(new_pkg.metadata.version) == "5.13.4" + # check that old version is removed + assert len(lm.get_installed()) == 1 + + # update to the latest + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.update("ArduinoJson", silent=True) + assert pkg.metadata.version == outdated.latest + + +def test_update_without_metadata(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + storage_dir.join("legacy-package").mkdir().join("library.json").write( + '{"name": "AsyncMqttClient-esphome", "version": "0.8.2"}' + ) + storage_dir.join("legacy-dep").mkdir().join("library.json").write( + '{"name": "AsyncTCP-esphome", "version": "1.1.1"}' + ) + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.get_package("AsyncMqttClient-esphome") + outdated = lm.outdated(pkg) + assert len(lm.get_installed()) == 2 + assert str(pkg.metadata.version) == "0.8.2" + assert outdated.latest > semantic_version.Version("0.8.2") + + # update + lm = LibraryPackageManager(str(storage_dir)) + new_pkg = lm.update(pkg, silent=True) + assert len(lm.get_installed()) == 3 + assert new_pkg.metadata.spec.owner == "ottowinter" diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index d9d205c7..d7d4b820 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -17,7 +17,27 @@ import os import jsondiff import semantic_version -from platformio.package.meta import PackageMetaData, PackageSpec, PackageType +from platformio.package.meta import ( + PackageMetaData, + PackageOutdatedResult, + PackageSpec, + PackageType, +) + + +def test_outdated_result(): + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0") + assert result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.5.4") + assert result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.2.3") + assert not result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", detached=True) + assert not result.is_outdated() + assert not result.is_outdated(allow_incompatible=True) def test_spec_owner(): @@ -45,9 +65,16 @@ def test_spec_name(): def test_spec_requirements(): assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec( + name="foo", requirements=semantic_version.Version("1.2.3") + ) == PackageSpec(name="foo", requirements="1.2.3") assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") + assert PackageSpec( + name="hello", requirements=semantic_version.SimpleSpec("~1.2.3") + ) == PackageSpec(name="hello", requirements="~1.2.3") spec = PackageSpec("id=20 @ !=1.2.3,<2.0") + assert not spec.external assert isinstance(spec.requirements, semantic_version.SimpleSpec) assert semantic_version.Version("1.3.0-beta.1") in spec.requirements assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") @@ -88,7 +115,8 @@ def test_spec_external_urls(): "Custom-Name=" "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" ) - assert spec.is_custom_name() + assert spec.external + assert spec.has_custom_name() assert spec.name == "Custom-Name" assert spec == PackageSpec( url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", @@ -163,6 +191,24 @@ def test_spec_as_dict(): ) +def test_spec_as_dependency(): + assert PackageSpec("owner/pkgname").as_dependency() == "owner/pkgname" + assert PackageSpec(owner="owner", name="pkgname").as_dependency() == "owner/pkgname" + assert PackageSpec("bob/foo @ ^1.2.3").as_dependency() == "bob/foo@^1.2.3" + assert ( + PackageSpec( + "https://github.com/o/r/a/develop.zip?param=value @ !=2" + ).as_dependency() + == "https://github.com/o/r/a/develop.zip?param=value @ !=2" + ) + assert ( + PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ).as_dependency() + == "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) + + def test_metadata_as_dict(): metadata = PackageMetaData(PackageType.LIBRARY, "foo", "1.2.3") # test setter diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 34d4ce68..07fbabf8 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -88,7 +88,9 @@ def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_clire validate_cliresult(result) assert "There are the new updates for libraries (ArduinoJson)" in result.output assert "Please wait while updating libraries" in result.output - assert re.search(r"Updating ArduinoJson\s+@ 6.12.0\s+\[[\d\.]+\]", result.output) + assert re.search( + r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[[\d\.]+\]", result.output + ) # check updated version result = clirunner.invoke(cli_pio, ["lib", "-g", "list", "--json-output"]) From d5451756fd2f5f1c5979695516c292a11760fffd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Aug 2020 20:09:10 +0300 Subject: [PATCH 145/223] Minor improvements --- platformio/builder/tools/piolib.py | 2 + platformio/builder/tools/pioplatform.py | 2 +- platformio/commands/lib/command.py | 6 +- platformio/package/manager/_download.py | 4 +- platformio/package/manager/_install.py | 40 ++++-------- platformio/package/manager/_registry.py | 13 ++-- platformio/package/manager/_uninstall.py | 12 ++-- platformio/package/manager/_update.py | 12 ++-- platformio/package/manager/base.py | 77 ++++++++++++++++-------- tests/commands/test_lib_complex.py | 1 - 10 files changed, 90 insertions(+), 79 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 24229b1c..3a7b3aad 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1022,6 +1022,8 @@ def ConfigureProjectLibBuilder(env): pkg = PackageSourceItem(lb.path) if pkg.metadata: title += " %s" % pkg.metadata.version + elif lb.version: + title += " %s" % lb.version click.echo("%s|-- %s" % (margin, title), nl=False) if int(ARGUMENTS.get("PIOVERBOSE", 0)): if pkg.metadata and pkg.metadata.spec.external: diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index e280372b..8c047365 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -139,7 +139,7 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements ) def _get_plaform_data(): - data = ["PLATFORM: %s %s" % (platform.title, platform.version)] + data = ["PLATFORM: %s (%s)" % (platform.title, platform.version)] if platform.src_version: data.append("#" + platform.src_version) if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url: diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 33249f3e..ec5fd8e6 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -283,12 +283,14 @@ def lib_update( # pylint: disable=too-many-arguments json_result[storage_dir] = result else: for library in _libraries: - spec = ( + to_spec = ( None if isinstance(library, PackageSourceItem) else PackageSpec(library) ) - lm.update(library, spec=spec, only_check=only_check, silent=silent) + lm.update( + library, to_spec=to_spec, only_check=only_check, silent=silent + ) if json_output: return click.echo( diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 83de9f37..34295287 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -17,8 +17,6 @@ import os import tempfile import time -import click - from platformio import app, compat from platformio.package.download import FileDownloader from platformio.package.lockfile import LockFile @@ -77,7 +75,7 @@ class PackageManagerDownloadMixin(object): except IOError: raise_error = True if raise_error: - click.secho( + self.print_message( "Error: Please read http://bit.ly/package-manager-ioerror", fg="red", err=True, diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index cb565f10..c63a504d 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -20,7 +20,7 @@ import tempfile import click from platformio import app, compat, fs, util -from platformio.package.exception import MissingPackageManifestError, PackageException +from platformio.package.exception import PackageException from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -71,7 +71,7 @@ class PackageManagerInstallMixin(object): if pkg: if not silent: - click.secho( + self.print_message( "{name} @ {version} is already installed".format( **pkg.metadata.as_dict() ), @@ -80,8 +80,9 @@ class PackageManagerInstallMixin(object): return pkg if not silent: - msg = "Installing %s" % click.style(spec.humanize(), fg="cyan") - self.print_message(msg) + self.print_message( + "Installing %s" % click.style(spec.humanize(), fg="cyan") + ) if spec.external: pkg = self.install_from_url(spec.url, spec, silent=silent) @@ -96,12 +97,10 @@ class PackageManagerInstallMixin(object): if not silent: self.print_message( - click.style( - "{name} @ {version} has been successfully installed!".format( - **pkg.metadata.as_dict() - ), - fg="green", - ) + "{name} @ {version} has been successfully installed!".format( + **pkg.metadata.as_dict() + ), + fg="green", ) self.memcache_reset() @@ -115,10 +114,10 @@ class PackageManagerInstallMixin(object): if not manifest.get("dependencies"): return if not silent: - self.print_message(click.style("Installing dependencies...", fg="yellow")) + self.print_message("Installing dependencies...") for dependency in manifest.get("dependencies"): if not self._install_dependency(dependency, silent) and not silent: - click.secho( + self.print_message( "Warning! Could not install dependency %s for package '%s'" % (dependency, pkg.metadata.name), fg="yellow", @@ -255,20 +254,3 @@ class PackageManagerInstallMixin(object): _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) - - def get_installed(self): - result = [] - for name in os.listdir(self.package_dir): - pkg_dir = os.path.join(self.package_dir, name) - if not os.path.isdir(pkg_dir): - continue - pkg = PackageSourceItem(pkg_dir) - if not pkg.metadata: - try: - spec = self.build_legacy_spec(pkg_dir) - pkg.metadata = self.build_metadata(pkg_dir, spec) - except MissingPackageManifestError: - pass - if pkg.metadata: - result.append(pkg) - return result diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 41ca58b3..0d1e45e1 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -108,8 +108,8 @@ class PackageManageRegistryMixin(object): silent=silent, ) except Exception as e: # pylint: disable=broad-except - click.secho("Warning! Package Mirror: %s" % e, fg="yellow") - click.secho("Looking for another mirror...", fg="yellow") + self.print_message("Warning! Package Mirror: %s" % e, fg="yellow") + self.print_message("Looking for another mirror...", fg="yellow") return None @@ -159,9 +159,8 @@ class PackageManageRegistryMixin(object): click.echo("") return packages[0]["id"] - @staticmethod - def print_multi_package_issue(packages, spec): - click.secho( + def print_multi_package_issue(self, packages, spec): + self.print_message( "Warning! More than one package has been found by ", fg="yellow", nl=False ) click.secho(spec.humanize(), fg="cyan", nl=False) @@ -174,9 +173,9 @@ class PackageManageRegistryMixin(object): version=item["version"]["name"], ) ) - click.secho( + self.print_message( "Please specify detailed REQUIREMENTS using package owner and version " - "(showed above) to avoid project compatibility issues.", + "(showed above) to avoid name conflicts", fg="yellow", ) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 813ada6d..603ad382 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -23,17 +23,17 @@ from platformio.package.meta import PackageSourceItem, PackageSpec class PackageManagerUninstallMixin(object): - def uninstall(self, pkg, silent=False, skip_dependencies=False): + def uninstall(self, spec, silent=False, skip_dependencies=False): try: self.lock() - return self._uninstall(pkg, silent, skip_dependencies) + return self._uninstall(spec, silent, skip_dependencies) finally: self.unlock() - def _uninstall(self, pkg, silent=False, skip_dependencies=False): - pkg = self.get_package(pkg) + def _uninstall(self, spec, silent=False, skip_dependencies=False): + pkg = self.get_package(spec) if not pkg or not pkg.metadata: - raise UnknownPackageError(pkg) + raise UnknownPackageError(spec) if not silent: self.print_message( @@ -78,7 +78,7 @@ class PackageManagerUninstallMixin(object): if not manifest.get("dependencies"): return if not silent: - self.print_message(click.style("Removing dependencies...", fg="yellow")) + self.print_message("Removing dependencies...", fg="yellow") for dependency in manifest.get("dependencies"): pkg = self.get_package( PackageSpec( diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 87d5e7f4..d120e030 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -78,18 +78,18 @@ class PackageManagerUpdateMixin(object): ).version ) - def update(self, pkg, spec=None, only_check=False, silent=False): - pkg = self.get_package(pkg) + def update(self, from_spec, to_spec=None, only_check=False, silent=False): + pkg = self.get_package(from_spec) if not pkg or not pkg.metadata: - raise UnknownPackageError(pkg) + raise UnknownPackageError(from_spec) if not silent: click.echo( "{} {:<45} {:<30}".format( "Checking" if only_check else "Updating", click.style(pkg.metadata.spec.humanize(), fg="cyan"), - "%s (%s)" % (pkg.metadata.version, spec.requirements) - if spec and spec.requirements + "%s (%s)" % (pkg.metadata.version, to_spec.requirements) + if to_spec and to_spec.requirements else str(pkg.metadata.version), ), nl=False, @@ -99,7 +99,7 @@ class PackageManagerUpdateMixin(object): click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) return pkg - outdated = self.outdated(pkg, spec) + outdated = self.outdated(pkg, to_spec) if not silent: self.print_outdated_state(outdated) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index ee2928c5..58c35d47 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -103,8 +103,11 @@ class BasePackageManager( # pylint: disable=too-many-public-methods def manifest_names(self): raise NotImplementedError - def print_message(self, message, nl=True): - click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) + def print_message(self, message, **kwargs): + click.echo( + "%s: " % str(self.__class__.__name__).replace("Package", " "), nl=False + ) + click.secho(message, **kwargs) def get_download_dir(self): if not self._download_dir: @@ -160,7 +163,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods return result except ManifestException as e: if not PlatformioCLI.in_silence(): - click.secho(str(e), fg="yellow") + self.print_message(str(e), fg="yellow") raise MissingPackageManifestError(", ".join(self.manifest_names)) @staticmethod @@ -186,37 +189,63 @@ class BasePackageManager( # pylint: disable=too-many-public-methods metadata.version = self.generate_rand_version() return metadata + def get_installed(self): + result = [] + for name in os.listdir(self.package_dir): + pkg_dir = os.path.join(self.package_dir, name) + if not os.path.isdir(pkg_dir): + continue + pkg = PackageSourceItem(pkg_dir) + if not pkg.metadata: + try: + spec = self.build_legacy_spec(pkg_dir) + pkg.metadata = self.build_metadata(pkg_dir, spec) + except MissingPackageManifestError: + pass + if pkg.metadata: + result.append(pkg) + return result + def get_package(self, spec): if isinstance(spec, PackageSourceItem): return spec - - if not isinstance(spec, PackageSpec) and os.path.isdir(spec): - for pkg in self.get_installed(): - if spec == pkg.path: - return pkg - return None - spec = self.ensure_spec(spec) best = None for pkg in self.get_installed(): - skip_conditions = [ - spec.owner - and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner), - spec.external and spec.url != pkg.metadata.spec.url, - spec.id and spec.id != pkg.metadata.spec.id, - not spec.id - and not spec.external - and not ci_strings_are_equal(spec.name, pkg.metadata.name), - ] - if any(skip_conditions): + if not self._test_pkg_with_spec(pkg, spec): continue - if self.pkg_type == PackageType.TOOL: - # TODO: check "system" for pkg - pass - assert isinstance(pkg.metadata.version, semantic_version.Version) if spec.requirements and pkg.metadata.version not in spec.requirements: continue if not best or (pkg.metadata.version > best.metadata.version): best = pkg return best + + def _test_pkg_with_spec(self, pkg, spec): + # "id" mismatch + if spec.id and spec.id != pkg.metadata.spec.id: + return False + + # "owner" mismatch + if spec.owner and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner): + return False + + # external "URL" mismatch + if spec.external: + # local folder mismatch + if spec.url == pkg.path or ( + spec.url.startswith("file://") and pkg.path == spec.url[7:] + ): + return True + if spec.url != pkg.metadata.spec.url: + return False + + # "name" mismatch + elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): + return False + + if self.pkg_type == PackageType.TOOL: + # TODO: check "system" for pkg + pass + + return True diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index 3f0f3725..a71330db 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -15,7 +15,6 @@ import json import re -from platformio import exception from platformio.commands import PlatformioCLI from platformio.commands.lib.command import cli as cmd_lib from platformio.package.exception import UnknownPackageError From 38ec51720039144cc298dd66e8b51ea149d6443d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Aug 2020 21:09:42 +0300 Subject: [PATCH 146/223] Update history --- HISTORY.rst | 26 ++++++++++++++++++++++---- docs | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b068fd59..147c4b5f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,16 +11,34 @@ PlatformIO Core 4 **A professional collaborative platform for embedded development** -* New `Account Management System `__ +* Integration with the new `Account Management System `__ - Manage own organizations - Manage organization teams - Manage resource access -* Registry Package Management +* Integration with the new **PlatformIO Trusted Registry** - - Publish a personal or organization package using `platformio package publish `__ command - - Remove a pushed package from the registry using `platformio package unpublish `__ command + - Enterprise-grade package storage with high availability (multi replicas) + - Secure, fast, and reliable global content delivery network (CDN) + - Universal support for all embedded packages: + + * Libraries + * Development platforms + * Toolchains + + - Built-in fine-grained access control (role based, teams, organizations) + - Command Line Interface: + + * `platformio package publish `__ – publish a personal or organization package + * `platformio package unpublish `__ – remove a pushed package from the registry + * Grant package access to the team members or maintainers + +* New **Package Management System** + + - Integrated PlatformIO Core with the new PlatformIO Trusted Registry + - Strict dependency declaration using owner name (resolves name conflicts) (`issue #1824 `_) + - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) * New `Custom Targets `__ diff --git a/docs b/docs index 13df46f9..e8ee370a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 13df46f9cf4e3f822f6f642c9c9b3085a0a93193 +Subproject commit e8ee370a338270b453d4d97bd286f537d5f06456 From fd7dba1d746d6a7fb10a04e7bc62e6fde4e5266e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 17:50:44 +0300 Subject: [PATCH 147/223] Package Manifest: increase package author.name field to the 100 chars --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index b886fb5a..8befab52 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -84,7 +84,7 @@ class StrictListField(fields.List): class AuthorSchema(StrictSchema): - name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) From 64ff6a0ff5e5d7d3d10bee8c3addd0a4395e3a46 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 18:30:04 +0300 Subject: [PATCH 148/223] Switch legacy core package manager to the new --- platformio/__init__.py | 10 + platformio/builder/tools/pioide.py | 2 +- platformio/builder/tools/piomisc.py | 2 +- platformio/commands/check/tools/clangtidy.py | 2 +- platformio/commands/check/tools/cppcheck.py | 2 +- platformio/commands/check/tools/pvsstudio.py | 2 +- platformio/commands/debug/command.py | 2 +- platformio/commands/home/command.py | 2 +- platformio/commands/remote/command.py | 2 +- platformio/commands/update.py | 2 +- platformio/maintenance.py | 24 +- platformio/managers/platform.py | 2 +- platformio/package/manager/_registry.py | 2 + .../{managers => package/manager}/core.py | 109 +++----- platformio/util.py | 2 +- tests/test_managers.py | 234 ------------------ 16 files changed, 83 insertions(+), 318 deletions(-) rename platformio/{managers => package/manager}/core.py (57%) delete mode 100644 tests/test_managers.py diff --git a/platformio/__init__.py b/platformio/__init__.py index 8da7e6e0..0a1bc48c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -40,3 +40,13 @@ __apiurl__ = "https://api.platformio.org" __accounts_api__ = "https://api.accounts.platformio.org" __registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" + +__core_packages__ = { + "contrib-piohome": "~3.2.3", + "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), + "tool-unity": "~1.20500.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~3.30102.0", + "tool-cppcheck": "~1.190.0", + "tool-clangtidy": "~1.100000.0", + "tool-pvs-studio": "~7.7.0", +} diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index acb36ae4..6a3d343d 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -20,7 +20,7 @@ from glob import glob from SCons.Defaults import processDefines # pylint: disable=import-error from platformio.compat import glob_escape -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command, where_is_program diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index aa5158fe..799b192f 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -25,7 +25,7 @@ import click from platformio import fs, util from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index f1610452..05be67b4 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -17,7 +17,7 @@ from os.path import join from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class ClangtidyCheckTool(CheckToolBase): diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 34129714..931b16ed 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -19,7 +19,7 @@ import click from platformio import proc from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class CppcheckCheckTool(CheckToolBase): diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index 871ec4bc..ce5d93ec 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -22,7 +22,7 @@ import click from platformio import proc, util from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-attributes diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 25286111..78a43eef 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -24,7 +24,7 @@ import click from platformio import app, exception, fs, proc, util from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError -from platformio.managers.core import inject_contrib_pysite +from platformio.package.manager.core import inject_contrib_pysite from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project, load_project_ide_data diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 32d28063..dd733bb6 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -22,7 +22,7 @@ import click from platformio import exception from platformio.compat import WINDOWS -from platformio.managers.core import get_core_package_dir, inject_contrib_pysite +from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite @click.command("home", short_help="PIO Home") diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index f9e24c29..66c10690 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -29,7 +29,7 @@ from platformio.commands.device.command import device_monitor as cmd_device_moni from platformio.commands.run.command import cli as cmd_run from platformio.commands.test.command import cli as cmd_test from platformio.compat import PY2 -from platformio.managers.core import inject_contrib_pysite +from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError diff --git a/platformio/commands/update.py b/platformio/commands/update.py index bf829165..b1e15a43 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -18,7 +18,7 @@ from platformio import app from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update -from platformio.managers.core import update_core_packages +from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager diff --git a/platformio/maintenance.py b/platformio/maintenance.py index b0e64f52..cf2e0698 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,9 +25,11 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.core import update_core_packages from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec from platformio.proc import is_container @@ -90,7 +92,8 @@ class Upgrader(object): ) self._upgraders = [ - (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms) + (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), + (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), ] def run(self, ctx): @@ -110,6 +113,22 @@ class Upgrader(object): ctx.invoke(cmd_platform_update) return True + @staticmethod + def _update_pkg_metadata(_): + pm = ToolPackageManager() + for pkg in pm.get_installed(): + if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id: + continue + result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name)) + if len(result) != 1: + continue + result = result[0] + pkg.metadata.spec = PackageSpec( + id=result["id"], owner=result["owner"]["username"], name=result["name"], + ) + pkg.dump_meta() + return True + def after_upgrade(ctx): terminal_width, _ = click.get_terminal_size() @@ -160,7 +179,6 @@ def after_upgrade(ctx): ) else: raise exception.UpgradeError("Auto upgrading...") - click.echo("") # PlatformIO banner click.echo("*" * terminal_width) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index ada4f4ac..8548bba6 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -30,8 +30,8 @@ from platformio.commands.debug.exception import ( DebugSupportError, ) from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module -from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +from platformio.package.manager.core import get_core_package_dir from platformio.project.config import ProjectConfig try: diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 0d1e45e1..7dc09964 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -119,6 +119,7 @@ class PackageManageRegistryMixin(object): return self._registry_client def search_registry_packages(self, spec, filters=None): + assert isinstance(spec, PackageSpec) filters = filters or {} if spec.id: filters["ids"] = str(spec.id) @@ -132,6 +133,7 @@ class PackageManageRegistryMixin(object): ] def fetch_registry_package(self, spec): + assert isinstance(spec, PackageSpec) result = None if spec.owner and spec.name: result = self.get_registry_client_instance().get_package( diff --git a/platformio/managers/core.py b/platformio/package/manager/core.py similarity index 57% rename from platformio/managers/core.py rename to platformio/package/manager/core.py index 27bee8c2..2b872ab6 100644 --- a/platformio/managers/core.py +++ b/platformio/package/manager/core.py @@ -17,89 +17,58 @@ import os import subprocess import sys -from platformio import exception, util +from platformio import __core_packages__, exception, util from platformio.compat import PY2 -from platformio.managers.package import PackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec from platformio.proc import get_pythonexe_path -from platformio.project.config import ProjectConfig - -CORE_PACKAGES = { - "contrib-piohome": "~3.2.3", - "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", - "tool-cppcheck": "~1.190.0", - "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.7.0", -} - -# pylint: disable=arguments-differ,signature-differs - - -class CorePackageManager(PackageManager): - def __init__(self): - config = ProjectConfig.get_instance() - packages_dir = config.get_optional_dir("packages") - super(CorePackageManager, self).__init__( - packages_dir, - [ - "https://dl.bintray.com/platformio/dl-packages/manifest.json", - "http%s://dl.platformio.org/packages/manifest.json" - % ("" if sys.version_info < (2, 7, 9) else "s"), - ], - ) - - def install( # pylint: disable=keyword-arg-before-vararg - self, name, requirements=None, *args, **kwargs - ): - PackageManager.install(self, name, requirements, *args, **kwargs) - self.cleanup_packages() - return self.get_package_dir(name, requirements) - - def update(self, *args, **kwargs): - result = PackageManager.update(self, *args, **kwargs) - self.cleanup_packages() - return result - - def cleanup_packages(self): - self.cache_reset() - best_pkg_versions = {} - for name, requirements in CORE_PACKAGES.items(): - pkg_dir = self.get_package_dir(name, requirements) - if not pkg_dir: - continue - best_pkg_versions[name] = self.load_manifest(pkg_dir)["version"] - for manifest in self.get_installed(): - if manifest["name"] not in best_pkg_versions: - continue - if manifest["version"] != best_pkg_versions[manifest["name"]]: - self.uninstall(manifest["__pkg_dir"], after_update=True) - self.cache_reset() - return True def get_core_package_dir(name): - if name not in CORE_PACKAGES: + if name not in __core_packages__: raise exception.PlatformioException("Please upgrade PIO Core") - requirements = CORE_PACKAGES[name] - pm = CorePackageManager() - pkg_dir = pm.get_package_dir(name, requirements) - if pkg_dir: - return pkg_dir - return pm.install(name, requirements) + pm = ToolPackageManager() + spec = PackageSpec( + owner="platformio", name=name, requirements=__core_packages__[name] + ) + pkg = pm.get_package(spec) + if pkg: + return pkg.path + pkg = pm.install(spec).path + _remove_unnecessary_packages() + return pkg def update_core_packages(only_check=False, silent=False): - pm = CorePackageManager() - for name, requirements in CORE_PACKAGES.items(): - pkg_dir = pm.get_package_dir(name) - if not pkg_dir: + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: continue - if not silent or pm.outdated(pkg_dir, requirements): - pm.update(name, requirements, only_check=only_check) + if not silent or pm.outdated(pkg, spec).is_outdated(): + pm.update(pkg, spec, only_check=only_check) + if not only_check: + _remove_unnecessary_packages() return True +def _remove_unnecessary_packages(): + pm = ToolPackageManager() + best_pkg_versions = {} + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: + continue + best_pkg_versions[pkg.metadata.name] = pkg.metadata.version + for pkg in pm.get_installed(): + if pkg.metadata.name not in best_pkg_versions: + continue + if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]: + pm.uninstall(pkg) + + def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-outside-toplevel from site import addsitedir diff --git a/platformio/util.py b/platformio/util.py index 36254586..5b38909d 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -196,7 +196,7 @@ def get_mdns_services(): import zeroconf except ImportError: from site import addsitedir - from platformio.managers.core import get_core_package_dir + from platformio.package.manager.core import get_core_package_dir contrib_pysite_dir = get_core_package_dir("contrib-pysite") addsitedir(contrib_pysite_dir) diff --git a/tests/test_managers.py b/tests/test_managers.py deleted file mode 100644 index 308523cd..00000000 --- a/tests/test_managers.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from os.path import join - -from platformio.managers.package import PackageManager -from platformio.project.helpers import get_project_core_dir - - -def test_pkg_input_parser(): - items = [ - ["PkgName", ("PkgName", None, None)], - [("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)], - ["PkgName@1.2.3", ("PkgName", "1.2.3", None)], - [("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)], - ["id=13", ("id=13", None, None)], - ["id=13@~1.2.3", ("id=13", "~1.2.3", None)], - [ - get_project_core_dir(), - (".platformio", None, "file://" + get_project_core_dir()), - ], - [ - "LocalName=" + get_project_core_dir(), - ("LocalName", None, "file://" + get_project_core_dir()), - ], - [ - "LocalName=%s@>2.3.0" % get_project_core_dir(), - ("LocalName", ">2.3.0", "file://" + get_project_core_dir()), - ], - [ - "https://github.com/user/package.git", - ("package", None, "git+https://github.com/user/package.git"), - ], - [ - "MyPackage=https://gitlab.com/user/package.git", - ("MyPackage", None, "git+https://gitlab.com/user/package.git"), - ], - [ - "MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2", - ("MyPackage", "3.2.1,!=2", "git+https://gitlab.com/user/package.git"), - ], - [ - "https://somedomain.com/path/LibraryName-1.2.3.zip", - ( - "LibraryName-1.2.3", - None, - "https://somedomain.com/path/LibraryName-1.2.3.zip", - ), - ], - [ - "https://github.com/user/package/archive/branch.zip", - ("branch", None, "https://github.com/user/package/archive/branch.zip"), - ], - [ - "https://github.com/user/package/archive/branch.zip@~1.2.3", - ("branch", "~1.2.3", "https://github.com/user/package/archive/branch.zip"), - ], - [ - "https://github.com/user/package/archive/branch.tar.gz", - ( - "branch.tar", - None, - "https://github.com/user/package/archive/branch.tar.gz", - ), - ], - [ - "https://github.com/user/package/archive/branch.tar.gz@!=5", - ( - "branch.tar", - "!=5", - "https://github.com/user/package/archive/branch.tar.gz", - ), - ], - [ - "https://developer.mbed.org/users/user/code/package/", - ("package", None, "hg+https://developer.mbed.org/users/user/code/package/"), - ], - [ - "https://os.mbed.com/users/user/code/package/", - ("package", None, "hg+https://os.mbed.com/users/user/code/package/"), - ], - [ - "https://github.com/user/package#v1.2.3", - ("package", None, "git+https://github.com/user/package#v1.2.3"), - ], - [ - "https://github.com/user/package.git#branch", - ("package", None, "git+https://github.com/user/package.git#branch"), - ], - [ - "PkgName=https://github.com/user/package.git#a13d344fg56", - ("PkgName", None, "git+https://github.com/user/package.git#a13d344fg56"), - ], - ["user/package", ("package", None, "git+https://github.com/user/package")], - [ - "PkgName=user/package", - ("PkgName", None, "git+https://github.com/user/package"), - ], - [ - "PkgName=user/package#master", - ("PkgName", None, "git+https://github.com/user/package#master"), - ], - [ - "git+https://github.com/user/package", - ("package", None, "git+https://github.com/user/package"), - ], - [ - "hg+https://example.com/user/package", - ("package", None, "hg+https://example.com/user/package"), - ], - [ - "git@github.com:user/package.git", - ("package", None, "git+git@github.com:user/package.git"), - ], - [ - "git@github.com:user/package.git#v1.2.0", - ("package", None, "git+git@github.com:user/package.git#v1.2.0"), - ], - [ - "LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0", - ("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0"), - ], - [ - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ( - "package", - None, - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ), - ], - [ - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ( - "package", - None, - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ), - ], - [ - "LocalName=git+ssh://user@gitlab.private-server.com:1234" - "/package#1.2.0@!=13", - ( - "LocalName", - "!=13", - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ), - ], - ] - for params, result in items: - if isinstance(params, tuple): - assert PackageManager.parse_pkg_uri(*params) == result - else: - assert PackageManager.parse_pkg_uri(params) == result - - -def test_install_packages(isolated_pio_core, tmpdir): - packages = [ - dict(id=1, name="name_1", version="shasum"), - dict(id=1, name="name_1", version="2.0.0"), - dict(id=1, name="name_1", version="2.1.0"), - dict(id=1, name="name_1", version="1.2"), - dict(id=1, name="name_1", version="1.0.0"), - dict(name="name_2", version="1.0.0"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - dict(name="name_2", version="3.0.0", __src_url="git+https://github2.com"), - dict(name="name_2", version="4.0.0", __src_url="git+https://github2.com"), - ] - - pm = PackageManager(join(get_project_core_dir(), "packages")) - for package in packages: - tmp_dir = tmpdir.mkdir("tmp-package") - tmp_dir.join("package.json").write(json.dumps(package)) - pm._install_from_url(package["name"], "file://%s" % str(tmp_dir)) - tmp_dir.remove(rec=1) - - assert len(pm.get_installed()) == len(packages) - 1 - - pkg_dirnames = [ - "name_1_ID1", - "name_1_ID1@1.0.0", - "name_1_ID1@1.2", - "name_1_ID1@2.0.0", - "name_1_ID1@shasum", - "name_2", - "name_2@src-177cbce1f0705580d17790fda1cc2ef5", - "name_2@src-f863b537ab00f4c7b5011fc44b120e1f", - ] - assert set( - [p.basename for p in isolated_pio_core.join("packages").listdir()] - ) == set(pkg_dirnames) - - -def test_get_package(): - tests = [ - [("unknown",), None], - [("1",), None], - [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")], - [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("name_1", "<2"), dict(id=1, name="name_1", version="1.2")], - [("name_1", ">2"), None], - [("name_1", "2-0-0"), None], - [("name_2",), dict(name="name_2", version="4.0.0")], - [ - ("url_has_higher_priority", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - ], - [ - ("name_2", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - ], - ] - - pm = PackageManager(join(get_project_core_dir(), "packages")) - for test in tests: - manifest = pm.get_package(*test[0]) - if test[1] is None: - assert manifest is None, test - continue - for key, value in test[1].items(): - assert manifest[key] == value, test From 26fdd0a62c2bcdec5e123b13819ae39f57f16169 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 18:30:33 +0300 Subject: [PATCH 149/223] Bump version to 4.4.0a8 --- platformio/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0a1bc48c..ee6d438d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a7") +import sys + +VERSION = (4, 4, "0a8") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From ecc369c2f8facbc1a783e9f873063b0aa86f3ade Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 20:19:27 +0300 Subject: [PATCH 150/223] Minor fixes --- platformio/commands/system/command.py | 8 +++++--- tests/package/test_manifest.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index af4071e0..76d2cb36 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -26,9 +26,9 @@ from platformio.commands.system.completion import ( install_completion_code, uninstall_completion_code, ) -from platformio.managers.package import PackageManager from platformio.managers.platform import PlatformManager from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig @@ -80,9 +80,11 @@ def system_info(json_output): "value": len(PlatformManager().get_installed()), } data["package_tool_nums"] = { - "title": "Package Tools", + "title": "Tools & Toolchains", "value": len( - PackageManager(project_config.get_optional_dir("packages")).get_installed() + ToolPackageManager( + project_config.get_optional_dir("packages") + ).get_installed() ), } diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 899f1cdf..35fdf367 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -485,7 +485,7 @@ depends=First Library (=2.0.0), Second Library (>=1.2.0), Third contents = """ name=Mozzi version=1.0.3 -author=Tim Barrass and contributors as documented in source, and at https://github.com/sensorium/Mozzi/graphs/contributors +author=Lorem Ipsum is simply dummy text of the printing and typesetting industry Lorem Ipsum has been the industry's standard dummy text ever since the 1500s when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries but also the leap into electronic typesetting remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. maintainer=Tim Barrass sentence=Sound synthesis library for Arduino paragraph=With Mozzi, you can construct sounds using familiar synthesis units like oscillators, delays, filters and envelopes. @@ -504,6 +504,7 @@ includes=MozziGuts.h ), ).as_dict() + errors = None try: ManifestSchema().load_manifest(raw_data) except ManifestValidationError as e: From ff8ec43a288d7314ef542e08ba662c06884af7c6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 21:46:46 +0300 Subject: [PATCH 151/223] Ensure tool-type package is compatible with a host system --- platformio/package/manager/base.py | 22 +++++++++++------- tests/package/test_manager.py | 36 ++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 58c35d47..c0e1e23f 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -202,8 +202,17 @@ class BasePackageManager( # pylint: disable=too-many-public-methods pkg.metadata = self.build_metadata(pkg_dir, spec) except MissingPackageManifestError: pass - if pkg.metadata: - result.append(pkg) + if not pkg.metadata: + continue + if self.pkg_type == PackageType.TOOL: + try: + if not self.is_system_compatible( + self.load_manifest(pkg).get("system") + ): + continue + except MissingPackageManifestError: + pass + result.append(pkg) return result def get_package(self, spec): @@ -212,7 +221,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods spec = self.ensure_spec(spec) best = None for pkg in self.get_installed(): - if not self._test_pkg_with_spec(pkg, spec): + if not self.test_pkg_spec(pkg, spec): continue assert isinstance(pkg.metadata.version, semantic_version.Version) if spec.requirements and pkg.metadata.version not in spec.requirements: @@ -221,7 +230,8 @@ class BasePackageManager( # pylint: disable=too-many-public-methods best = pkg return best - def _test_pkg_with_spec(self, pkg, spec): + @staticmethod + def test_pkg_spec(pkg, spec): # "id" mismatch if spec.id and spec.id != pkg.metadata.spec.id: return False @@ -244,8 +254,4 @@ class BasePackageManager( # pylint: disable=too-many-public-methods elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): return False - if self.pkg_type == PackageType.TOOL: - # TODO: check "system" for pkg - pass - return True diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 131346af..c91924f4 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -239,7 +239,7 @@ def test_install_force(isolated_pio_core, tmpdir_factory): def test_get_installed(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") - lm = LibraryPackageManager(str(storage_dir)) + pm = ToolPackageManager(str(storage_dir)) # VCS package ( @@ -259,7 +259,7 @@ def test_get_installed(isolated_pio_core, tmpdir_factory): "requirements": null, "url": "git+https://github.com/username/repo.git" }, - "type": "library", + "type": "tool", "version": "0.0.0+sha.1ea4d5e" } """ @@ -270,13 +270,13 @@ def test_get_installed(isolated_pio_core, tmpdir_factory): ( storage_dir.join("foo@3.4.5") .mkdir() - .join("library.json") + .join("package.json") .write('{"name": "foo", "version": "3.4.5"}') ) # package with metadata file foo_dir = storage_dir.join("foo").mkdir() - foo_dir.join("library.json").write('{"name": "foo", "version": "3.6.0"}') + foo_dir.join("package.json").write('{"name": "foo", "version": "3.6.0"}') foo_dir.join(".piopm").write( """ { @@ -286,21 +286,33 @@ def test_get_installed(isolated_pio_core, tmpdir_factory): "owner": null, "requirements": "^3" }, - "type": "library", + "type": "tool", "version": "3.6.0" } """ ) - # invalid package - storage_dir.join("invalid-package").mkdir().join("package.json").write( - '{"name": "tool-scons", "version": "4.0.0"}' + # test "system" + storage_dir.join("pkg-incompatible-system").mkdir().join("package.json").write( + '{"name": "check-system", "version": "4.0.0", "system": ["unknown"]}' + ) + storage_dir.join("pkg-compatible-system").mkdir().join("package.json").write( + '{"name": "check-system", "version": "3.0.0", "system": "%s"}' + % util.get_systype() ) - installed = lm.get_installed() - assert len(installed) == 3 - assert set(["pkg-via-vcs", "foo"]) == set(p.metadata.name for p in installed) - assert str(lm.get_package("foo").metadata.version) == "3.6.0" + # invalid package + storage_dir.join("invalid-package").mkdir().join("library.json").write( + '{"name": "SomeLib", "version": "4.0.0"}' + ) + + installed = pm.get_installed() + assert len(installed) == 4 + assert set(["pkg-via-vcs", "foo", "check-system"]) == set( + p.metadata.name for p in installed + ) + assert str(pm.get_package("foo").metadata.version) == "3.6.0" + assert str(pm.get_package("check-system").metadata.version) == "3.0.0" def test_uninstall(isolated_pio_core, tmpdir_factory): From 5f3ad70190cc068f23b6785fe2238ff4a9c7a418 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 14 Aug 2020 16:38:46 +0300 Subject: [PATCH 152/223] Rename meta.PackageSourceItem or PackageItem --- platformio/builder/tools/piolib.py | 4 ++-- platformio/commands/lib/command.py | 8 +++----- platformio/package/manager/_install.py | 18 +++++++++--------- platformio/package/manager/_legacy.py | 6 ++++-- platformio/package/manager/_uninstall.py | 4 ++-- platformio/package/manager/_update.py | 8 ++------ platformio/package/manager/base.py | 15 ++++++++------- platformio/package/meta.py | 6 +++--- platformio/package/pack.py | 4 ++-- 9 files changed, 35 insertions(+), 38 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3a7b3aad..8cc1ad58 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -40,7 +40,7 @@ from platformio.package.manifest.parser import ( ManifestParserError, ManifestParserFactory, ) -from platformio.package.meta import PackageSourceItem +from platformio.package.meta import PackageItem from platformio.project.options import ProjectOptions @@ -1019,7 +1019,7 @@ def ConfigureProjectLibBuilder(env): margin = "| " * (level) for lb in root.depbuilders: title = "<%s>" % lb.name - pkg = PackageSourceItem(lb.path) + pkg = PackageItem(lb.path) if pkg.metadata: title += " %s" % pkg.metadata.version elif lb.version: diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index ec5fd8e6..03463aab 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -30,7 +30,7 @@ from platformio.commands.lib.helpers import ( from platformio.compat import dump_json_to_unicode from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager -from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -262,7 +262,7 @@ def lib_update( # pylint: disable=too-many-arguments for library in _libraries: spec = None pkg = None - if isinstance(library, PackageSourceItem): + if isinstance(library, PackageItem): pkg = library else: spec = PackageSpec(library) @@ -284,9 +284,7 @@ def lib_update( # pylint: disable=too-many-arguments else: for library in _libraries: to_spec = ( - None - if isinstance(library, PackageSourceItem) - else PackageSpec(library) + None if isinstance(library, PackageItem) else PackageSpec(library) ) lm.update( library, to_spec=to_spec, only_check=only_check, silent=silent diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index c63a504d..04a41f26 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,7 @@ import click from platformio import app, compat, fs, util from platformio.package.exception import PackageException -from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.meta import PackageItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -109,7 +109,7 @@ class PackageManagerInstallMixin(object): return pkg def _install_dependencies(self, pkg, silent=False): - assert isinstance(pkg, PackageSourceItem) + assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): return @@ -155,7 +155,7 @@ class PackageManagerInstallMixin(object): assert vcs.export() root_dir = self.find_pkg_root(tmp_dir, spec) - pkg_item = PackageSourceItem( + pkg_item = PackageItem( root_dir, self.build_metadata( root_dir, spec, vcs.get_current_revision() if vcs else None @@ -168,7 +168,7 @@ class PackageManagerInstallMixin(object): fs.rmtree(tmp_dir) def _install_tmp_pkg(self, tmp_pkg): - assert isinstance(tmp_pkg, PackageSourceItem) + assert isinstance(tmp_pkg, PackageItem) # validate package version and declared requirements if ( tmp_pkg.metadata.spec.requirements @@ -182,7 +182,7 @@ class PackageManagerInstallMixin(object): tmp_pkg.metadata, ) ) - dst_pkg = PackageSourceItem( + dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.get_safe_dirname()) ) @@ -190,7 +190,7 @@ class PackageManagerInstallMixin(object): action = "overwrite" if tmp_pkg.metadata.spec.has_custom_name(): action = "overwrite" - dst_pkg = PackageSourceItem( + dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) ) elif dst_pkg.metadata and dst_pkg.metadata.spec.external: @@ -231,7 +231,7 @@ class PackageManagerInstallMixin(object): # move new source to the destination location _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) - return PackageSourceItem(dst_pkg.path) + return PackageItem(dst_pkg.path) if action == "detach-new": target_dirname = "%s@%s" % ( @@ -248,9 +248,9 @@ class PackageManagerInstallMixin(object): pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) shutil.move(tmp_pkg.path, pkg_dir) - return PackageSourceItem(pkg_dir) + return PackageItem(pkg_dir) # otherwise, overwrite existing _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) - return PackageSourceItem(dst_pkg.path) + return PackageItem(dst_pkg.path) diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py index 22478eff..95f628d0 100644 --- a/platformio/package/manager/_legacy.py +++ b/platformio/package/manager/_legacy.py @@ -15,7 +15,7 @@ import os from platformio import fs -from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.meta import PackageItem, PackageSpec class PackageManagerLegacyMixin(object): @@ -42,7 +42,9 @@ class PackageManagerLegacyMixin(object): return PackageSpec(name=manifest.get("name")) def legacy_load_manifest(self, pkg): - assert isinstance(pkg, PackageSourceItem) + if not isinstance(pkg, PackageItem): + assert os.path.isdir(pkg) + pkg = PackageItem(pkg) manifest = self.load_manifest(pkg) manifest["__pkg_dir"] = pkg.path for key in ("name", "version"): diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 603ad382..e2656401 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -19,7 +19,7 @@ import click from platformio import fs from platformio.package.exception import UnknownPackageError -from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.meta import PackageItem, PackageSpec class PackageManagerUninstallMixin(object): @@ -73,7 +73,7 @@ class PackageManagerUninstallMixin(object): return pkg def _uninstall_dependencies(self, pkg, silent=False): - assert isinstance(pkg, PackageSourceItem) + assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): return diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index d120e030..b0b976de 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -18,17 +18,13 @@ import click from platformio import util from platformio.package.exception import UnknownPackageError -from platformio.package.meta import ( - PackageOutdatedResult, - PackageSourceItem, - PackageSpec, -) +from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec from platformio.package.vcsclient import VCSBaseException, VCSClientFactory class PackageManagerUpdateMixin(object): def outdated(self, pkg, spec=None): - assert isinstance(pkg, PackageSourceItem) + assert isinstance(pkg, PackageItem) assert not spec or isinstance(spec, PackageSpec) assert os.path.isdir(pkg.path) and pkg.metadata diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index c0e1e23f..dc024edc 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -31,8 +31,8 @@ from platformio.package.manager._uninstall import PackageManagerUninstallMixin from platformio.package.manager._update import PackageManagerUpdateMixin from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import ( + PackageItem, PackageMetaData, - PackageSourceItem, PackageSpec, PackageType, ) @@ -144,7 +144,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods return self.get_manifest_path(pkg_dir) def load_manifest(self, src): - path = src.path if isinstance(src, PackageSourceItem) else src + path = src.path if isinstance(src, PackageItem) else src cache_key = "load_manifest-%s" % path result = self.memcache_get(cache_key) if result: @@ -191,11 +191,11 @@ class BasePackageManager( # pylint: disable=too-many-public-methods def get_installed(self): result = [] - for name in os.listdir(self.package_dir): + for name in sorted(os.listdir(self.package_dir)): pkg_dir = os.path.join(self.package_dir, name) if not os.path.isdir(pkg_dir): continue - pkg = PackageSourceItem(pkg_dir) + pkg = PackageItem(pkg_dir) if not pkg.metadata: try: spec = self.build_legacy_spec(pkg_dir) @@ -216,7 +216,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods return result def get_package(self, spec): - if isinstance(spec, PackageSourceItem): + if isinstance(spec, PackageItem): return spec spec = self.ensure_spec(spec) best = None @@ -243,8 +243,9 @@ class BasePackageManager( # pylint: disable=too-many-public-methods # external "URL" mismatch if spec.external: # local folder mismatch - if spec.url == pkg.path or ( - spec.url.startswith("file://") and pkg.path == spec.url[7:] + if os.path.realpath(spec.url) == os.path.realpath(pkg.path) or ( + spec.url.startswith("file://") + and os.path.realpath(pkg.path) == os.path.realpath(spec.url[7:]) ): return True if spec.url != pkg.metadata.spec.url: diff --git a/platformio/package/meta.py b/platformio/package/meta.py index af1e0baa..f57f0495 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -315,7 +315,7 @@ class PackageMetaData(object): def __init__( # pylint: disable=redefined-builtin self, type, name, version, spec=None ): - assert type in PackageType.items().values() + # assert type in PackageType.items().values() if spec: assert isinstance(spec, PackageSpec) self.type = type @@ -395,7 +395,7 @@ class PackageMetaData(object): return PackageMetaData(**data) -class PackageSourceItem(object): +class PackageItem(object): METAFILE_NAME = ".piopm" @@ -406,7 +406,7 @@ class PackageSourceItem(object): self.metadata = self.load_meta() def __repr__(self): - return "PackageSourceItem Date: Fri, 14 Aug 2020 16:39:15 +0300 Subject: [PATCH 153/223] Refactor dev-platform API --- Makefile | 2 +- platformio/builder/main.py | 2 +- platformio/builder/tools/pioplatform.py | 32 +- platformio/commands/debug/helpers.py | 11 +- platformio/commands/device/command.py | 4 +- platformio/commands/lib/helpers.py | 5 +- platformio/commands/platform.py | 17 +- platformio/commands/project.py | 5 +- platformio/commands/run/processor.py | 10 +- platformio/commands/test/embedded.py | 4 +- platformio/exception.py | 43 -- platformio/maintenance.py | 7 +- platformio/managers/platform.py | 744 +----------------------- platformio/platform/__init__.py | 13 + platformio/platform/_packages.py | 126 ++++ platformio/platform/_run.py | 193 ++++++ platformio/platform/base.py | 274 +++++++++ platformio/platform/board.py | 158 +++++ platformio/platform/exception.py | 49 ++ platformio/platform/factory.py | 60 ++ 20 files changed, 941 insertions(+), 818 deletions(-) create mode 100644 platformio/platform/__init__.py create mode 100644 platformio/platform/_packages.py create mode 100644 platformio/platform/_run.py create mode 100644 platformio/platform/base.py create mode 100644 platformio/platform/board.py create mode 100644 platformio/platform/exception.py create mode 100644 platformio/platform/factory.py diff --git a/Makefile b/Makefile index 36b5d396..6b22d261 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ clean: clean-docs profile: # Usage $ > make PIOARGS="boards" profile - python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS} + python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS} snakeviz .tox/.tmp/cprofile.prof publish: diff --git a/platformio/builder/main.py b/platformio/builder/main.py index a0a8ab12..e73f6869 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -30,7 +30,7 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import compat, fs from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformBase +from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 8c047365..fe7d6e28 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -14,15 +14,16 @@ from __future__ import absolute_import +import os import sys -from os.path import isdir, isfile, join from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error -from platformio import exception, fs, util +from platformio import fs, util from platformio.compat import WINDOWS -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions # pylint: disable=too-many-branches, too-many-locals @@ -34,7 +35,7 @@ def PioPlatform(env): if "framework" in variables: # support PIO Core 3.0 dev/platforms variables["pioframework"] = variables["framework"] - p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"]) + p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"])) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p @@ -46,7 +47,7 @@ def BoardConfig(env, board=None): board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" return p.board_config(board) - except (AssertionError, exception.UnknownBoard) as e: + except (AssertionError, UnknownBoard) as e: sys.stderr.write("Error: %s\n" % str(e)) env.Exit(1) @@ -55,8 +56,8 @@ def GetFrameworkScript(env, framework): p = env.PioPlatform() assert p.frameworks and framework in p.frameworks script_path = env.subst(p.frameworks[framework]["script"]) - if not isfile(script_path): - script_path = join(p.get_dir(), script_path) + if not os.path.isfile(script_path): + script_path = os.path.join(p.get_dir(), script_path) return script_path @@ -75,17 +76,24 @@ def LoadPioPlatform(env): continue pkg_dir = p.get_package_dir(name) env.PrependENVPath( - "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir + "PATH", + os.path.join(pkg_dir, "bin") + if os.path.isdir(os.path.join(pkg_dir, "bin")) + else pkg_dir, ) - if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain": + if ( + not WINDOWS + and os.path.isdir(os.path.join(pkg_dir, "lib")) + and type_ != "toolchain" + ): env.PrependENVPath( "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH", - join(pkg_dir, "lib"), + os.path.join(pkg_dir, "lib"), ) # Platform specific LD Scripts - if isdir(join(p.get_dir(), "ldscripts")): - env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")]) + if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")): + env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: return diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 4604a861..657e8c48 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -20,13 +20,14 @@ from hashlib import sha1 from io import BytesIO from os.path import isfile -from platformio import exception, fs, util +from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.options import ProjectOptions @@ -94,14 +95,14 @@ def validate_debug_options(cmd_ctx, env_options): return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] try: - platform = PlatformFactory.newPlatform(env_options["platform"]) - except exception.UnknownPlatform: + platform = PlatformFactory.new(env_options["platform"]) + except UnknownPlatform: cmd_ctx.invoke( cmd_platform_install, platforms=[env_options["platform"]], skip_default_package=True, ) - platform = PlatformFactory.newPlatform(env_options["platform"]) + platform = PlatformFactory.new(env_options["platform"]) board_config = platform.board_config(env_options["board"]) tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index e93b1214..463116f9 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -22,7 +22,7 @@ from serial.tools import miniterm from platformio import exception, fs, util from platformio.commands.device import helpers as device_helpers from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformFactory +from platformio.platform.factory import PlatformFactory from platformio.project.exception import NotPlatformIOProjectError @@ -192,7 +192,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches platform = None if "platform" in project_options: with fs.cd(kwargs["project_dir"]): - platform = PlatformFactory.newPlatform(project_options["platform"]) + platform = PlatformFactory.new(project_options["platform"]) device_helpers.register_platform_filters( platform, kwargs["project_dir"], kwargs["environment"] ) diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index a5cc07e3..23892ac8 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -15,8 +15,9 @@ import os from platformio.compat import ci_strings_are_equal -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import InvalidProjectConfError @@ -29,7 +30,7 @@ def get_builtin_libs(storage_names=None): storage_names = storage_names or [] pm = PlatformManager() for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for storage in p.get_lib_storages(): if storage_names and storage["name"] not in storage_names: continue diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index deaeb431..14d5a1f2 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -16,10 +16,12 @@ from os.path import dirname, isdir import click -from platformio import app, exception, util +from platformio import app, util from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory @click.group(short_help="Platform Manager") @@ -64,12 +66,12 @@ def _get_registry_platforms(): def _get_platform_data(*args, **kwargs): try: return _get_installed_platform_data(*args, **kwargs) - except exception.UnknownPlatform: + except UnknownPlatform: return _get_registry_platform_data(*args, **kwargs) def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) data = dict( name=p.name, title=p.title, @@ -232,7 +234,7 @@ def platform_list(json_output): def platform_show(platform, json_output): # pylint: disable=too-many-branches data = _get_platform_data(platform) if not data: - raise exception.UnknownPlatform(platform) + raise UnknownPlatform(platform) if json_output: return click.echo(dump_json_to_unicode(data)) @@ -384,10 +386,7 @@ def platform_update( # pylint: disable=too-many-locals if not pkg_dir: continue latest = pm.outdated(pkg_dir, requirements) - if ( - not latest - and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages() - ): + if not latest and not PlatformFactory.new(pkg_dir).are_outdated_packages(): continue data = _get_installed_platform_data( pkg_dir, with_boards=False, expose_packages=False diff --git a/platformio/commands/project.py b/platformio/commands/project.py index c261a9d9..6194a915 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -20,10 +20,11 @@ import os import click from tabulate import tabulate -from platformio import exception, fs +from platformio import fs from platformio.commands.platform import platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager +from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -112,7 +113,7 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 for id_ in value: try: pm.board_config(id_) - except exception.UnknownBoard: + except UnknownBoard: raise click.BadParameter( "`%s`. Please search for board ID using `platformio boards` " "command" % id_ diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 23ccc333..d07c581c 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio import exception from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError # pylint: disable=too-many-instance-attributes @@ -67,14 +67,14 @@ class EnvironmentProcessor(object): build_targets.remove("monitor") try: - p = PlatformFactory.newPlatform(self.options["platform"]) - except exception.UnknownPlatform: + p = PlatformFactory.new(self.options["platform"]) + except UnknownPlatform: self.cmd_ctx.invoke( cmd_platform_install, platforms=[self.options["platform"]], skip_default_package=True, ) - p = PlatformFactory.newPlatform(self.options["platform"]) + p = PlatformFactory.new(self.options["platform"]) result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs) return result["returncode"] == 0 diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 6f47eafc..ca658496 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -19,7 +19,7 @@ import serial from platformio import exception, util from platformio.commands.test.processor import TestProcessorBase -from platformio.managers.platform import PlatformFactory +from platformio.platform.factory import PlatformFactory class EmbeddedTestProcessor(TestProcessorBase): @@ -108,7 +108,7 @@ class EmbeddedTestProcessor(TestProcessorBase): return self.env_options.get("test_port") assert set(["platform", "board"]) & set(self.env_options.keys()) - p = PlatformFactory.newPlatform(self.env_options["platform"]) + p = PlatformFactory.new(self.env_options["platform"]) board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", []) port = None elapsed = 0 diff --git a/platformio/exception.py b/platformio/exception.py index 9ab0e4d8..91fd67cc 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -47,44 +47,6 @@ class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" -# -# Development Platform -# - - -class UnknownPlatform(PlatformioException): - - MESSAGE = "Unknown development platform '{0}'" - - -class IncompatiblePlatform(PlatformioException): - - MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" - - -class PlatformNotInstalledYet(PlatformioException): - - MESSAGE = ( - "The platform '{0}' has not been installed yet. " - "Use `platformio platform install {0}` command" - ) - - -class UnknownBoard(PlatformioException): - - MESSAGE = "Unknown board ID '{0}'" - - -class InvalidBoardManifest(PlatformioException): - - MESSAGE = "Invalid board JSON manifest '{0}'" - - -class UnknownFramework(PlatformioException): - - MESSAGE = "Unknown framework '{0}'" - - # Package Manager @@ -195,11 +157,6 @@ class InternetIsOffline(UserSideException): ) -class BuildScriptNotFound(PlatformioException): - - MESSAGE = "Invalid path '{0}' to build script" - - class InvalidSettingName(UserSideException): MESSAGE = "Invalid setting with the name '{0}'" diff --git a/platformio/maintenance.py b/platformio/maintenance.py index cf2e0698..4b47d50a 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,11 +25,12 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory from platformio.proc import is_container @@ -278,9 +279,7 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches conds = [ pm.outdated(manifest["__pkg_dir"]), what == "platforms" - and PlatformFactory.newPlatform( - manifest["__pkg_dir"] - ).are_outdated_packages(), + and PlatformFactory.new(manifest["__pkg_dir"]).are_outdated_packages(), ] if any(conds): outdated_items.append(manifest["name"]) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8548bba6..c0e0f98e 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -14,31 +14,16 @@ # pylint: disable=too-many-public-methods, too-many-instance-attributes -import base64 -import os -import re -import subprocess -import sys -from os.path import basename, dirname, isdir, isfile, join -import click -import semantic_version +from os.path import isdir, isfile, join -from platformio import __version__, app, exception, fs, proc, telemetry, util -from platformio.commands.debug.exception import ( - DebugInvalidOptionsError, - DebugSupportError, -) -from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module +from platformio import app, exception, util from platformio.managers.package import BasePkgManager, PackageManager -from platformio.package.manager.core import get_core_package_dir +from platformio.platform.base import PlatformBase # pylint: disable=unused-import +from platformio.platform.exception import UnknownBoard, UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - class PlatformManager(BasePkgManager): def __init__(self, package_dir=None, repositories=None): @@ -83,7 +68,7 @@ class PlatformManager(BasePkgManager): platform_dir = BasePkgManager.install( self, name, requirements, silent=silent, force=force ) - p = PlatformFactory.newPlatform(platform_dir) + p = PlatformFactory.new(platform_dir) if with_all_packages: with_packages = list(p.packages.keys()) @@ -114,9 +99,9 @@ class PlatformManager(BasePkgManager): pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPlatform(package) + raise UnknownPlatform(package) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) BasePkgManager.uninstall(self, pkg_dir, requirements) p.uninstall_python_packages() p.on_uninstalled() @@ -138,15 +123,15 @@ class PlatformManager(BasePkgManager): pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPlatform(package) + raise UnknownPlatform(package) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) pkgs_before = list(p.get_installed_packages()) missed_pkgs = set() if not only_packages: BasePkgManager.update(self, pkg_dir, requirements, only_check) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) missed_pkgs = set(pkgs_before) & set(p.packages) missed_pkgs -= set(p.get_installed_packages()) @@ -164,7 +149,7 @@ class PlatformManager(BasePkgManager): self.cache_reset() deppkgs = {} for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for pkgname, pkgmanifest in p.get_installed_packages().items(): if pkgname not in deppkgs: deppkgs[pkgname] = set() @@ -190,7 +175,7 @@ class PlatformManager(BasePkgManager): def get_installed_boards(self): boards = [] for manifest in self.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for config in p.get_boards().values(): board = config.get_brief_data() if board not in boards: @@ -224,705 +209,4 @@ class PlatformManager(BasePkgManager): not platform or manifest["platform"] == platform ): return manifest - raise exception.UnknownBoard(id_) - - -class PlatformFactory(object): - @staticmethod - def get_clsname(name): - name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) - return "%s%sPlatform" % (name.upper()[0], name.lower()[1:]) - - @staticmethod - def load_module(name, path): - try: - return load_python_module("platformio.managers.platform.%s" % name, path) - except ImportError: - raise exception.UnknownPlatform(name) - - @classmethod - def newPlatform(cls, name, requirements=None): - pm = PlatformManager() - platform_dir = None - if isdir(name): - platform_dir = name - name = pm.load_manifest(platform_dir)["name"] - elif name.endswith("platform.json") and isfile(name): - platform_dir = dirname(name) - name = fs.load_json(name)["name"] - else: - name, requirements, url = pm.parse_pkg_uri(name, requirements) - platform_dir = pm.get_package_dir(name, requirements, url) - if platform_dir: - name = pm.load_manifest(platform_dir)["name"] - - if not platform_dir: - raise exception.UnknownPlatform( - name if not requirements else "%s@%s" % (name, requirements) - ) - - platform_cls = None - if isfile(join(platform_dir, "platform.py")): - platform_cls = getattr( - cls.load_module(name, join(platform_dir, "platform.py")), - cls.get_clsname(name), - ) - else: - platform_cls = type(str(cls.get_clsname(name)), (PlatformBase,), {}) - - _instance = platform_cls(join(platform_dir, "platform.json")) - assert isinstance(_instance, PlatformBase) - return _instance - - -class PlatformPackagesMixin(object): - def install_packages( # pylint: disable=too-many-arguments - self, - with_packages=None, - without_packages=None, - skip_default_package=False, - silent=False, - force=False, - ): - with_packages = set(self.find_pkg_names(with_packages or [])) - without_packages = set(self.find_pkg_names(without_packages or [])) - - upkgs = with_packages | without_packages - ppkgs = set(self.packages) - if not upkgs.issubset(ppkgs): - raise exception.UnknownPackage(", ".join(upkgs - ppkgs)) - - for name, opts in self.packages.items(): - version = opts.get("version", "") - if name in without_packages: - continue - if name in with_packages or not ( - skip_default_package or opts.get("optional", False) - ): - if ":" in version: - self.pm.install( - "%s=%s" % (name, version), silent=silent, force=force - ) - else: - self.pm.install(name, version, silent=silent, force=force) - - return True - - def find_pkg_names(self, candidates): - result = [] - for candidate in candidates: - found = False - - # lookup by package types - for _name, _opts in self.packages.items(): - if _opts.get("type") == candidate: - result.append(_name) - found = True - - if ( - self.frameworks - and candidate.startswith("framework-") - and candidate[10:] in self.frameworks - ): - result.append(self.frameworks[candidate[10:]]["package"]) - found = True - - if not found: - result.append(candidate) - - return result - - def update_packages(self, only_check=False): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - self.pm.update(manifest["__pkg_dir"], requirements, only_check) - - def get_installed_packages(self): - items = {} - for name in self.packages: - pkg_dir = self.get_package_dir(name) - if pkg_dir: - items[name] = self.pm.load_manifest(pkg_dir) - return items - - def are_outdated_packages(self): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - if self.pm.outdated(manifest["__pkg_dir"], requirements): - return True - return False - - def get_package_dir(self, name): - version = self.packages[name].get("version", "") - if ":" in version: - return self.pm.get_package_dir( - *self.pm.parse_pkg_uri("%s=%s" % (name, version)) - ) - return self.pm.get_package_dir(name, version) - - def get_package_version(self, name): - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - return None - return self.pm.load_manifest(pkg_dir).get("version") - - def dump_used_packages(self): - result = [] - for name, options in self.packages.items(): - if options.get("optional"): - continue - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - continue - manifest = self.pm.load_manifest(pkg_dir) - item = {"name": manifest["name"], "version": manifest["version"]} - if manifest.get("__src_url"): - item["src_url"] = manifest.get("__src_url") - result.append(item) - return result - - -class PlatformRunMixin(object): - - LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) - - @staticmethod - def encode_scons_arg(value): - data = base64.urlsafe_b64encode(hashlib_encode_data(value)) - return data.decode() if is_bytes(data) else data - - @staticmethod - def decode_scons_arg(data): - value = base64.urlsafe_b64decode(data) - return value.decode() if is_bytes(value) else value - - def run( # pylint: disable=too-many-arguments - self, variables, targets, silent, verbose, jobs - ): - assert isinstance(variables, dict) - assert isinstance(targets, list) - - options = self.config.items(env=variables["pioenv"], as_dict=True) - if "framework" in options: - # support PIO Core 3.0 dev/platforms - options["pioframework"] = options["framework"] - self.configure_default_packages(options, targets) - self.install_packages(silent=True) - - self._report_non_sensitive_data(options, targets) - - self.silent = silent - self.verbose = verbose or app.get_setting("force_verbose") - - if "clean" in targets: - targets = ["-c", "."] - - variables["platform_manifest"] = self.manifest_path - - if "build_script" not in variables: - variables["build_script"] = self.get_build_script() - if not isfile(variables["build_script"]): - raise exception.BuildScriptNotFound(variables["build_script"]) - - result = self._run_scons(variables, targets, jobs) - assert "returncode" in result - - return result - - def _report_non_sensitive_data(self, options, targets): - topts = options.copy() - topts["platform_packages"] = [ - dict(name=item["name"], version=item["version"]) - for item in self.dump_used_packages() - ] - topts["platform"] = {"name": self.name, "version": self.version} - if self.src_version: - topts["platform"]["src_version"] = self.src_version - telemetry.send_run_environment(topts, targets) - - def _run_scons(self, variables, targets, jobs): - args = [ - proc.get_pythonexe_path(), - join(get_core_package_dir("tool-scons"), "script", "scons"), - "-Q", - "--warn=no-no-parallel-support", - "--jobs", - str(jobs), - "--sconstruct", - join(fs.get_source_dir(), "builder", "main.py"), - ] - args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) - # pylint: disable=protected-access - args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) - args += targets - - # encode and append variables - for key, value in variables.items(): - args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - - proc.copy_pythonpath_to_osenv() - - if targets and "menuconfig" in targets: - return proc.exec_command( - args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin - ) - - if click._compat.isatty(sys.stdout): - - def _write_and_flush(stream, data): - try: - stream.write(data) - stream.flush() - except IOError: - pass - - return proc.exec_command( - args, - stdout=proc.BuildAsyncPipe( - line_callback=self._on_stdout_line, - data_callback=lambda data: _write_and_flush(sys.stdout, data), - ), - stderr=proc.BuildAsyncPipe( - line_callback=self._on_stderr_line, - data_callback=lambda data: _write_and_flush(sys.stderr, data), - ), - ) - - return proc.exec_command( - args, - stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), - stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), - ) - - def _on_stdout_line(self, line): - if "`buildprog' is up to date." in line: - return - self._echo_line(line, level=1) - - def _on_stderr_line(self, line): - is_error = self.LINE_ERROR_RE.search(line) is not None - self._echo_line(line, level=3 if is_error else 2) - - a_pos = line.find("fatal error:") - b_pos = line.rfind(": No such file or directory") - if a_pos == -1 or b_pos == -1: - return - self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) - - def _echo_line(self, line, level): - if line.startswith("scons: "): - line = line[7:] - assert 1 <= level <= 3 - if self.silent and (level < 2 or not line): - return - fg = (None, "yellow", "red")[level - 1] - if level == 1 and "is up to date" in line: - fg = "green" - click.secho(line, fg=fg, err=level > 1, nl=False) - - @staticmethod - def _echo_missed_dependency(filename): - if "/" in filename or not filename.endswith((".h", ".hpp")): - return - banner = """ -{dots} -* Looking for {filename_styled} dependency? Check our library registry! -* -* CLI > platformio lib search "header:{filename}" -* Web > {link} -* -{dots} -""".format( - filename=filename, - filename_styled=click.style(filename, fg="cyan"), - link=click.style( - "https://platformio.org/lib/search?query=header:%s" - % quote(filename, safe=""), - fg="blue", - ), - dots="*" * (56 + len(filename)), - ) - click.echo(banner, err=True) - - -class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): - - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) - _BOARDS_CACHE = {} - - def __init__(self, manifest_path): - self.manifest_path = manifest_path - self.silent = False - self.verbose = False - - self._manifest = fs.load_json(manifest_path) - self._BOARDS_CACHE = {} - self._custom_packages = None - - self.config = ProjectConfig.get_instance() - self.pm = PackageManager( - self.config.get_optional_dir("packages"), self.package_repositories - ) - - self._src_manifest = None - src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) - if src_manifest_path: - self._src_manifest = fs.load_json(src_manifest_path) - - # if self.engines and "platformio" in self.engines: - # if self.PIO_VERSION not in semantic_version.SimpleSpec( - # self.engines['platformio']): - # raise exception.IncompatiblePlatform(self.name, - # str(self.PIO_VERSION)) - - @property - def name(self): - return self._manifest["name"] - - @property - def title(self): - return self._manifest["title"] - - @property - def description(self): - return self._manifest["description"] - - @property - def version(self): - return self._manifest["version"] - - @property - def src_version(self): - return self._src_manifest.get("version") if self._src_manifest else None - - @property - def src_url(self): - return self._src_manifest.get("url") if self._src_manifest else None - - @property - def homepage(self): - return self._manifest.get("homepage") - - @property - def repository_url(self): - return self._manifest.get("repository", {}).get("url") - - @property - def license(self): - return self._manifest.get("license") - - @property - def frameworks(self): - return self._manifest.get("frameworks") - - @property - def engines(self): - return self._manifest.get("engines") - - @property - def package_repositories(self): - return self._manifest.get("packageRepositories") - - @property - def manifest(self): - return self._manifest - - @property - def packages(self): - packages = self._manifest.get("packages", {}) - for item in self._custom_packages or []: - name = item - version = "*" - if "@" in item: - name, version = item.split("@", 2) - name = name.strip() - if name not in packages: - packages[name] = {} - packages[name].update({"version": version.strip(), "optional": False}) - return packages - - @property - def python_packages(self): - return self._manifest.get("pythonPackages") - - def get_dir(self): - return dirname(self.manifest_path) - - def get_build_script(self): - main_script = join(self.get_dir(), "builder", "main.py") - if isfile(main_script): - return main_script - raise NotImplementedError() - - def is_embedded(self): - for opts in self.packages.values(): - if opts.get("type") == "uploader": - return True - return False - - def get_boards(self, id_=None): - def _append_board(board_id, manifest_path): - config = PlatformBoardConfig(manifest_path) - if "platform" in config and config.get("platform") != self.name: - return - if "platforms" in config and self.name not in config.get("platforms"): - return - config.manifest["platform"] = self.name - self._BOARDS_CACHE[board_id] = config - - bdirs = [ - self.config.get_optional_dir("boards"), - join(self.config.get_optional_dir("core"), "boards"), - join(self.get_dir(), "boards"), - ] - - if id_ is None: - for boards_dir in bdirs: - if not isdir(boards_dir): - continue - for item in sorted(os.listdir(boards_dir)): - _id = item[:-5] - if not item.endswith(".json") or _id in self._BOARDS_CACHE: - continue - _append_board(_id, join(boards_dir, item)) - else: - if id_ not in self._BOARDS_CACHE: - for boards_dir in bdirs: - if not isdir(boards_dir): - continue - manifest_path = join(boards_dir, "%s.json" % id_) - if isfile(manifest_path): - _append_board(id_, manifest_path) - break - if id_ not in self._BOARDS_CACHE: - raise exception.UnknownBoard(id_) - return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE - - def board_config(self, id_): - return self.get_boards(id_) - - def get_package_type(self, name): - return self.packages[name].get("type") - - def configure_default_packages(self, options, targets): - # override user custom packages - self._custom_packages = options.get("platform_packages") - - # enable used frameworks - for framework in options.get("framework", []): - if not self.frameworks: - continue - framework = framework.lower().strip() - if not framework or framework not in self.frameworks: - continue - _pkg_name = self.frameworks[framework].get("package") - if _pkg_name: - self.packages[_pkg_name]["optional"] = False - - # enable upload tools for upload targets - if any(["upload" in t for t in targets] + ["program" in targets]): - for name, opts in self.packages.items(): - if opts.get("type") == "uploader": - self.packages[name]["optional"] = False - # skip all packages in "nobuild" mode - # allow only upload tools and frameworks - elif "nobuild" in targets and opts.get("type") != "framework": - self.packages[name]["optional"] = True - - def get_lib_storages(self): - storages = {} - for opts in (self.frameworks or {}).values(): - if "package" not in opts: - continue - pkg_dir = self.get_package_dir(opts["package"]) - if not pkg_dir or not isdir(join(pkg_dir, "libraries")): - continue - libs_dir = join(pkg_dir, "libraries") - storages[libs_dir] = opts["package"] - libcores_dir = join(libs_dir, "__cores__") - if not isdir(libcores_dir): - continue - for item in os.listdir(libcores_dir): - libcore_dir = join(libcores_dir, item) - if not isdir(libcore_dir): - continue - storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) - - return [dict(name=name, path=path) for path, name in storages.items()] - - def on_installed(self): - pass - - def on_uninstalled(self): - pass - - def install_python_packages(self): - if not self.python_packages: - return None - click.echo( - "Installing Python packages: %s" - % ", ".join(list(self.python_packages.keys())), - ) - args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] - for name, requirements in self.python_packages.items(): - if any(c in requirements for c in ("<", ">", "=")): - args.append("%s%s" % (name, requirements)) - else: - args.append("%s==%s" % (name, requirements)) - try: - return subprocess.call(args) == 0 - except Exception as e: # pylint: disable=broad-except - click.secho( - "Could not install Python packages -> %s" % e, fg="red", err=True - ) - - def uninstall_python_packages(self): - if not self.python_packages: - return - click.echo("Uninstalling Python packages") - args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] - args.extend(list(self.python_packages.keys())) - try: - subprocess.call(args) == 0 - except Exception as e: # pylint: disable=broad-except - click.secho( - "Could not install Python packages -> %s" % e, fg="red", err=True - ) - - -class PlatformBoardConfig(object): - def __init__(self, manifest_path): - self._id = basename(manifest_path)[:-5] - assert isfile(manifest_path) - self.manifest_path = manifest_path - try: - self._manifest = fs.load_json(manifest_path) - except ValueError: - raise exception.InvalidBoardManifest(manifest_path) - if not set(["name", "url", "vendor"]) <= set(self._manifest): - raise exception.PlatformioException( - "Please specify name, url and vendor fields for " + manifest_path - ) - - def get(self, path, default=None): - try: - value = self._manifest - for k in path.split("."): - value = value[k] - # pylint: disable=undefined-variable - if PY2 and isinstance(value, unicode): - # cast to plain string from unicode for PY2, resolves issue in - # dev/platform when BoardConfig.get() is used in pair with - # os.path.join(file_encoding, unicode_encoding) - try: - value = value.encode("utf-8") - except UnicodeEncodeError: - pass - return value - except KeyError: - if default is not None: - return default - raise KeyError("Invalid board option '%s'" % path) - - def update(self, path, value): - newdict = None - for key in path.split(".")[::-1]: - if newdict is None: - newdict = {key: value} - else: - newdict = {key: newdict} - util.merge_dicts(self._manifest, newdict) - - def __contains__(self, key): - try: - self.get(key) - return True - except KeyError: - return False - - @property - def id(self): - return self._id - - @property - def id_(self): - return self.id - - @property - def manifest(self): - return self._manifest - - def get_brief_data(self): - result = { - "id": self.id, - "name": self._manifest["name"], - "platform": self._manifest.get("platform"), - "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), - "fcpu": int( - "".join( - [ - c - for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) - if c.isdigit() - ] - ) - ), - "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), - "rom": self._manifest.get("upload", {}).get("maximum_size", 0), - "frameworks": self._manifest.get("frameworks"), - "vendor": self._manifest["vendor"], - "url": self._manifest["url"], - } - if self._manifest.get("connectivity"): - result["connectivity"] = self._manifest.get("connectivity") - debug = self.get_debug_data() - if debug: - result["debug"] = debug - return result - - def get_debug_data(self): - if not self._manifest.get("debug", {}).get("tools"): - return None - tools = {} - for name, options in self._manifest["debug"]["tools"].items(): - tools[name] = {} - for key, value in options.items(): - if key in ("default", "onboard") and value: - tools[name][key] = value - return {"tools": tools} - - def get_debug_tool_name(self, custom=None): - debug_tools = self._manifest.get("debug", {}).get("tools") - tool_name = custom - if tool_name == "custom": - return tool_name - if not debug_tools: - telemetry.send_event("Debug", "Request", self.id) - raise DebugSupportError(self._manifest["name"]) - if tool_name: - if tool_name in debug_tools: - return tool_name - raise DebugInvalidOptionsError( - "Unknown debug tool `%s`. Please use one of `%s` or `custom`" - % (tool_name, ", ".join(sorted(list(debug_tools)))) - ) - - # automatically select best tool - data = {"default": [], "onboard": [], "external": []} - for key, value in debug_tools.items(): - if value.get("default"): - data["default"].append(key) - elif value.get("onboard"): - data["onboard"].append(key) - data["external"].append(key) - - for key, value in data.items(): - if not value: - continue - return sorted(value)[0] - - assert any(item for item in data) + raise UnknownBoard(id_) diff --git a/platformio/platform/__init__.py b/platformio/platform/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/platform/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py new file mode 100644 index 00000000..e626eb4b --- /dev/null +++ b/platformio/platform/_packages.py @@ -0,0 +1,126 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.exception import UnknownPackageError + + +class PlatformPackagesMixin(object): + def install_packages( # pylint: disable=too-many-arguments + self, + with_packages=None, + without_packages=None, + skip_default_package=False, + silent=False, + force=False, + ): + with_packages = set(self.find_pkg_names(with_packages or [])) + without_packages = set(self.find_pkg_names(without_packages or [])) + + upkgs = with_packages | without_packages + ppkgs = set(self.packages) + if not upkgs.issubset(ppkgs): + raise UnknownPackageError(", ".join(upkgs - ppkgs)) + + for name, opts in self.packages.items(): + version = opts.get("version", "") + if name in without_packages: + continue + if name in with_packages or not ( + skip_default_package or opts.get("optional", False) + ): + if ":" in version: + self.pm.install( + "%s=%s" % (name, version), silent=silent, force=force + ) + else: + self.pm.install(name, version, silent=silent, force=force) + + return True + + def find_pkg_names(self, candidates): + result = [] + for candidate in candidates: + found = False + + # lookup by package types + for _name, _opts in self.packages.items(): + if _opts.get("type") == candidate: + result.append(_name) + found = True + + if ( + self.frameworks + and candidate.startswith("framework-") + and candidate[10:] in self.frameworks + ): + result.append(self.frameworks[candidate[10:]]["package"]) + found = True + + if not found: + result.append(candidate) + + return result + + def update_packages(self, only_check=False): + for name, manifest in self.get_installed_packages().items(): + requirements = self.packages[name].get("version", "") + if ":" in requirements: + _, requirements, __ = self.pm.parse_pkg_uri(requirements) + self.pm.update(manifest["__pkg_dir"], requirements, only_check) + + def get_installed_packages(self): + items = {} + for name in self.packages: + pkg_dir = self.get_package_dir(name) + if pkg_dir: + items[name] = self.pm.load_manifest(pkg_dir) + return items + + def are_outdated_packages(self): + for name, manifest in self.get_installed_packages().items(): + requirements = self.packages[name].get("version", "") + if ":" in requirements: + _, requirements, __ = self.pm.parse_pkg_uri(requirements) + if self.pm.outdated(manifest["__pkg_dir"], requirements): + return True + return False + + def get_package_dir(self, name): + version = self.packages[name].get("version", "") + if ":" in version: + return self.pm.get_package_dir( + *self.pm.parse_pkg_uri("%s=%s" % (name, version)) + ) + return self.pm.get_package_dir(name, version) + + def get_package_version(self, name): + pkg_dir = self.get_package_dir(name) + if not pkg_dir: + return None + return self.pm.load_manifest(pkg_dir).get("version") + + def dump_used_packages(self): + result = [] + for name, options in self.packages.items(): + if options.get("optional"): + continue + pkg_dir = self.get_package_dir(name) + if not pkg_dir: + continue + manifest = self.pm.load_manifest(pkg_dir) + item = {"name": manifest["name"], "version": manifest["version"]} + if manifest.get("__src_url"): + item["src_url"] = manifest.get("__src_url") + result.append(item) + return result diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py new file mode 100644 index 00000000..39e30fce --- /dev/null +++ b/platformio/platform/_run.py @@ -0,0 +1,193 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import re +import sys + +import click + +from platformio import app, fs, proc, telemetry +from platformio.compat import hashlib_encode_data, is_bytes +from platformio.package.manager.core import get_core_package_dir +from platformio.platform.exception import BuildScriptNotFound + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + + +class PlatformRunMixin(object): + + LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) + + @staticmethod + def encode_scons_arg(value): + data = base64.urlsafe_b64encode(hashlib_encode_data(value)) + return data.decode() if is_bytes(data) else data + + @staticmethod + def decode_scons_arg(data): + value = base64.urlsafe_b64decode(data) + return value.decode() if is_bytes(value) else value + + def run( # pylint: disable=too-many-arguments + self, variables, targets, silent, verbose, jobs + ): + assert isinstance(variables, dict) + assert isinstance(targets, list) + + options = self.config.items(env=variables["pioenv"], as_dict=True) + if "framework" in options: + # support PIO Core 3.0 dev/platforms + options["pioframework"] = options["framework"] + self.configure_default_packages(options, targets) + self.install_packages(silent=True) + + self._report_non_sensitive_data(options, targets) + + self.silent = silent + self.verbose = verbose or app.get_setting("force_verbose") + + if "clean" in targets: + targets = ["-c", "."] + + variables["platform_manifest"] = self.manifest_path + + if "build_script" not in variables: + variables["build_script"] = self.get_build_script() + if not os.path.isfile(variables["build_script"]): + raise BuildScriptNotFound(variables["build_script"]) + + result = self._run_scons(variables, targets, jobs) + assert "returncode" in result + + return result + + def _report_non_sensitive_data(self, options, targets): + topts = options.copy() + topts["platform_packages"] = [ + dict(name=item["name"], version=item["version"]) + for item in self.dump_used_packages() + ] + topts["platform"] = {"name": self.name, "version": self.version} + if self.src_version: + topts["platform"]["src_version"] = self.src_version + telemetry.send_run_environment(topts, targets) + + def _run_scons(self, variables, targets, jobs): + args = [ + proc.get_pythonexe_path(), + os.path.join(get_core_package_dir("tool-scons"), "script", "scons"), + "-Q", + "--warn=no-no-parallel-support", + "--jobs", + str(jobs), + "--sconstruct", + os.path.join(fs.get_source_dir(), "builder", "main.py"), + ] + args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + # pylint: disable=protected-access + args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) + args += targets + + # encode and append variables + for key, value in variables.items(): + args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) + + proc.copy_pythonpath_to_osenv() + + if targets and "menuconfig" in targets: + return proc.exec_command( + args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin + ) + + if click._compat.isatty(sys.stdout): + + def _write_and_flush(stream, data): + try: + stream.write(data) + stream.flush() + except IOError: + pass + + return proc.exec_command( + args, + stdout=proc.BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: _write_and_flush(sys.stdout, data), + ), + stderr=proc.BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data), + ), + ) + + return proc.exec_command( + args, + stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), + stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + ) + + def _on_stdout_line(self, line): + if "`buildprog' is up to date." in line: + return + self._echo_line(line, level=1) + + def _on_stderr_line(self, line): + is_error = self.LINE_ERROR_RE.search(line) is not None + self._echo_line(line, level=3 if is_error else 2) + + a_pos = line.find("fatal error:") + b_pos = line.rfind(": No such file or directory") + if a_pos == -1 or b_pos == -1: + return + self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) + + def _echo_line(self, line, level): + if line.startswith("scons: "): + line = line[7:] + assert 1 <= level <= 3 + if self.silent and (level < 2 or not line): + return + fg = (None, "yellow", "red")[level - 1] + if level == 1 and "is up to date" in line: + fg = "green" + click.secho(line, fg=fg, err=level > 1, nl=False) + + @staticmethod + def _echo_missed_dependency(filename): + if "/" in filename or not filename.endswith((".h", ".hpp")): + return + banner = """ +{dots} +* Looking for {filename_styled} dependency? Check our library registry! +* +* CLI > platformio lib search "header:{filename}" +* Web > {link} +* +{dots} +""".format( + filename=filename, + filename_styled=click.style(filename, fg="cyan"), + link=click.style( + "https://platformio.org/lib/search?query=header:%s" + % quote(filename, safe=""), + fg="blue", + ), + dots="*" * (56 + len(filename)), + ) + click.echo(banner, err=True) diff --git a/platformio/platform/base.py b/platformio/platform/base.py new file mode 100644 index 00000000..a2fcd158 --- /dev/null +++ b/platformio/platform/base.py @@ -0,0 +1,274 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +import click +import semantic_version + +from platformio import __version__, fs, proc, util +from platformio.managers.package import PackageManager +from platformio.platform._packages import PlatformPackagesMixin +from platformio.platform._run import PlatformRunMixin +from platformio.platform.board import PlatformBoardConfig +from platformio.platform.exception import UnknownBoard +from platformio.project.config import ProjectConfig + + +class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods + PlatformPackagesMixin, PlatformRunMixin +): + + PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) + _BOARDS_CACHE = {} + + def __init__(self, manifest_path): + self.manifest_path = manifest_path + self.silent = False + self.verbose = False + + self._manifest = fs.load_json(manifest_path) + self._BOARDS_CACHE = {} + self._custom_packages = None + + self.config = ProjectConfig.get_instance() + self.pm = PackageManager( + self.config.get_optional_dir("packages"), self.package_repositories + ) + + self._src_manifest = None + src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) + if src_manifest_path: + self._src_manifest = fs.load_json(src_manifest_path) + + # if self.engines and "platformio" in self.engines: + # if self.PIO_VERSION not in semantic_version.SimpleSpec( + # self.engines['platformio']): + # raise exception.IncompatiblePlatform(self.name, + # str(self.PIO_VERSION)) + + @property + def name(self): + return self._manifest["name"] + + @property + def title(self): + return self._manifest["title"] + + @property + def description(self): + return self._manifest["description"] + + @property + def version(self): + return self._manifest["version"] + + @property + def src_version(self): + return self._src_manifest.get("version") if self._src_manifest else None + + @property + def src_url(self): + return self._src_manifest.get("url") if self._src_manifest else None + + @property + def homepage(self): + return self._manifest.get("homepage") + + @property + def repository_url(self): + return self._manifest.get("repository", {}).get("url") + + @property + def license(self): + return self._manifest.get("license") + + @property + def frameworks(self): + return self._manifest.get("frameworks") + + @property + def engines(self): + return self._manifest.get("engines") + + @property + def package_repositories(self): + return self._manifest.get("packageRepositories") + + @property + def manifest(self): + return self._manifest + + @property + def packages(self): + packages = self._manifest.get("packages", {}) + for item in self._custom_packages or []: + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 2) + name = name.strip() + if name not in packages: + packages[name] = {} + packages[name].update({"version": version.strip(), "optional": False}) + return packages + + @property + def python_packages(self): + return self._manifest.get("pythonPackages") + + def get_dir(self): + return os.path.dirname(self.manifest_path) + + def get_build_script(self): + main_script = os.path.join(self.get_dir(), "builder", "main.py") + if os.path.isfile(main_script): + return main_script + raise NotImplementedError() + + def is_embedded(self): + for opts in self.packages.values(): + if opts.get("type") == "uploader": + return True + return False + + def get_boards(self, id_=None): + def _append_board(board_id, manifest_path): + config = PlatformBoardConfig(manifest_path) + if "platform" in config and config.get("platform") != self.name: + return + if "platforms" in config and self.name not in config.get("platforms"): + return + config.manifest["platform"] = self.name + self._BOARDS_CACHE[board_id] = config + + bdirs = [ + self.config.get_optional_dir("boards"), + os.path.join(self.config.get_optional_dir("core"), "boards"), + os.path.join(self.get_dir(), "boards"), + ] + + if id_ is None: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + for item in sorted(os.listdir(boards_dir)): + _id = item[:-5] + if not item.endswith(".json") or _id in self._BOARDS_CACHE: + continue + _append_board(_id, os.path.join(boards_dir, item)) + else: + if id_ not in self._BOARDS_CACHE: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + manifest_path = os.path.join(boards_dir, "%s.json" % id_) + if os.path.isfile(manifest_path): + _append_board(id_, manifest_path) + break + if id_ not in self._BOARDS_CACHE: + raise UnknownBoard(id_) + return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE + + def board_config(self, id_): + return self.get_boards(id_) + + def get_package_type(self, name): + return self.packages[name].get("type") + + def configure_default_packages(self, options, targets): + # override user custom packages + self._custom_packages = options.get("platform_packages") + + # enable used frameworks + for framework in options.get("framework", []): + if not self.frameworks: + continue + framework = framework.lower().strip() + if not framework or framework not in self.frameworks: + continue + _pkg_name = self.frameworks[framework].get("package") + if _pkg_name: + self.packages[_pkg_name]["optional"] = False + + # enable upload tools for upload targets + if any(["upload" in t for t in targets] + ["program" in targets]): + for name, opts in self.packages.items(): + if opts.get("type") == "uploader": + self.packages[name]["optional"] = False + # skip all packages in "nobuild" mode + # allow only upload tools and frameworks + elif "nobuild" in targets and opts.get("type") != "framework": + self.packages[name]["optional"] = True + + def get_lib_storages(self): + storages = {} + for opts in (self.frameworks or {}).values(): + if "package" not in opts: + continue + pkg_dir = self.get_package_dir(opts["package"]) + if not pkg_dir or not os.path.isdir(os.path.join(pkg_dir, "libraries")): + continue + libs_dir = os.path.join(pkg_dir, "libraries") + storages[libs_dir] = opts["package"] + libcores_dir = os.path.join(libs_dir, "__cores__") + if not os.path.isdir(libcores_dir): + continue + for item in os.listdir(libcores_dir): + libcore_dir = os.path.join(libcores_dir, item) + if not os.path.isdir(libcore_dir): + continue + storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) + + return [dict(name=name, path=path) for path, name in storages.items()] + + def on_installed(self): + pass + + def on_uninstalled(self): + pass + + def install_python_packages(self): + if not self.python_packages: + return None + click.echo( + "Installing Python packages: %s" + % ", ".join(list(self.python_packages.keys())), + ) + args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] + for name, requirements in self.python_packages.items(): + if any(c in requirements for c in ("<", ">", "=")): + args.append("%s%s" % (name, requirements)) + else: + args.append("%s==%s" % (name, requirements)) + try: + return subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) + + def uninstall_python_packages(self): + if not self.python_packages: + return + click.echo("Uninstalling Python packages") + args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] + args.extend(list(self.python_packages.keys())) + try: + subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) diff --git a/platformio/platform/board.py b/platformio/platform/board.py new file mode 100644 index 00000000..900892cd --- /dev/null +++ b/platformio/platform/board.py @@ -0,0 +1,158 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs, telemetry, util +from platformio.commands.debug.exception import ( + DebugInvalidOptionsError, + DebugSupportError, +) +from platformio.compat import PY2 +from platformio.exception import UserSideException +from platformio.platform.exception import InvalidBoardManifest + + +class PlatformBoardConfig(object): + def __init__(self, manifest_path): + self._id = os.path.basename(manifest_path)[:-5] + assert os.path.isfile(manifest_path) + self.manifest_path = manifest_path + try: + self._manifest = fs.load_json(manifest_path) + except ValueError: + raise InvalidBoardManifest(manifest_path) + if not set(["name", "url", "vendor"]) <= set(self._manifest): + raise UserSideException( + "Please specify name, url and vendor fields for " + manifest_path + ) + + def get(self, path, default=None): + try: + value = self._manifest + for k in path.split("."): + value = value[k] + # pylint: disable=undefined-variable + if PY2 and isinstance(value, unicode): + # cast to plain string from unicode for PY2, resolves issue in + # dev/platform when BoardConfig.get() is used in pair with + # os.path.join(file_encoding, unicode_encoding) + try: + value = value.encode("utf-8") + except UnicodeEncodeError: + pass + return value + except KeyError: + if default is not None: + return default + raise KeyError("Invalid board option '%s'" % path) + + def update(self, path, value): + newdict = None + for key in path.split(".")[::-1]: + if newdict is None: + newdict = {key: value} + else: + newdict = {key: newdict} + util.merge_dicts(self._manifest, newdict) + + def __contains__(self, key): + try: + self.get(key) + return True + except KeyError: + return False + + @property + def id(self): + return self._id + + @property + def id_(self): + return self.id + + @property + def manifest(self): + return self._manifest + + def get_brief_data(self): + result = { + "id": self.id, + "name": self._manifest["name"], + "platform": self._manifest.get("platform"), + "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), + "fcpu": int( + "".join( + [ + c + for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) + if c.isdigit() + ] + ) + ), + "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), + "rom": self._manifest.get("upload", {}).get("maximum_size", 0), + "frameworks": self._manifest.get("frameworks"), + "vendor": self._manifest["vendor"], + "url": self._manifest["url"], + } + if self._manifest.get("connectivity"): + result["connectivity"] = self._manifest.get("connectivity") + debug = self.get_debug_data() + if debug: + result["debug"] = debug + return result + + def get_debug_data(self): + if not self._manifest.get("debug", {}).get("tools"): + return None + tools = {} + for name, options in self._manifest["debug"]["tools"].items(): + tools[name] = {} + for key, value in options.items(): + if key in ("default", "onboard") and value: + tools[name][key] = value + return {"tools": tools} + + def get_debug_tool_name(self, custom=None): + debug_tools = self._manifest.get("debug", {}).get("tools") + tool_name = custom + if tool_name == "custom": + return tool_name + if not debug_tools: + telemetry.send_event("Debug", "Request", self.id) + raise DebugSupportError(self._manifest["name"]) + if tool_name: + if tool_name in debug_tools: + return tool_name + raise DebugInvalidOptionsError( + "Unknown debug tool `%s`. Please use one of `%s` or `custom`" + % (tool_name, ", ".join(sorted(list(debug_tools)))) + ) + + # automatically select best tool + data = {"default": [], "onboard": [], "external": []} + for key, value in debug_tools.items(): + if value.get("default"): + data["default"].append(key) + elif value.get("onboard"): + data["onboard"].append(key) + data["external"].append(key) + + for key, value in data.items(): + if not value: + continue + return sorted(value)[0] + + assert any(item for item in data) diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py new file mode 100644 index 00000000..40431d7f --- /dev/null +++ b/platformio/platform/exception.py @@ -0,0 +1,49 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException + + +class PlatformException(PlatformioException): + pass + + +class UnknownPlatform(PlatformException): + + MESSAGE = "Unknown development platform '{0}'" + + +class IncompatiblePlatform(PlatformException): + + MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" + + +class UnknownBoard(PlatformException): + + MESSAGE = "Unknown board ID '{0}'" + + +class InvalidBoardManifest(PlatformException): + + MESSAGE = "Invalid board JSON manifest '{0}'" + + +class UnknownFramework(PlatformException): + + MESSAGE = "Unknown framework '{0}'" + + +class BuildScriptNotFound(PlatformException): + + MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py new file mode 100644 index 00000000..99e5f7c4 --- /dev/null +++ b/platformio/platform/factory.py @@ -0,0 +1,60 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from platformio.compat import load_python_module +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.base import PlatformBase +from platformio.platform.exception import UnknownPlatform + + +class PlatformFactory(object): + @staticmethod + def get_clsname(name): + name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) + return "%s%sPlatform" % (name.upper()[0], name.lower()[1:]) + + @staticmethod + def load_module(name, path): + try: + return load_python_module("platformio.platform.%s" % name, path) + except ImportError: + raise UnknownPlatform(name) + + @classmethod + def new(cls, pkg_or_spec): + pkg = PlatformPackageManager().get_package( + "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec + ) + if not pkg: + raise UnknownPlatform(pkg_or_spec) + + platform_cls = None + if os.path.isfile(os.path.join(pkg.path, "platform.py")): + platform_cls = getattr( + cls.load_module( + pkg.metadata.name, os.path.join(pkg.path, "platform.py") + ), + cls.get_clsname(pkg.metadata.name), + ) + else: + platform_cls = type( + str(cls.get_clsname(pkg.metadata.name)), (PlatformBase,), {} + ) + + _instance = platform_cls(os.path.join(pkg.path, "platform.json")) + assert isinstance(_instance, PlatformBase) + return _instance From 332874cd4b392e718590255fc5cc72fac5ae4e85 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 14 Aug 2020 16:48:12 +0300 Subject: [PATCH 154/223] Fix relative import of platform module on Py27 --- platformio/app.py | 2 ++ platformio/util.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/platformio/app.py b/platformio/app.py index a1a4ee73..00a8e89f 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + import codecs import getpass import hashlib diff --git a/platformio/util.py b/platformio/util.py index 5b38909d..a1974f17 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + import json import math import os From 4ec64f89800d81437f1d931c22fa7f3ea6f10b0f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 14 Aug 2020 17:00:18 +0300 Subject: [PATCH 155/223] Fix a test for examples --- tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index c9f64c18..d0a580d4 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -35,7 +35,7 @@ def pytest_generate_tests(metafunc): # dev/platforms for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) examples_dir = join(p.get_dir(), "examples") assert isdir(examples_dir) examples_dirs.append(examples_dir) From bb6fb3fdf8a72ccfb01c85a31c19718e0f148b84 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Aug 2020 15:24:35 +0300 Subject: [PATCH 156/223] Fix bug with parsing detached packages --- platformio/package/meta.py | 7 ++++++- tests/package/test_meta.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index f57f0495..fa93780e 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -229,6 +229,8 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes def _parse_requirements(self, raw): if "@" not in raw: return raw + if raw.startswith("file://") and os.path.exists(raw[7:]): + return raw tokens = raw.rsplit("@", 1) if any(s in tokens[1] for s in (":", "/")): return raw @@ -296,7 +298,10 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes def _parse_name_from_url(url): if url.endswith("/"): url = url[:-1] - for c in ("#", "?"): + stop_chars = ["#", "?"] + if url.startswith("file://"): + stop_chars.append("@") # detached path + for c in stop_chars: if c in url: url = url[: url.index(c)] diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index d7d4b820..0629f274 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -80,7 +80,7 @@ def test_spec_requirements(): assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") -def test_spec_local_urls(): +def test_spec_local_urls(tmpdir_factory): assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( url="file:///tmp/foo.tar.gz", name="foo" ) @@ -93,6 +93,11 @@ def test_spec_local_urls(): assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" ) + # detached folder with "@" symbol + pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir() + assert PackageSpec("file://%s" % str(pkg_dir)) == PackageSpec( + name="detached", url="file://%s" % pkg_dir + ) def test_spec_external_urls(): From 04694b4126ba793bd24dbcd537f35f554f6973fc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Aug 2020 23:11:01 +0300 Subject: [PATCH 157/223] Switch legacy platform manager to the new --- platformio/builder/tools/pioide.py | 6 +- platformio/builder/tools/pioplatform.py | 29 +- platformio/commands/boards.py | 4 +- .../commands/home/rpc/handlers/project.py | 10 +- platformio/commands/lib/helpers.py | 8 +- platformio/commands/platform.py | 115 +-- platformio/commands/project.py | 10 +- platformio/commands/system/command.py | 4 +- platformio/maintenance.py | 30 +- platformio/managers/package.py | 816 ------------------ platformio/managers/platform.py | 198 +---- platformio/package/manager/_install.py | 20 +- platformio/package/manager/_registry.py | 7 +- platformio/package/manager/_uninstall.py | 4 +- platformio/package/manager/_update.py | 43 +- platformio/package/manager/base.py | 6 + platformio/package/manager/platform.py | 164 ++++ platformio/package/manager/tool.py | 7 +- platformio/platform/_packages.py | 129 +-- platformio/platform/_run.py | 6 +- platformio/platform/base.py | 66 +- platformio/platform/exception.py | 5 +- platformio/platform/factory.py | 15 +- tests/commands/test_platform.py | 29 +- tests/test_maintenance.py | 10 +- 25 files changed, 470 insertions(+), 1271 deletions(-) delete mode 100644 platformio/managers/package.py diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 6a3d343d..c21b1500 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -45,10 +45,10 @@ def _dump_includes(env): # includes from toolchains p = env.PioPlatform() includes["toolchain"] = [] - for name in p.get_installed_packages(): - if p.get_package_type(name) != "toolchain": + for pkg in p.get_installed_packages(): + if p.get_package_type(pkg.metadata.name) != "toolchain": continue - toolchain_dir = glob_escape(p.get_package_dir(name)) + toolchain_dir = glob_escape(pkg.path) toolchain_incglobs = [ os.path.join(toolchain_dir, "*", "include", "c++", "*"), os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index fe7d6e28..740f6148 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -22,6 +22,7 @@ from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import fs, util from platformio.compat import WINDOWS +from platformio.package.meta import PackageItem from platformio.platform.exception import UnknownBoard from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions @@ -63,32 +64,30 @@ def GetFrameworkScript(env, framework): def LoadPioPlatform(env): p = env.PioPlatform() - installed_packages = p.get_installed_packages() # Ensure real platform name env["PIOPLATFORM"] = p.name # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH systype = util.get_systype() - for name in installed_packages: - type_ = p.get_package_type(name) + for pkg in p.get_installed_packages(): + type_ = p.get_package_type(pkg.metadata.name) if type_ not in ("toolchain", "uploader", "debugger"): continue - pkg_dir = p.get_package_dir(name) env.PrependENVPath( "PATH", - os.path.join(pkg_dir, "bin") - if os.path.isdir(os.path.join(pkg_dir, "bin")) - else pkg_dir, + os.path.join(pkg.path, "bin") + if os.path.isdir(os.path.join(pkg.path, "bin")) + else pkg.path, ) if ( not WINDOWS - and os.path.isdir(os.path.join(pkg_dir, "lib")) + and os.path.isdir(os.path.join(pkg.path, "lib")) and type_ != "toolchain" ): env.PrependENVPath( "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH", - os.path.join(pkg_dir, "lib"), + os.path.join(pkg.path, "lib"), ) # Platform specific LD Scripts @@ -133,6 +132,7 @@ def LoadPioPlatform(env): def PrintConfiguration(env): # pylint: disable=too-many-statements platform = env.PioPlatform() + pkg_metadata = PackageItem(platform.get_dir()).metadata board_config = env.BoardConfig() if "BOARD" in env else None def _get_configuration_data(): @@ -147,11 +147,12 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements ) def _get_plaform_data(): - data = ["PLATFORM: %s (%s)" % (platform.title, platform.version)] - if platform.src_version: - data.append("#" + platform.src_version) - if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url: - data.append("(%s)" % platform.src_url) + data = [ + "PLATFORM: %s (%s)" + % (platform.title, pkg_metadata.version or platform.version) + ] + if int(ARGUMENTS.get("PIOVERBOSE", 0)) and pkg_metadata.spec.external: + data.append("(%s)" % pkg_metadata.spec.url) if board_config: data.extend([">", board_config.get("name")]) return data diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index e7e2ecd6..21614b13 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -19,7 +19,7 @@ from tabulate import tabulate from platformio import fs from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager @click.command("boards", short_help="Embedded Board Explorer") @@ -71,7 +71,7 @@ def print_boards(boards): def _get_boards(installed=False): - pm = PlatformManager() + pm = PlatformPackageManager() return pm.get_installed_boards() if installed else pm.get_all_boards() diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 3f4cdc88..2db966b6 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -25,7 +25,7 @@ from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError from platformio.project.helpers import get_project_dir, is_platformio_project @@ -105,7 +105,7 @@ class ProjectRPC(object): return (os.path.sep).join(path.split(os.path.sep)[-2:]) result = [] - pm = PlatformManager() + pm = PlatformPackageManager() for project_dir in AppRPC.load_state()["storage"]["recentProjects"]: if not os.path.isdir(project_dir): continue @@ -148,8 +148,9 @@ class ProjectRPC(object): @staticmethod def get_project_examples(): result = [] - for manifest in PlatformManager().get_installed(): - examples_dir = os.path.join(manifest["__pkg_dir"], "examples") + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + examples_dir = os.path.join(pkg.path, "examples") if not os.path.isdir(examples_dir): continue items = [] @@ -172,6 +173,7 @@ class ProjectRPC(object): "description": project_description, } ) + manifest = pm.load_manifest(pkg) result.append( { "platform": { diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index 23892ac8..a5b0e260 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -15,7 +15,7 @@ import os from platformio.compat import ci_strings_are_equal -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -28,9 +28,9 @@ def get_builtin_libs(storage_names=None): items = [] storage_names = storage_names or [] - pm = PlatformManager() - for manifest in pm.get_installed(): - p = PlatformFactory.new(manifest["__pkg_dir"]) + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) for storage in p.get_lib_storages(): if storage_names and storage["name"] not in storage_names: continue diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 14d5a1f2..b996a16a 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import dirname, isdir +import os import click from platformio import app, util from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageItem, PackageSpec from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory @@ -48,7 +49,7 @@ def _print_platforms(platforms): if "version" in platform: if "__src_url" in platform: click.echo( - "Version: #%s (%s)" % (platform["version"], platform["__src_url"]) + "Version: %s (%s)" % (platform["version"], platform["__src_url"]) ) else: click.echo("Version: " + platform["version"]) @@ -56,11 +57,7 @@ def _print_platforms(platforms): def _get_registry_platforms(): - platforms = util.get_api_result("/platforms", cache_valid="7d") - pm = PlatformManager() - for platform in platforms or []: - platform["versions"] = pm.get_all_repo_versions(platform["name"]) - return platforms + return util.get_api_result("/platforms", cache_valid="7d") def _get_platform_data(*args, **kwargs): @@ -91,7 +88,9 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru # return data # overwrite VCS version and add extra fields - manifest = PlatformManager().load_manifest(dirname(p.manifest_path)) + manifest = PlatformPackageManager().legacy_load_manifest( + os.path.dirname(p.manifest_path) + ) assert manifest for key in manifest: if key == "version" or key.startswith("__"): @@ -104,13 +103,15 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru return data data["packages"] = [] - installed_pkgs = p.get_installed_packages() - for name, opts in p.packages.items(): + installed_pkgs = { + pkg.metadata.name: p.pm.load_manifest(pkg) for pkg in p.get_installed_packages() + } + for name, options in p.packages.items(): item = dict( name=name, type=p.get_package_type(name), - requirements=opts.get("version"), - optional=opts.get("optional") is True, + requirements=options.get("version"), + optional=options.get("optional") is True, ) if name in installed_pkgs: for key, value in installed_pkgs[name].items(): @@ -147,13 +148,13 @@ def _get_registry_platform_data( # pylint: disable=unused-argument forDesktop=_data["forDesktop"], frameworks=_data["frameworks"], packages=_data["packages"], - versions=_data["versions"], + versions=_data.get("versions"), ) if with_boards: data["boards"] = [ board - for board in PlatformManager().get_registered_boards() + for board in PlatformPackageManager().get_registered_boards() if board["platform"] == _data["name"] ] @@ -213,12 +214,10 @@ def platform_frameworks(query, json_output): @click.option("--json-output", is_flag=True) def platform_list(json_output): platforms = [] - pm = PlatformManager() - for manifest in pm.get_installed(): + pm = PlatformPackageManager() + for pkg in pm.get_installed(): platforms.append( - _get_installed_platform_data( - manifest["__pkg_dir"], with_boards=False, expose_packages=False - ) + _get_installed_platform_data(pkg, with_boards=False, expose_packages=False) ) platforms = sorted(platforms, key=lambda manifest: manifest["name"]) @@ -300,6 +299,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches @click.option("--without-package", multiple=True) @click.option("--skip-default-package", is_flag=True) @click.option("--with-all-packages", is_flag=True) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( "-f", "--force", @@ -312,21 +312,24 @@ def platform_install( # pylint: disable=too-many-arguments without_package, skip_default_package, with_all_packages, + silent, force, ): - pm = PlatformManager() + pm = PlatformPackageManager() for platform in platforms: - if pm.install( - name=platform, + pkg = pm.install( + spec=platform, with_packages=with_package, without_packages=without_package, skip_default_package=skip_default_package, with_all_packages=with_all_packages, + silent=silent, force=force, - ): + ) + if pkg and not silent: click.secho( "The platform '%s' has been successfully installed!\n" - "The rest of packages will be installed automatically " + "The rest of the packages will be installed later " "depending on your build environment." % platform, fg="green", ) @@ -335,11 +338,11 @@ def platform_install( # pylint: disable=too-many-arguments @cli.command("uninstall", short_help="Uninstall development platform") @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") def platform_uninstall(platforms): - pm = PlatformManager() + pm = PlatformPackageManager() for platform in platforms: if pm.uninstall(platform): click.secho( - "The platform '%s' has been successfully uninstalled!" % platform, + "The platform '%s' has been successfully removed!" % platform, fg="green", ) @@ -358,41 +361,40 @@ def platform_uninstall(platforms): @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) -def platform_update( # pylint: disable=too-many-locals - platforms, only_packages, only_check, dry_run, json_output +def platform_update( # pylint: disable=too-many-locals, too-many-arguments + platforms, only_packages, only_check, dry_run, silent, json_output ): - pm = PlatformManager() - pkg_dir_to_name = {} - if not platforms: - platforms = [] - for manifest in pm.get_installed(): - platforms.append(manifest["__pkg_dir"]) - pkg_dir_to_name[manifest["__pkg_dir"]] = manifest.get( - "title", manifest["name"] - ) - + pm = PlatformPackageManager() + platforms = platforms or pm.get_installed() only_check = dry_run or only_check if only_check and json_output: result = [] for platform in platforms: - pkg_dir = platform if isdir(platform) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = pm.parse_pkg_uri(platform) - pkg_dir = pm.get_package_dir(name, requirements, url) - if not pkg_dir: + spec = None + pkg = None + if isinstance(platform, PackageItem): + pkg = platform + else: + spec = PackageSpec(platform) + pkg = pm.get_package(spec) + if not pkg: continue - latest = pm.outdated(pkg_dir, requirements) - if not latest and not PlatformFactory.new(pkg_dir).are_outdated_packages(): + outdated = pm.outdated(pkg, spec) + if ( + not outdated.is_outdated(allow_incompatible=True) + and not PlatformFactory.new(pkg).are_outdated_packages() + ): continue data = _get_installed_platform_data( - pkg_dir, with_boards=False, expose_packages=False + pkg, with_boards=False, expose_packages=False ) - if latest: - data["versionLatest"] = latest + if outdated.is_outdated(allow_incompatible=True): + data["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) result.append(data) return click.echo(dump_json_to_unicode(result)) @@ -401,10 +403,17 @@ def platform_update( # pylint: disable=too-many-locals for platform in platforms: click.echo( "Platform %s" - % click.style(pkg_dir_to_name.get(platform, platform), fg="cyan") + % click.style( + platform.metadata.name + if isinstance(platform, PackageItem) + else platform, + fg="cyan", + ) ) click.echo("--------") - pm.update(platform, only_packages=only_packages, only_check=only_check) + pm.update( + platform, only_packages=only_packages, only_check=only_check, silent=silent + ) click.echo() return True diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 6194a915..6900ce74 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -23,7 +23,7 @@ from tabulate import tabulate from platformio import fs from platformio.commands.platform import platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -109,7 +109,7 @@ def project_idedata(project_dir, environment, json_output): def validate_boards(ctx, param, value): # pylint: disable=W0613 - pm = PlatformManager() + pm = PlatformPackageManager() for id_ in value: try: pm.board_config(id_) @@ -367,7 +367,7 @@ def fill_project_envs( if all(cond): used_boards.append(config.get(section, "board")) - pm = PlatformManager() + pm = PlatformPackageManager() used_platforms = [] modified = False for id_ in board_ids: @@ -404,7 +404,9 @@ def fill_project_envs( def _install_dependent_platforms(ctx, platforms): - installed_platforms = [p["name"] for p in PlatformManager().get_installed()] + installed_platforms = [ + pkg.metadata.name for pkg in PlatformPackageManager().get_installed() + ] if set(platforms) <= set(installed_platforms): return ctx.invoke( diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 76d2cb36..2fee5471 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -26,8 +26,8 @@ from platformio.commands.system.completion import ( install_completion_code, uninstall_completion_code, ) -from platformio.managers.platform import PlatformManager from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig @@ -77,7 +77,7 @@ def system_info(json_output): } data["dev_platform_nums"] = { "title": "Development Platforms", - "value": len(PlatformManager().get_installed()), + "value": len(PlatformPackageManager().get_installed()), } data["package_tool_nums"] = { "title": "Tools & Toolchains", diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 4b47d50a..54b0ad6d 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,9 +25,9 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.platform import PlatformManager from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory @@ -271,24 +271,16 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches util.internet_on(raise_exception=True) outdated_items = [] - pm = PlatformManager() if what == "platforms" else LibraryPackageManager() - if isinstance(pm, PlatformManager): - for manifest in pm.get_installed(): - if manifest["name"] in outdated_items: - continue - conds = [ - pm.outdated(manifest["__pkg_dir"]), - what == "platforms" - and PlatformFactory.new(manifest["__pkg_dir"]).are_outdated_packages(), - ] - if any(conds): - outdated_items.append(manifest["name"]) - else: - for pkg in pm.get_installed(): - if pkg.metadata.name in outdated_items: - continue - if pm.outdated(pkg).is_outdated(): - outdated_items.append(pkg.metadata.name) + pm = PlatformPackageManager() if what == "platforms" else LibraryPackageManager() + for pkg in pm.get_installed(): + if pkg.metadata.name in outdated_items: + continue + conds = [ + pm.outdated(pkg).is_outdated(), + what == "platforms" and PlatformFactory.new(pkg).are_outdated_packages(), + ] + if any(conds): + outdated_items.append(pkg.metadata.name) if not outdated_items: return diff --git a/platformio/managers/package.py b/platformio/managers/package.py deleted file mode 100644 index 071d6788..00000000 --- a/platformio/managers/package.py +++ /dev/null @@ -1,816 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib -import json -import os -import re -import shutil -from os.path import basename, getsize, isdir, isfile, islink, join, realpath -from tempfile import mkdtemp - -import click -import requests -import semantic_version - -from platformio import __version__, app, exception, fs, util -from platformio.compat import hashlib_encode_data -from platformio.package.download import FileDownloader -from platformio.package.exception import ManifestException -from platformio.package.lockfile import LockFile -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.package.unpack import FileUnpacker -from platformio.package.vcsclient import VCSClientFactory - -# pylint: disable=too-many-arguments, too-many-return-statements - - -class PackageRepoIterator(object): - def __init__(self, package, repositories): - assert isinstance(repositories, list) - self.package = package - self.repositories = iter(repositories) - - def __iter__(self): - return self - - def __next__(self): - return self.next() # pylint: disable=not-callable - - @staticmethod - @util.memoized(expire="60s") - def load_manifest(url): - r = None - try: - r = requests.get(url, headers={"User-Agent": app.get_user_agent()}) - r.raise_for_status() - return r.json() - except: # pylint: disable=bare-except - pass - finally: - if r: - r.close() - return None - - def next(self): - repo = next(self.repositories) - manifest = repo if isinstance(repo, dict) else self.load_manifest(repo) - if manifest and self.package in manifest: - return manifest[self.package] - return next(self) - - -class PkgRepoMixin(object): - - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) - - @staticmethod - def is_system_compatible(valid_systems): - if not valid_systems or "*" in valid_systems: - return True - if not isinstance(valid_systems, list): - valid_systems = list([valid_systems]) - return util.get_systype() in valid_systems - - def max_satisfying_repo_version(self, versions, requirements=None): - item = None - reqspec = None - try: - reqspec = ( - semantic_version.SimpleSpec(requirements) if requirements else None - ) - except ValueError: - pass - - for v in versions: - if not self.is_system_compatible(v.get("system")): - continue - # if "platformio" in v.get("engines", {}): - # if PkgRepoMixin.PIO_VERSION not in requirements.SimpleSpec( - # v['engines']['platformio']): - # continue - specver = semantic_version.Version(v["version"]) - if reqspec and specver not in reqspec: - continue - if not item or semantic_version.Version(item["version"]) < specver: - item = v - return item - - def get_latest_repo_version( # pylint: disable=unused-argument - self, name, requirements, silent=False - ): - version = None - for versions in PackageRepoIterator(name, self.repositories): - pkgdata = self.max_satisfying_repo_version(versions, requirements) - if not pkgdata: - continue - if ( - not version - or semantic_version.compare(pkgdata["version"], version) == 1 - ): - version = pkgdata["version"] - return version - - def get_all_repo_versions(self, name): - result = [] - for versions in PackageRepoIterator(name, self.repositories): - result.extend([semantic_version.Version(v["version"]) for v in versions]) - return [str(v) for v in sorted(set(result))] - - -class PkgInstallerMixin(object): - - SRC_MANIFEST_NAME = ".piopkgmanager.json" - TMP_FOLDER_PREFIX = "_tmp_installing-" - - FILE_CACHE_VALID = None # for example, 1 week = "7d" - FILE_CACHE_MAX_SIZE = 1024 * 1024 * 50 # 50 Mb - - MEMORY_CACHE = {} # cache for package manifests and read dirs - - def cache_get(self, key, default=None): - return self.MEMORY_CACHE.get(key, default) - - def cache_set(self, key, value): - self.MEMORY_CACHE[key] = value - - def cache_reset(self): - self.MEMORY_CACHE.clear() - - def read_dirs(self, src_dir): - cache_key = "read_dirs-%s" % src_dir - result = self.cache_get(cache_key) - if result: - return result - result = [ - join(src_dir, name) - for name in sorted(os.listdir(src_dir)) - if isdir(join(src_dir, name)) - ] - self.cache_set(cache_key, result) - return result - - def download(self, url, dest_dir, sha1=None): - cache_key_fname = app.ContentCache.key_from_args(url, "fname") - cache_key_data = app.ContentCache.key_from_args(url, "data") - if self.FILE_CACHE_VALID: - with app.ContentCache() as cc: - fname = str(cc.get(cache_key_fname)) - cache_path = cc.get_cache_path(cache_key_data) - if fname and isfile(cache_path): - dst_path = join(dest_dir, fname) - shutil.copy(cache_path, dst_path) - click.echo("Using cache: %s" % cache_path) - return dst_path - - with_progress = not app.is_disabled_progressbar() - try: - fd = FileDownloader(url, dest_dir) - fd.start(with_progress=with_progress) - except IOError as e: - raise_error = not with_progress - if with_progress: - try: - fd = FileDownloader(url, dest_dir) - fd.start(with_progress=False) - except IOError: - raise_error = True - if raise_error: - click.secho( - "Error: Please read http://bit.ly/package-manager-ioerror", - fg="red", - err=True, - ) - raise e - - if sha1: - fd.verify(sha1) - dst_path = fd.get_filepath() - if ( - not self.FILE_CACHE_VALID - or getsize(dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE - ): - return dst_path - - with app.ContentCache() as cc: - cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID) - cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID) - shutil.copy(dst_path, cc.get_cache_path(cache_key_data)) - return dst_path - - @staticmethod - def unpack(source_path, dest_dir): - with_progress = not app.is_disabled_progressbar() - try: - with FileUnpacker(source_path) as fu: - return fu.unpack(dest_dir, with_progress=with_progress) - except IOError as e: - if not with_progress: - raise e - with FileUnpacker(source_path) as fu: - return fu.unpack(dest_dir, with_progress=False) - - @staticmethod - def parse_semver_version(value, raise_exception=False): - try: - try: - return semantic_version.Version(value) - except ValueError: - if "." not in str(value) and not str(value).isdigit(): - raise ValueError("Invalid SemVer version %s" % value) - return semantic_version.Version.coerce(value) - except ValueError as e: - if raise_exception: - raise e - return None - - @staticmethod - def parse_pkg_uri(text, requirements=None): # pylint: disable=too-many-branches - text = str(text) - name, url = None, None - - # Parse requirements - req_conditions = [ - "@" in text, - not requirements, - ":" not in text or text.rfind("/") < text.rfind("@"), - ] - if all(req_conditions): - text, requirements = text.rsplit("@", 1) - - # Handle PIO Library Registry ID - if text.isdigit(): - text = "id=" + text - # Parse custom name - elif "=" in text and not text.startswith("id="): - name, text = text.split("=", 1) - - # Parse URL - # if valid URL with scheme vcs+protocol:// - if "+" in text and text.find("+") < text.find("://"): - url = text - elif "/" in text or "\\" in text: - git_conditions = [ - # Handle GitHub URL (https://github.com/user/package) - text.startswith("https://github.com/") - and not text.endswith((".zip", ".tar.gz")), - (text.split("#", 1)[0] if "#" in text else text).endswith(".git"), - ] - hg_conditions = [ - # Handle Developer Mbed URL - # (https://developer.mbed.org/users/user/code/package/) - # (https://os.mbed.com/users/user/code/package/) - text.startswith("https://developer.mbed.org"), - text.startswith("https://os.mbed.com"), - ] - if any(git_conditions): - url = "git+" + text - elif any(hg_conditions): - url = "hg+" + text - elif "://" not in text and (isfile(text) or isdir(text)): - url = "file://" + text - elif "://" in text: - url = text - # Handle short version of GitHub URL - elif text.count("/") == 1: - url = "git+https://github.com/" + text - - # Parse name from URL - if url and not name: - _url = url.split("#", 1)[0] if "#" in url else url - if _url.endswith(("\\", "/")): - _url = _url[:-1] - name = basename(_url) - if "." in name and not name.startswith("."): - name = name.rsplit(".", 1)[0] - - return (name or text, requirements, url) - - @staticmethod - def get_install_dirname(manifest): - name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest["name"], flags=re.I) - if "id" in manifest: - name += "_ID%d" % manifest["id"] - return str(name) - - @classmethod - def get_src_manifest_path(cls, pkg_dir): - if not isdir(pkg_dir): - return None - for item in os.listdir(pkg_dir): - if not isdir(join(pkg_dir, item)): - continue - if isfile(join(pkg_dir, item, cls.SRC_MANIFEST_NAME)): - return join(pkg_dir, item, cls.SRC_MANIFEST_NAME) - return None - - def get_manifest_path(self, pkg_dir): - if not isdir(pkg_dir): - return None - for name in self.manifest_names: - manifest_path = join(pkg_dir, name) - if isfile(manifest_path): - return manifest_path - return None - - def manifest_exists(self, pkg_dir): - return self.get_manifest_path(pkg_dir) or self.get_src_manifest_path(pkg_dir) - - def load_manifest(self, pkg_dir): # pylint: disable=too-many-branches - cache_key = "load_manifest-%s" % pkg_dir - result = self.cache_get(cache_key) - if result: - return result - - manifest = {} - src_manifest = None - manifest_path = self.get_manifest_path(pkg_dir) - src_manifest_path = self.get_src_manifest_path(pkg_dir) - if src_manifest_path: - src_manifest = fs.load_json(src_manifest_path) - - if not manifest_path and not src_manifest_path: - return None - - try: - manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict() - except ManifestException: - pass - - if src_manifest: - if "version" in src_manifest: - manifest["version"] = src_manifest["version"] - manifest["__src_url"] = src_manifest["url"] - # handle a custom package name - autogen_name = self.parse_pkg_uri(manifest["__src_url"])[0] - if "name" not in manifest or autogen_name != src_manifest["name"]: - manifest["name"] = src_manifest["name"] - - if "name" not in manifest: - manifest["name"] = basename(pkg_dir) - if "version" not in manifest: - manifest["version"] = "0.0.0" - - manifest["__pkg_dir"] = realpath(pkg_dir) - self.cache_set(cache_key, manifest) - return manifest - - def get_installed(self): - items = [] - for pkg_dir in self.read_dirs(self.package_dir): - if self.TMP_FOLDER_PREFIX in pkg_dir: - continue - manifest = self.load_manifest(pkg_dir) - if not manifest: - continue - assert "name" in manifest - items.append(manifest) - return items - - def get_package(self, name, requirements=None, url=None): - pkg_id = int(name[3:]) if name.startswith("id=") else 0 - best = None - for manifest in self.get_installed(): - if url: - if manifest.get("__src_url") != url: - continue - elif pkg_id and manifest.get("id") != pkg_id: - continue - elif not pkg_id and manifest["name"] != name: - continue - elif not PkgRepoMixin.is_system_compatible(manifest.get("system")): - continue - - # strict version or VCS HASH - if requirements and requirements == manifest["version"]: - return manifest - - try: - if requirements and not semantic_version.SimpleSpec(requirements).match( - self.parse_semver_version(manifest["version"], raise_exception=True) - ): - continue - if not best or ( - self.parse_semver_version(manifest["version"], raise_exception=True) - > self.parse_semver_version(best["version"], raise_exception=True) - ): - best = manifest - except ValueError: - pass - - return best - - def get_package_dir(self, name, requirements=None, url=None): - manifest = self.get_package(name, requirements, url) - return ( - manifest.get("__pkg_dir") - if manifest and isdir(manifest.get("__pkg_dir")) - else None - ) - - def get_package_by_dir(self, pkg_dir): - for manifest in self.get_installed(): - if manifest["__pkg_dir"] == realpath(pkg_dir): - return manifest - return None - - def find_pkg_root(self, src_dir): - if self.manifest_exists(src_dir): - return src_dir - for root, _, _ in os.walk(src_dir): - if self.manifest_exists(root): - return root - raise exception.MissingPackageManifest(", ".join(self.manifest_names)) - - def _install_from_piorepo(self, name, requirements): - pkg_dir = None - pkgdata = None - versions = None - last_exc = None - for versions in PackageRepoIterator(name, self.repositories): - pkgdata = self.max_satisfying_repo_version(versions, requirements) - if not pkgdata: - continue - try: - pkg_dir = self._install_from_url( - name, pkgdata["url"], requirements, pkgdata.get("sha1") - ) - break - except Exception as e: # pylint: disable=broad-except - last_exc = e - click.secho("Warning! Package Mirror: %s" % e, fg="yellow") - click.secho("Looking for another mirror...", fg="yellow") - - if versions is None: - util.internet_on(raise_exception=True) - raise exception.UnknownPackage( - name + (". Error -> %s" % last_exc if last_exc else "") - ) - if not pkgdata: - raise exception.UndefinedPackageVersion( - requirements or "latest", util.get_systype() - ) - return pkg_dir - - def _install_from_url(self, name, url, requirements=None, sha1=None, track=False): - tmp_dir = mkdtemp("-package", self.TMP_FOLDER_PREFIX, self.package_dir) - src_manifest_dir = None - src_manifest = {"name": name, "url": url, "requirements": requirements} - - try: - if url.startswith("file://"): - _url = url[7:] - if isfile(_url): - self.unpack(_url, tmp_dir) - else: - fs.rmtree(tmp_dir) - shutil.copytree(_url, tmp_dir, symlinks=True) - elif url.startswith(("http://", "https://")): - dlpath = self.download(url, tmp_dir, sha1) - assert isfile(dlpath) - self.unpack(dlpath, tmp_dir) - os.remove(dlpath) - else: - vcs = VCSClientFactory.new(tmp_dir, url) - assert vcs.export() - src_manifest_dir = vcs.storage_dir - src_manifest["version"] = vcs.get_current_revision() - - _tmp_dir = tmp_dir - if not src_manifest_dir: - _tmp_dir = self.find_pkg_root(tmp_dir) - src_manifest_dir = join(_tmp_dir, ".pio") - - # write source data to a special manifest - if track: - self._update_src_manifest(src_manifest, src_manifest_dir) - - return self._install_from_tmp_dir(_tmp_dir, requirements) - finally: - if isdir(tmp_dir): - fs.rmtree(tmp_dir) - return None - - def _update_src_manifest(self, data, src_dir): - if not isdir(src_dir): - os.makedirs(src_dir) - src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME) - _data = {} - if isfile(src_manifest_path): - _data = fs.load_json(src_manifest_path) - _data.update(data) - with open(src_manifest_path, "w") as fp: - json.dump(_data, fp) - - def _install_from_tmp_dir( # pylint: disable=too-many-branches - self, tmp_dir, requirements=None - ): - tmp_manifest = self.load_manifest(tmp_dir) - assert set(["name", "version"]) <= set(tmp_manifest) - - pkg_dirname = self.get_install_dirname(tmp_manifest) - pkg_dir = join(self.package_dir, pkg_dirname) - cur_manifest = self.load_manifest(pkg_dir) - - tmp_semver = self.parse_semver_version(tmp_manifest["version"]) - cur_semver = None - if cur_manifest: - cur_semver = self.parse_semver_version(cur_manifest["version"]) - - # package should satisfy requirements - if requirements: - mismatch_error = "Package version %s doesn't satisfy requirements %s" % ( - tmp_manifest["version"], - requirements, - ) - try: - assert tmp_semver and tmp_semver in semantic_version.SimpleSpec( - requirements - ), mismatch_error - except (AssertionError, ValueError): - assert tmp_manifest["version"] == requirements, mismatch_error - - # check if package already exists - if cur_manifest: - # 0-overwrite, 1-rename, 2-fix to a version - action = 0 - if "__src_url" in cur_manifest: - if cur_manifest["__src_url"] != tmp_manifest.get("__src_url"): - action = 1 - elif "__src_url" in tmp_manifest: - action = 2 - else: - if tmp_semver and (not cur_semver or tmp_semver > cur_semver): - action = 1 - elif tmp_semver and cur_semver and tmp_semver != cur_semver: - action = 2 - - # rename - if action == 1: - target_dirname = "%s@%s" % (pkg_dirname, cur_manifest["version"]) - if "__src_url" in cur_manifest: - target_dirname = "%s@src-%s" % ( - pkg_dirname, - hashlib.md5( - hashlib_encode_data(cur_manifest["__src_url"]) - ).hexdigest(), - ) - shutil.move(pkg_dir, join(self.package_dir, target_dirname)) - # fix to a version - elif action == 2: - target_dirname = "%s@%s" % (pkg_dirname, tmp_manifest["version"]) - if "__src_url" in tmp_manifest: - target_dirname = "%s@src-%s" % ( - pkg_dirname, - hashlib.md5( - hashlib_encode_data(tmp_manifest["__src_url"]) - ).hexdigest(), - ) - pkg_dir = join(self.package_dir, target_dirname) - - # remove previous/not-satisfied package - if isdir(pkg_dir): - fs.rmtree(pkg_dir) - shutil.copytree(tmp_dir, pkg_dir, symlinks=True) - try: - shutil.rmtree(tmp_dir) - except: # pylint: disable=bare-except - pass - assert isdir(pkg_dir) - self.cache_reset() - return pkg_dir - - -class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): - - # Handle circle dependencies - INSTALL_HISTORY = None - - def __init__(self, package_dir, repositories=None): - self.repositories = repositories - self.package_dir = package_dir - if not isdir(self.package_dir): - os.makedirs(self.package_dir) - assert isdir(self.package_dir) - - @property - def manifest_names(self): - raise NotImplementedError() - - def print_message(self, message, nl=True): - click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) - - def outdated(self, pkg_dir, requirements=None): - """ - Has 3 different results: - `None` - unknown package, VCS is detached to commit - `False` - package is up-to-date - `String` - a found latest version - """ - if not isdir(pkg_dir): - return None - latest = None - manifest = self.load_manifest(pkg_dir) - # skip detached package to a specific version - if "@" in pkg_dir and "__src_url" not in manifest and not requirements: - return None - - if "__src_url" in manifest: - try: - vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"], silent=True) - except (AttributeError, exception.PlatformioException): - return None - if not vcs.can_be_updated: - return None - latest = vcs.get_latest_revision() - else: - try: - latest = self.get_latest_repo_version( - "id=%d" % manifest["id"] if "id" in manifest else manifest["name"], - requirements, - silent=True, - ) - except (exception.PlatformioException, ValueError): - return None - - if not latest: - return None - - up_to_date = False - try: - assert "__src_url" not in manifest - up_to_date = self.parse_semver_version( - manifest["version"], raise_exception=True - ) >= self.parse_semver_version(latest, raise_exception=True) - except (AssertionError, ValueError): - up_to_date = latest == manifest["version"] - - return False if up_to_date else latest - - def install( - self, name, requirements=None, silent=False, after_update=False, force=False - ): # pylint: disable=unused-argument - pkg_dir = None - # interprocess lock - with LockFile(self.package_dir): - self.cache_reset() - - name, requirements, url = self.parse_pkg_uri(name, requirements) - package_dir = self.get_package_dir(name, requirements, url) - - # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] - history_key = "%s-%s-%s" % (name, requirements or "", url or "") - if history_key in self.INSTALL_HISTORY: - return package_dir - self.INSTALL_HISTORY.append(history_key) - - if package_dir and force: - self.uninstall(package_dir) - package_dir = None - - if not package_dir or not silent: - msg = "Installing " + click.style(name, fg="cyan") - if requirements: - msg += " @ " + requirements - self.print_message(msg) - if package_dir: - if not silent: - click.secho( - "{name} @ {version} is already installed".format( - **self.load_manifest(package_dir) - ), - fg="yellow", - ) - return package_dir - - if url: - pkg_dir = self._install_from_url(name, url, requirements, track=True) - else: - pkg_dir = self._install_from_piorepo(name, requirements) - - if not pkg_dir or not self.manifest_exists(pkg_dir): - raise exception.PackageInstallError( - name, requirements or "*", util.get_systype() - ) - - manifest = self.load_manifest(pkg_dir) - assert manifest - - click.secho( - "{name} @ {version} has been successfully installed!".format( - **manifest - ), - fg="green", - ) - - return pkg_dir - - def uninstall( - self, package, requirements=None, after_update=False - ): # pylint: disable=unused-argument - # interprocess lock - with LockFile(self.package_dir): - self.cache_reset() - - if isdir(package) and self.get_package_by_dir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise exception.UnknownPackage( - "%s @ %s" % (package, requirements or "*") - ) - - manifest = self.load_manifest(pkg_dir) - click.echo( - "Uninstalling %s @ %s: \t" - % (click.style(manifest["name"], fg="cyan"), manifest["version"]), - nl=False, - ) - - if islink(pkg_dir): - os.unlink(pkg_dir) - else: - fs.rmtree(pkg_dir) - self.cache_reset() - - # unfix package with the same name - pkg_dir = self.get_package_dir(manifest["name"]) - if pkg_dir and "@" in pkg_dir: - shutil.move( - pkg_dir, join(self.package_dir, self.get_install_dirname(manifest)) - ) - self.cache_reset() - - click.echo("[%s]" % click.style("OK", fg="green")) - - return True - - def update(self, package, requirements=None, only_check=False): - self.cache_reset() - if isdir(package) and self.get_package_by_dir(package): - pkg_dir = package - else: - pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package)) - - if not pkg_dir: - raise exception.UnknownPackage("%s @ %s" % (package, requirements or "*")) - - manifest = self.load_manifest(pkg_dir) - name = manifest["name"] - - click.echo( - "{} {:<40} @ {:<15}".format( - "Checking" if only_check else "Updating", - click.style(manifest["name"], fg="cyan"), - manifest["version"], - ), - nl=False, - ) - if not util.internet_on(): - click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) - return None - - latest = self.outdated(pkg_dir, requirements) - if latest: - click.echo("[%s]" % (click.style(latest, fg="red"))) - elif latest is False: - click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) - else: - click.echo("[%s]" % (click.style("Detached", fg="yellow"))) - - if only_check or not latest: - return True - - if "__src_url" in manifest: - vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"]) - assert vcs.update() - self._update_src_manifest( - dict(version=vcs.get_current_revision()), vcs.storage_dir - ) - else: - self.uninstall(pkg_dir, after_update=True) - self.install(name, latest, after_update=True) - - return True - - -class PackageManager(BasePkgManager): - @property - def manifest_names(self): - return ["package.json"] diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index c0e0f98e..9ef5e646 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -12,201 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-public-methods, too-many-instance-attributes - - -from os.path import isdir, isfile, join - -from platformio import app, exception, util -from platformio.managers.package import BasePkgManager, PackageManager +# Backward compatibility with legacy dev-platforms from platformio.platform.base import PlatformBase # pylint: disable=unused-import -from platformio.platform.exception import UnknownBoard, UnknownPlatform -from platformio.platform.factory import PlatformFactory -from platformio.project.config import ProjectConfig - - -class PlatformManager(BasePkgManager): - def __init__(self, package_dir=None, repositories=None): - if not repositories: - repositories = [ - "https://dl.bintray.com/platformio/dl-platforms/manifest.json", - "{0}://dl.platformio.org/platforms/manifest.json".format( - "https" if app.get_setting("strict_ssl") else "http" - ), - ] - self.config = ProjectConfig.get_instance() - BasePkgManager.__init__( - self, package_dir or self.config.get_optional_dir("platforms"), repositories - ) - - @property - def manifest_names(self): - return ["platform.json"] - - def get_manifest_path(self, pkg_dir): - if not isdir(pkg_dir): - return None - for name in self.manifest_names: - manifest_path = join(pkg_dir, name) - if isfile(manifest_path): - return manifest_path - return None - - def install( - self, - name, - requirements=None, - with_packages=None, - without_packages=None, - skip_default_package=False, - with_all_packages=False, - after_update=False, - silent=False, - force=False, - **_ - ): # pylint: disable=too-many-arguments, arguments-differ - platform_dir = BasePkgManager.install( - self, name, requirements, silent=silent, force=force - ) - p = PlatformFactory.new(platform_dir) - - if with_all_packages: - with_packages = list(p.packages.keys()) - - # don't cleanup packages or install them after update - # we check packages for updates in def update() - if after_update: - p.install_python_packages() - p.on_installed() - return True - - p.install_packages( - with_packages, - without_packages, - skip_default_package, - silent=silent, - force=force, - ) - p.install_python_packages() - p.on_installed() - return self.cleanup_packages(list(p.packages)) - - def uninstall(self, package, requirements=None, after_update=False): - if isdir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise UnknownPlatform(package) - - p = PlatformFactory.new(pkg_dir) - BasePkgManager.uninstall(self, pkg_dir, requirements) - p.uninstall_python_packages() - p.on_uninstalled() - - # don't cleanup packages or install them after update - # we check packages for updates in def update() - if after_update: - return True - - return self.cleanup_packages(list(p.packages)) - - def update( # pylint: disable=arguments-differ - self, package, requirements=None, only_check=False, only_packages=False - ): - if isdir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise UnknownPlatform(package) - - p = PlatformFactory.new(pkg_dir) - pkgs_before = list(p.get_installed_packages()) - - missed_pkgs = set() - if not only_packages: - BasePkgManager.update(self, pkg_dir, requirements, only_check) - p = PlatformFactory.new(pkg_dir) - missed_pkgs = set(pkgs_before) & set(p.packages) - missed_pkgs -= set(p.get_installed_packages()) - - p.update_packages(only_check) - self.cleanup_packages(list(p.packages)) - - if missed_pkgs: - p.install_packages( - with_packages=list(missed_pkgs), skip_default_package=True - ) - - return True - - def cleanup_packages(self, names): - self.cache_reset() - deppkgs = {} - for manifest in PlatformManager().get_installed(): - p = PlatformFactory.new(manifest["__pkg_dir"]) - for pkgname, pkgmanifest in p.get_installed_packages().items(): - if pkgname not in deppkgs: - deppkgs[pkgname] = set() - deppkgs[pkgname].add(pkgmanifest["version"]) - - pm = PackageManager(self.config.get_optional_dir("packages")) - for manifest in pm.get_installed(): - if manifest["name"] not in names: - continue - if ( - manifest["name"] not in deppkgs - or manifest["version"] not in deppkgs[manifest["name"]] - ): - try: - pm.uninstall(manifest["__pkg_dir"], after_update=True) - except exception.UnknownPackage: - pass - - self.cache_reset() - return True - - @util.memoized(expire="5s") - def get_installed_boards(self): - boards = [] - for manifest in self.get_installed(): - p = PlatformFactory.new(manifest["__pkg_dir"]) - for config in p.get_boards().values(): - board = config.get_brief_data() - if board not in boards: - boards.append(board) - return boards - - @staticmethod - def get_registered_boards(): - return util.get_api_result("/boards", cache_valid="7d") - - def get_all_boards(self): - boards = self.get_installed_boards() - know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] - try: - for board in self.get_registered_boards(): - key = "%s:%s" % (board["platform"], board["id"]) - if key not in know_boards: - boards.append(board) - except (exception.APIRequestError, exception.InternetIsOffline): - pass - return sorted(boards, key=lambda b: b["name"]) - - def board_config(self, id_, platform=None): - for manifest in self.get_installed_boards(): - if manifest["id"] == id_ and ( - not platform or manifest["platform"] == platform - ): - return manifest - for manifest in self.get_registered_boards(): - if manifest["id"] == id_ and ( - not platform or manifest["platform"] == platform - ): - return manifest - raise UnknownBoard(id_) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 04a41f26..003c2386 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -42,17 +42,26 @@ class PackageManagerInstallMixin(object): with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=False) - def install(self, spec, silent=False, force=False): + def install(self, spec, silent=False, skip_dependencies=False, force=False): try: self.lock() - pkg = self._install(spec, silent=silent, force=force) + pkg = self._install( + spec, silent=silent, skip_dependencies=skip_dependencies, force=force + ) self.memcache_reset() self.cleanup_expired_downloads() return pkg finally: self.unlock() - def _install(self, spec, search_filters=None, silent=False, force=False): + def _install( # pylint: disable=too-many-arguments + self, + spec, + search_filters=None, + silent=False, + skip_dependencies=False, + force=False, + ): spec = self.ensure_spec(spec) # avoid circle dependencies @@ -104,11 +113,12 @@ class PackageManagerInstallMixin(object): ) self.memcache_reset() - self._install_dependencies(pkg, silent) + if not skip_dependencies: + self.install_dependencies(pkg, silent) self._INSTALL_HISTORY[spec] = pkg return pkg - def _install_dependencies(self, pkg, silent=False): + def install_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 7dc09964..72f189fb 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -91,6 +91,9 @@ class PackageManageRegistryMixin(object): self.print_multi_package_issue(packages, spec) package, version = self.find_best_registry_version(packages, spec) + if not package or not version: + raise UnknownPackageError(spec.humanize()) + pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None if not pkgfile: raise UnknownPackageError(spec.humanize()) @@ -189,7 +192,7 @@ class PackageManageRegistryMixin(object): return (package, version) if not spec.requirements: - return None + return (None, None) # if the custom version requirements, check ALL package versions for package in packages: @@ -206,7 +209,7 @@ class PackageManageRegistryMixin(object): if version: return (package, version) time.sleep(1) - return None + return (None, None) def pick_best_registry_version(self, versions, spec=None): assert not spec or isinstance(spec, PackageSpec) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index e2656401..2cca8505 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -44,7 +44,7 @@ class PackageManagerUninstallMixin(object): # firstly, remove dependencies if not skip_dependencies: - self._uninstall_dependencies(pkg, silent) + self.uninstall_dependencies(pkg, silent) if os.path.islink(pkg.path): os.unlink(pkg.path) @@ -72,7 +72,7 @@ class PackageManagerUninstallMixin(object): return pkg - def _uninstall_dependencies(self, pkg, silent=False): + def uninstall_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index b0b976de..d3e8dbb1 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -74,17 +74,24 @@ class PackageManagerUpdateMixin(object): ).version ) - def update(self, from_spec, to_spec=None, only_check=False, silent=False): + def update( # pylint: disable=too-many-arguments + self, + from_spec, + to_spec=None, + only_check=False, + silent=False, + show_incompatible=True, + ): pkg = self.get_package(from_spec) if not pkg or not pkg.metadata: raise UnknownPackageError(from_spec) if not silent: click.echo( - "{} {:<45} {:<30}".format( + "{} {:<45} {:<35}".format( "Checking" if only_check else "Updating", click.style(pkg.metadata.spec.humanize(), fg="cyan"), - "%s (%s)" % (pkg.metadata.version, to_spec.requirements) + "%s @ %s" % (pkg.metadata.version, to_spec.requirements) if to_spec and to_spec.requirements else str(pkg.metadata.version), ), @@ -97,17 +104,9 @@ class PackageManagerUpdateMixin(object): outdated = self.outdated(pkg, to_spec) if not silent: - self.print_outdated_state(outdated) + self.print_outdated_state(outdated, show_incompatible) - up_to_date = any( - [ - outdated.detached, - not outdated.latest, - outdated.latest and outdated.current == outdated.latest, - outdated.wanted and outdated.current == outdated.wanted, - ] - ) - if only_check or up_to_date: + if only_check or not outdated.is_outdated(allow_incompatible=False): return pkg try: @@ -117,18 +116,26 @@ class PackageManagerUpdateMixin(object): self.unlock() @staticmethod - def print_outdated_state(outdated): + def print_outdated_state(outdated, show_incompatible=True): if outdated.detached: return click.echo("[%s]" % (click.style("Detached", fg="yellow"))) - if not outdated.latest or outdated.current == outdated.latest: + if ( + not outdated.latest + or outdated.current == outdated.latest + or (not show_incompatible and outdated.current == outdated.wanted) + ): return click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) if outdated.wanted and outdated.current == outdated.wanted: return click.echo( - "[%s]" - % (click.style("Incompatible (%s)" % outdated.latest, fg="yellow")) + "[%s]" % (click.style("Incompatible %s" % outdated.latest, fg="yellow")) ) return click.echo( - "[%s]" % (click.style(str(outdated.wanted or outdated.latest), fg="red")) + "[%s]" + % ( + click.style( + "Outdated %s" % str(outdated.wanted or outdated.latest), fg="red" + ) + ) ) def _update(self, pkg, outdated, silent=False): diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index dc024edc..b307753c 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -190,6 +190,10 @@ class BasePackageManager( # pylint: disable=too-many-public-methods return metadata def get_installed(self): + cache_key = "get_installed" + if self.memcache_get(cache_key): + return self.memcache_get(cache_key) + result = [] for name in sorted(os.listdir(self.package_dir)): pkg_dir = os.path.join(self.package_dir, name) @@ -213,6 +217,8 @@ class BasePackageManager( # pylint: disable=too-many-public-methods except MissingPackageManifestError: pass result.append(pkg) + + self.memcache_set(cache_key, result) return result def get_package(self, spec): diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index c79e7d10..91eabf6a 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -12,8 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from platformio import util +from platformio.exception import APIRequestError, InternetIsOffline +from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageType +from platformio.platform.exception import IncompatiblePlatform, UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -28,3 +34,161 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an @property def manifest_names(self): return PackageType.get_manifest_map()[PackageType.PLATFORM] + + def install( # pylint: disable=arguments-differ, too-many-arguments + self, + spec, + with_packages=None, + without_packages=None, + skip_default_package=False, + with_all_packages=False, + silent=False, + force=False, + ): + pkg = super(PlatformPackageManager, self).install( + spec, silent=silent, force=force, skip_dependencies=True + ) + try: + p = PlatformFactory.new(pkg) + p.ensure_engine_compatible() + except IncompatiblePlatform as e: + super(PlatformPackageManager, self).uninstall( + pkg, silent=silent, skip_dependencies=True + ) + raise e + + if with_all_packages: + with_packages = list(p.packages) + + p.install_packages( + with_packages, + without_packages, + skip_default_package, + silent=silent, + force=force, + ) + p.install_python_packages() + p.on_installed() + self.cleanup_packages(list(p.packages)) + return pkg + + def uninstall(self, spec, silent=False, skip_dependencies=False): + pkg = self.get_package(spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(spec) + p = PlatformFactory.new(pkg) + assert super(PlatformPackageManager, self).uninstall( + pkg, silent=silent, skip_dependencies=True + ) + if not skip_dependencies: + p.uninstall_python_packages() + p.on_uninstalled() + self.cleanup_packages(list(p.packages)) + return pkg + + def update( # pylint: disable=arguments-differ, too-many-arguments + self, + from_spec, + to_spec=None, + only_check=False, + silent=False, + show_incompatible=True, + only_packages=False, + ): + pkg = self.get_package(from_spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(from_spec) + p = PlatformFactory.new(pkg) + pkgs_before = [item.metadata.name for item in p.get_installed_packages()] + + new_pkg = None + missed_pkgs = set() + if not only_packages: + new_pkg = super(PlatformPackageManager, self).update( + from_spec, + to_spec, + only_check=only_check, + silent=silent, + show_incompatible=show_incompatible, + ) + p = PlatformFactory.new(new_pkg) + missed_pkgs = set(pkgs_before) & set(p.packages) + missed_pkgs -= set( + item.metadata.name for item in p.get_installed_packages() + ) + + p.update_packages(only_check) + self.cleanup_packages(list(p.packages)) + + if missed_pkgs: + p.install_packages( + with_packages=list(missed_pkgs), skip_default_package=True + ) + + return new_pkg or pkg + + def cleanup_packages(self, names): + self.memcache_reset() + deppkgs = {} + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for pkg in p.get_installed_packages(): + if pkg.metadata.name not in deppkgs: + deppkgs[pkg.metadata.name] = set() + deppkgs[pkg.metadata.name].add(pkg.metadata.version) + + pm = ToolPackageManager() + for pkg in pm.get_installed(): + if pkg.metadata.name not in names: + continue + if ( + pkg.metadata.name not in deppkgs + or pkg.metadata.version not in deppkgs[pkg.metadata.name] + ): + try: + pm.uninstall(pkg.metadata.spec) + except UnknownPackageError: + pass + + self.memcache_reset() + return True + + @util.memoized(expire="5s") + def get_installed_boards(self): + boards = [] + for pkg in self.get_installed(): + p = PlatformFactory.new(pkg) + for config in p.get_boards().values(): + board = config.get_brief_data() + if board not in boards: + boards.append(board) + return boards + + @staticmethod + def get_registered_boards(): + return util.get_api_result("/boards", cache_valid="7d") + + def get_all_boards(self): + boards = self.get_installed_boards() + know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] + try: + for board in self.get_registered_boards(): + key = "%s:%s" % (board["platform"], board["id"]) + if key not in know_boards: + boards.append(board) + except (APIRequestError, InternetIsOffline): + pass + return sorted(boards, key=lambda b: b["name"]) + + def board_config(self, id_, platform=None): + for manifest in self.get_installed_boards(): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): + return manifest + for manifest in self.get_registered_boards(): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): + return manifest + raise UnknownBoard(id_) diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py index ae111798..60aededd 100644 --- a/platformio/package/manager/tool.py +++ b/platformio/package/manager/tool.py @@ -19,10 +19,9 @@ from platformio.project.config import ProjectConfig class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): - self.config = ProjectConfig.get_instance() - super(ToolPackageManager, self).__init__( - PackageType.TOOL, package_dir or self.config.get_optional_dir("packages"), - ) + if not package_dir: + package_dir = ProjectConfig.get_instance().get_optional_dir("packages") + super(ToolPackageManager, self).__init__(PackageType.TOOL, package_dir) @property def manifest_names(self): diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index e626eb4b..ac495b48 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -13,9 +13,62 @@ # limitations under the License. from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSpec class PlatformPackagesMixin(object): + def get_package_spec(self, name): + version = self.packages[name].get("version", "") + if any(c in version for c in (":", "/", "@")): + return PackageSpec("%s=%s" % (name, version)) + return PackageSpec( + owner=self.packages[name].get("owner"), name=name, requirements=version + ) + + def get_package(self, name): + if not name: + return None + return self.pm.get_package(self.get_package_spec(name)) + + def get_package_dir(self, name): + pkg = self.get_package(name) + return pkg.path if pkg else None + + def get_package_version(self, name): + pkg = self.get_package(name) + return str(pkg.metadata.version) if pkg else None + + def get_installed_packages(self): + result = [] + for name in self.packages: + pkg = self.get_package(name) + if pkg: + result.append(pkg) + return result + + def dump_used_packages(self): + result = [] + for name, options in self.packages.items(): + if options.get("optional"): + continue + pkg = self.get_package(name) + if not pkg or not pkg.metadata: + continue + item = {"name": pkg.metadata.name, "version": str(pkg.metadata.version)} + if pkg.metadata.spec.external: + item["src_url"] = pkg.metadata.spec.url + result.append(item) + return result + + def autoinstall_runtime_packages(self): + for name, options in self.packages.items(): + if options.get("optional", False): + continue + if self.get_package(name): + continue + self.pm.install(self.get_package_spec(name)) + return True + def install_packages( # pylint: disable=too-many-arguments self, with_packages=None, @@ -24,31 +77,25 @@ class PlatformPackagesMixin(object): silent=False, force=False, ): - with_packages = set(self.find_pkg_names(with_packages or [])) - without_packages = set(self.find_pkg_names(without_packages or [])) + with_packages = set(self._find_pkg_names(with_packages or [])) + without_packages = set(self._find_pkg_names(without_packages or [])) upkgs = with_packages | without_packages ppkgs = set(self.packages) if not upkgs.issubset(ppkgs): raise UnknownPackageError(", ".join(upkgs - ppkgs)) - for name, opts in self.packages.items(): - version = opts.get("version", "") + for name, options in self.packages.items(): if name in without_packages: continue if name in with_packages or not ( - skip_default_package or opts.get("optional", False) + skip_default_package or options.get("optional", False) ): - if ":" in version: - self.pm.install( - "%s=%s" % (name, version), silent=silent, force=force - ) - else: - self.pm.install(name, version, silent=silent, force=force) + self.pm.install(self.get_package_spec(name), silent=silent, force=force) return True - def find_pkg_names(self, candidates): + def _find_pkg_names(self, candidates): result = [] for candidate in candidates: found = False @@ -73,54 +120,18 @@ class PlatformPackagesMixin(object): return result def update_packages(self, only_check=False): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - self.pm.update(manifest["__pkg_dir"], requirements, only_check) - - def get_installed_packages(self): - items = {} - for name in self.packages: - pkg_dir = self.get_package_dir(name) - if pkg_dir: - items[name] = self.pm.load_manifest(pkg_dir) - return items + for pkg in self.get_installed_packages(): + self.pm.update( + pkg, + to_spec=self.get_package_spec(pkg.metadata.name), + only_check=only_check, + show_incompatible=False, + ) def are_outdated_packages(self): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - if self.pm.outdated(manifest["__pkg_dir"], requirements): + for pkg in self.get_installed_packages(): + if self.pm.outdated( + pkg, self.get_package_spec(pkg.metadata.name) + ).is_outdated(allow_incompatible=False): return True return False - - def get_package_dir(self, name): - version = self.packages[name].get("version", "") - if ":" in version: - return self.pm.get_package_dir( - *self.pm.parse_pkg_uri("%s=%s" % (name, version)) - ) - return self.pm.get_package_dir(name, version) - - def get_package_version(self, name): - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - return None - return self.pm.load_manifest(pkg_dir).get("version") - - def dump_used_packages(self): - result = [] - for name, options in self.packages.items(): - if options.get("optional"): - continue - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - continue - manifest = self.pm.load_manifest(pkg_dir) - item = {"name": manifest["name"], "version": manifest["version"]} - if manifest.get("__src_url"): - item["src_url"] = manifest.get("__src_url") - result.append(item) - return result diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 39e30fce..38cf232c 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -50,12 +50,14 @@ class PlatformRunMixin(object): assert isinstance(variables, dict) assert isinstance(targets, list) + self.ensure_engine_compatible() + options = self.config.items(env=variables["pioenv"], as_dict=True) if "framework" in options: # support PIO Core 3.0 dev/platforms options["pioframework"] = options["framework"] self.configure_default_packages(options, targets) - self.install_packages(silent=True) + self.autoinstall_runtime_packages() self._report_non_sensitive_data(options, targets) @@ -84,8 +86,6 @@ class PlatformRunMixin(object): for item in self.dump_used_packages() ] topts["platform"] = {"name": self.name, "version": self.version} - if self.src_version: - topts["platform"]["src_version"] = self.src_version telemetry.send_run_environment(topts, targets) def _run_scons(self, variables, targets, jobs): diff --git a/platformio/platform/base.py b/platformio/platform/base.py index a2fcd158..38602950 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -19,11 +19,11 @@ import click import semantic_version from platformio import __version__, fs, proc, util -from platformio.managers.package import PackageManager +from platformio.package.manager.tool import ToolPackageManager from platformio.platform._packages import PlatformPackagesMixin from platformio.platform._run import PlatformRunMixin from platformio.platform.board import PlatformBoardConfig -from platformio.platform.exception import UnknownBoard +from platformio.platform.exception import IncompatiblePlatform, UnknownBoard from platformio.project.config import ProjectConfig @@ -44,20 +44,7 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub self._custom_packages = None self.config = ProjectConfig.get_instance() - self.pm = PackageManager( - self.config.get_optional_dir("packages"), self.package_repositories - ) - - self._src_manifest = None - src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) - if src_manifest_path: - self._src_manifest = fs.load_json(src_manifest_path) - - # if self.engines and "platformio" in self.engines: - # if self.PIO_VERSION not in semantic_version.SimpleSpec( - # self.engines['platformio']): - # raise exception.IncompatiblePlatform(self.name, - # str(self.PIO_VERSION)) + self.pm = ToolPackageManager(self.config.get_optional_dir("packages")) @property def name(self): @@ -75,14 +62,6 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub def version(self): return self._manifest["version"] - @property - def src_version(self): - return self._src_manifest.get("version") if self._src_manifest else None - - @property - def src_url(self): - return self._src_manifest.get("url") if self._src_manifest else None - @property def homepage(self): return self._manifest.get("homepage") @@ -103,10 +82,6 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub def engines(self): return self._manifest.get("engines") - @property - def package_repositories(self): - return self._manifest.get("packageRepositories") - @property def manifest(self): return self._manifest @@ -114,21 +89,32 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub @property def packages(self): packages = self._manifest.get("packages", {}) - for item in self._custom_packages or []: - name = item - version = "*" - if "@" in item: - name, version = item.split("@", 2) - name = name.strip() - if name not in packages: - packages[name] = {} - packages[name].update({"version": version.strip(), "optional": False}) + for spec in self._custom_packages or []: + spec = self.pm.ensure_spec(spec) + if spec.external: + version = spec.url + else: + version = str(spec.requirements) or "*" + if spec.name not in packages: + packages[spec.name] = {} + packages[spec.name].update( + {"owner": spec.owner, "version": version, "optional": False} + ) return packages @property def python_packages(self): return self._manifest.get("pythonPackages") + def ensure_engine_compatible(self): + if not self.engines or "platformio" not in self.engines: + return True + if self.PIO_VERSION in semantic_version.SimpleSpec(self.engines["platformio"]): + return True + raise IncompatiblePlatform( + self.name, str(self.PIO_VERSION), self.engines["platformio"] + ) + def get_dir(self): return os.path.dirname(self.manifest_path) @@ -218,10 +204,10 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub for opts in (self.frameworks or {}).values(): if "package" not in opts: continue - pkg_dir = self.get_package_dir(opts["package"]) - if not pkg_dir or not os.path.isdir(os.path.join(pkg_dir, "libraries")): + pkg = self.get_package(opts["package"]) + if not pkg or not os.path.isdir(os.path.join(pkg.path, "libraries")): continue - libs_dir = os.path.join(pkg_dir, "libraries") + libs_dir = os.path.join(pkg.path, "libraries") storages[libs_dir] = opts["package"] libcores_dir = os.path.join(libs_dir, "__cores__") if not os.path.isdir(libcores_dir): diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py index 40431d7f..604c3228 100644 --- a/platformio/platform/exception.py +++ b/platformio/platform/exception.py @@ -26,7 +26,10 @@ class UnknownPlatform(PlatformException): class IncompatiblePlatform(PlatformException): - MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" + MESSAGE = ( + "Development platform '{0}' is not compatible with PlatformIO Core v{1} and " + "depends on PlatformIO Core {2}.\n" + ) class UnknownBoard(PlatformException): diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index 99e5f7c4..e48142ce 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -16,7 +16,7 @@ import os import re from platformio.compat import load_python_module -from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageItem from platformio.platform.base import PlatformBase from platformio.platform.exception import UnknownPlatform @@ -36,9 +36,16 @@ class PlatformFactory(object): @classmethod def new(cls, pkg_or_spec): - pkg = PlatformPackageManager().get_package( - "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec - ) + if isinstance(pkg_or_spec, PackageItem): + pkg = pkg_or_spec + else: + from platformio.package.manager.platform import ( # pylint: disable=import-outside-toplevel + PlatformPackageManager, + ) + + pkg = PlatformPackageManager().get_package( + "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec + ) if not pkg: raise UnknownPlatform(pkg_or_spec) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 377c1e28..74bb6f45 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -14,8 +14,9 @@ import json -from platformio import exception from platformio.commands import platform as cli_platform +from platformio.package.exception import UnknownPackageError +from platformio.platform.exception import IncompatiblePlatform def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): @@ -39,22 +40,30 @@ def test_search_raw_output(clirunner, validate_cliresult): def test_install_unknown_version(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) assert result.exit_code != 0 - assert isinstance(result.exception, exception.UndefinedPackageVersion) + assert isinstance(result.exception, UnknownPackageError) def test_install_unknown_from_registry(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) + assert isinstance(result.exception, UnknownPackageError) + + +def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], + ) + assert result.exit_code != 0 + assert isinstance(result.exception, IncompatiblePlatform) def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, - ["atmelavr@1.2.0", "--skip-default-package", "--with-package", "tool-avrdude"], + ["atmelavr@2.0.0", "--skip-default-package", "--with-package", "tool-avrdude"], ) validate_cliresult(result) - assert "atmelavr @ 1.2.0" in result.output + assert "atmelavr @ 2.0.0" in result.output assert "Installing tool-avrdude @" in result.output assert len(isolated_pio_core.join("packages").listdir()) == 1 @@ -63,7 +72,7 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, [ - "https://github.com/platformio/" "platform-espressif8266.git", + "https://github.com/platformio/platform-espressif8266.git", "--skip-default-package", ], ) @@ -90,7 +99,7 @@ def test_list_raw_output(clirunner, validate_cliresult): def test_update_check(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cli_platform.platform_update, ["--only-check", "--json-output"] + cli_platform.platform_update, ["--dry-run", "--json-output"] ) validate_cliresult(result) output = json.loads(result.output) @@ -102,9 +111,9 @@ def test_update_check(clirunner, validate_cliresult, isolated_pio_core): def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cli_platform.platform_update) validate_cliresult(result) - assert "Uninstalling atmelavr @ 1.2.0:" in result.output - assert "PlatformManager: Installing atmelavr @" in result.output - assert len(isolated_pio_core.join("packages").listdir()) == 1 + assert "Removing atmelavr @ 2.0.0:" in result.output + assert "Platform Manager: Installing platformio/atmelavr @" in result.output + assert len(isolated_pio_core.join("packages").listdir()) == 2 def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 07fbabf8..4150bcdd 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -13,13 +13,13 @@ # limitations under the License. import json +import os import re from time import time from platformio import app, maintenance from platformio.__main__ import cli as cli_pio from platformio.commands import upgrade as cmd_upgrade -from platformio.managers.platform import PlatformManager def test_check_pio_upgrade(clirunner, isolated_pio_core, validate_cliresult): @@ -89,7 +89,8 @@ def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_clire assert "There are the new updates for libraries (ArduinoJson)" in result.output assert "Please wait while updating libraries" in result.output assert re.search( - r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[[\d\.]+\]", result.output + r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[Outdated [\d\.]+\]", + result.output, ) # check updated version @@ -102,12 +103,11 @@ def test_check_platform_updates(clirunner, isolated_pio_core, validate_cliresult # install obsolete platform result = clirunner.invoke(cli_pio, ["platform", "install", "native"]) validate_cliresult(result) + os.remove(str(isolated_pio_core.join("platforms", "native", ".piopm"))) manifest_path = isolated_pio_core.join("platforms", "native", "platform.json") manifest = json.loads(manifest_path.read()) manifest["version"] = "0.0.0" manifest_path.write(json.dumps(manifest)) - # reset cached manifests - PlatformManager().cache_reset() # reset check time interval = int(app.get_setting("check_platforms_interval")) * 3600 * 24 @@ -141,7 +141,7 @@ def test_check_and_update_platforms(clirunner, isolated_pio_core, validate_clire validate_cliresult(result) assert "There are the new updates for platforms (native)" in result.output assert "Please wait while updating platforms" in result.output - assert re.search(r"Updating native\s+@ 0.0.0\s+\[[\d\.]+\]", result.output) + assert re.search(r"Updating native\s+0.0.0\s+\[Outdated [\d\.]+\]", result.output) # check updated version result = clirunner.invoke(cli_pio, ["platform", "list", "--json-output"]) From 67e6d177b443c189660f8d21efed05a3ceaacf80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 16 Aug 2020 18:48:05 +0300 Subject: [PATCH 158/223] Minor fixes for dev-platform factory --- platformio/builder/tools/pioplatform.py | 11 ++++++-- platformio/platform/factory.py | 36 ++++++++++++++++++------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 740f6148..5ca7794f 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -149,9 +149,16 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements def _get_plaform_data(): data = [ "PLATFORM: %s (%s)" - % (platform.title, pkg_metadata.version or platform.version) + % ( + platform.title, + pkg_metadata.version if pkg_metadata else platform.version, + ) ] - if int(ARGUMENTS.get("PIOVERBOSE", 0)) and pkg_metadata.spec.external: + if ( + int(ARGUMENTS.get("PIOVERBOSE", 0)) + and pkg_metadata + and pkg_metadata.spec.external + ): data.append("(%s)" % pkg_metadata.spec.url) if board_config: data.extend([">", board_config.get("name")]) diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index e48142ce..0f2bd15f 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -15,6 +15,7 @@ import os import re +from platformio import fs from platformio.compat import load_python_module from platformio.package.meta import PackageItem from platformio.platform.base import PlatformBase @@ -36,32 +37,47 @@ class PlatformFactory(object): @classmethod def new(cls, pkg_or_spec): + platform_dir = None + platform_name = None if isinstance(pkg_or_spec, PackageItem): - pkg = pkg_or_spec + platform_dir = pkg_or_spec.path + platform_name = pkg_or_spec.metadata.name + elif os.path.isdir(pkg_or_spec): + platform_dir = pkg_or_spec else: from platformio.package.manager.platform import ( # pylint: disable=import-outside-toplevel PlatformPackageManager, ) - pkg = PlatformPackageManager().get_package( - "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec - ) - if not pkg: + pkg = PlatformPackageManager().get_package(pkg_or_spec) + if not pkg: + raise UnknownPlatform(pkg_or_spec) + platform_dir = pkg.path + platform_name = pkg.metadata.name + + if not platform_dir or not os.path.isfile( + os.path.join(platform_dir, "platform.json") + ): raise UnknownPlatform(pkg_or_spec) + if not platform_name: + platform_name = fs.load_json(os.path.join(platform_dir, "platform.json"))[ + "name" + ] + platform_cls = None - if os.path.isfile(os.path.join(pkg.path, "platform.py")): + if os.path.isfile(os.path.join(platform_dir, "platform.py")): platform_cls = getattr( cls.load_module( - pkg.metadata.name, os.path.join(pkg.path, "platform.py") + platform_name, os.path.join(platform_dir, "platform.py") ), - cls.get_clsname(pkg.metadata.name), + cls.get_clsname(platform_name), ) else: platform_cls = type( - str(cls.get_clsname(pkg.metadata.name)), (PlatformBase,), {} + str(cls.get_clsname(platform_name)), (PlatformBase,), {} ) - _instance = platform_cls(os.path.join(pkg.path, "platform.json")) + _instance = platform_cls(os.path.join(platform_dir, "platform.json")) assert isinstance(_instance, PlatformBase) return _instance From 808852f4cc37ac97f031f850c995c7690015ca71 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 16 Aug 2020 20:21:30 +0300 Subject: [PATCH 159/223] Set default timeout for http requests // Resolve #3623 --- platformio/__init__.py | 2 ++ platformio/app.py | 13 ++++------- platformio/clients/http.py | 7 +++++- platformio/commands/home/rpc/handlers/os.py | 10 +++++--- platformio/commands/upgrade.py | 20 ++++++---------- platformio/package/download.py | 5 ++-- platformio/package/manifest/parser.py | 7 ++---- platformio/package/manifest/schema.py | 11 +++++---- platformio/util.py | 26 +++++++++++++++++++-- 9 files changed, 62 insertions(+), 39 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index ee6d438d..6ef8af69 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,6 +14,8 @@ import sys +DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) + VERSION = (4, 4, "0a8") __version__ = ".".join([str(s) for s in VERSION]) diff --git a/platformio/app.py b/platformio/app.py index 00a8e89f..21adba1c 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import codecs import getpass import hashlib +import json import os import platform import socket @@ -25,9 +26,7 @@ from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath from time import time -import requests - -from platformio import __version__, exception, fs, proc +from platformio import __version__, exception, fs, proc, util from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import ( @@ -403,16 +402,14 @@ def get_cid(): uid = getenv("C9_UID") elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): try: - uid = ( - requests.get( + uid = json.loads( + util.fetch_remote_content( "{api}/user?token={token}".format( api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), token=getenv("USER_TOKEN"), ) ) - .json() - .get("id") - ) + ).get("id") except: # pylint: disable=bare-except pass if not uid: diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 974017b7..e18d2eed 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -15,7 +15,7 @@ import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import app, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util from platformio.exception import PlatformioException @@ -58,6 +58,11 @@ class HTTPClient(object): # check Internet before and resolve issue with 60 seconds timeout # print(self, method, path, kwargs) util.internet_on(raise_exception=True) + + # set default timeout + if "timeout" not in kwargs: + kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + try: return getattr(self._session, method)(self.base_url + path, **kwargs) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 3b4bd4e1..2b1662f2 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,7 @@ from functools import cmp_to_key import click from twisted.internet import defer # pylint: disable=import-error -from platformio import app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, glob_recursive @@ -51,9 +51,13 @@ class OSRPC(object): session = helpers.requests_session() if data: - r = yield session.post(uri, data=data, headers=headers) + r = yield session.post( + uri, data=data, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + ) else: - r = yield session.get(uri, headers=headers) + r = yield session.get( + uri, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + ) r.raise_for_status() result = r.text diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 6303ea69..c8c8b9fe 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re from zipfile import ZipFile import click -import requests -from platformio import VERSION, __version__, app, exception +from platformio import VERSION, __version__, app, exception, util from platformio.compat import WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -130,13 +130,11 @@ def get_latest_version(): def get_develop_latest_version(): version = None - r = requests.get( + content = util.fetch_remote_content( "https://raw.githubusercontent.com/platformio/platformio" - "/develop/platformio/__init__.py", - headers={"User-Agent": app.get_user_agent()}, + "/develop/platformio/__init__.py" ) - r.raise_for_status() - for line in r.text.split("\n"): + for line in content.split("\n"): line = line.strip() if not line.startswith("VERSION"): continue @@ -152,9 +150,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - r = requests.get( - "https://pypi.org/pypi/platformio/json", - headers={"User-Agent": app.get_user_agent()}, - ) - r.raise_for_status() - return r.json()["info"]["version"] + content = util.fetch_remote_content("https://pypi.org/pypi/platformio/json") + return json.loads(content)["info"]["version"] diff --git a/platformio/package/download.py b/platformio/package/download.py index 3c723c4b..7f29e7ac 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -14,7 +14,6 @@ import io import math -import sys from email.utils import parsedate_tz from os.path import getsize, join from time import mktime @@ -22,7 +21,7 @@ from time import mktime import click import requests -from platformio import app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util from platformio.package.exception import PackageException @@ -34,7 +33,7 @@ class FileDownloader(object): url, stream=True, headers={"User-Agent": app.get_user_agent()}, - verify=sys.version_info >= (2, 7, 9), + timeout=DEFAULT_REQUESTS_TIMEOUT, ) if self._request.status_code != 200: raise PackageException( diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index b4a93d98..d453c83e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -19,8 +19,6 @@ import os import re import tarfile -import requests - from platformio import util from platformio.compat import get_object_members, string_types from platformio.package.exception import ManifestParserError, UnknownManifestError @@ -108,10 +106,9 @@ class ManifestParserFactory(object): @staticmethod def new_from_url(remote_url): - r = requests.get(remote_url) - r.raise_for_status() + content = util.fetch_remote_content(remote_url) return ManifestParserFactory.new( - r.text, + content, ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, remote_url, ) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 8befab52..7dafaa23 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -14,11 +14,14 @@ # pylint: disable=too-many-ancestors +import json + import marshmallow import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates +from platformio import util from platformio.package.exception import ManifestValidationError from platformio.util import memoized @@ -248,9 +251,9 @@ class ManifestSchema(BaseSchema): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - r = requests.get( + version = "3.10" + spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.10/json/licenses.json" + "/v%s/json/licenses.json" % version ) - r.raise_for_status() - return r.json() + return json.loads(util.fetch_remote_content(spdx_data_url)) diff --git a/platformio/util.py b/platformio/util.py index a1974f17..982b0bab 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -29,7 +29,7 @@ from glob import glob import click import requests -from platformio import __apiurl__, __version__, exception +from platformio import DEFAULT_REQUESTS_TIMEOUT, __apiurl__, __version__, exception from platformio.commands import PlatformioCLI from platformio.compat import PY2, WINDOWS from platformio.fs import cd # pylint: disable=unused-import @@ -303,10 +303,16 @@ def _get_api_result( headers=headers, auth=auth, verify=verify_ssl, + timeout=DEFAULT_REQUESTS_TIMEOUT, ) else: r = _api_request_session().get( - url, params=params, headers=headers, auth=auth, verify=verify_ssl + url, + params=params, + headers=headers, + auth=auth, + verify=verify_ssl, + timeout=DEFAULT_REQUESTS_TIMEOUT, ) result = r.json() r.raise_for_status() @@ -398,6 +404,22 @@ def internet_on(raise_exception=False): return result +def fetch_remote_content(*args, **kwargs): + # pylint: disable=import-outside-toplevel + from platformio.app import get_user_agent + + kwargs["headers"] = kwargs.get("headers", {}) + if "User-Agent" not in kwargs["headers"]: + kwargs["headers"]["User-Agent"] = get_user_agent() + + if "timeout" not in kwargs: + kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + + r = requests.get(*args, **kwargs) + r.raise_for_status() + return r.text + + def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) From 74e27a2edc94e9531eaf81bb7047199f6a7a9296 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 16 Aug 2020 20:26:59 +0300 Subject: [PATCH 160/223] Enable "cyclic reference" for GCC linker only for the embedded dev-platforms // Resolve #3570 --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 147c4b5f..7f86e40f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -54,6 +54,7 @@ PlatformIO Core 4 * Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command * Do not generate ".travis.yml" for a new project, let the user have a choice * Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) +* Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) * Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 * Do not escape compiler arguments in VSCode template on Windows * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 560cbe37..5d8f8e8b 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -66,7 +66,11 @@ def BuildProgram(env): env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) # enable "cyclic reference" for linker - if env.get("LIBS") and env.GetCompilerType() == "gcc": + if ( + env.get("LIBS") + and env.GetCompilerType() == "gcc" + and env.PioPlatform().is_embedded() + ): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") From 2459e85c1dc9a1ecfe632c611594c6c302c8af6b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Aug 2020 12:13:25 +0300 Subject: [PATCH 161/223] Fix a bug with the custom platform packages // Resolve #3628 --- platformio/platform/base.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 38602950..a5f3dd01 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -89,17 +89,20 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub @property def packages(self): packages = self._manifest.get("packages", {}) - for spec in self._custom_packages or []: - spec = self.pm.ensure_spec(spec) - if spec.external: - version = spec.url - else: - version = str(spec.requirements) or "*" + for item in self._custom_packages or []: + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 2) + spec = self.pm.ensure_spec(name) + options = {"version": version.strip(), "optional": False} + if spec.owner: + options["owner"] = spec.owner if spec.name not in packages: packages[spec.name] = {} - packages[spec.name].update( - {"owner": spec.owner, "version": version, "optional": False} - ) + packages[spec.name].update(**options) + + print(13, packages) return packages @property From 6f7fc638c7dad4315a318ea00b357916558ca6a3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Aug 2020 12:56:57 +0300 Subject: [PATCH 162/223] Fix PyLint errors in tests --- .pylintrc | 3 +++ Makefile | 3 ++- platformio/platform/base.py | 2 -- tests/__init__.py | 13 +++++++++++++ tests/commands/__init__.py | 13 +++++++++++++ tests/commands/test_account_org_team.py | 5 ++++- tests/commands/test_boards.py | 2 +- tests/commands/test_check.py | 2 ++ tests/commands/test_lib.py | 2 ++ tests/commands/test_lib_complex.py | 8 ++++++-- tests/commands/test_platform.py | 2 ++ tests/commands/test_test.py | 1 + tests/commands/test_update.py | 2 ++ tests/ino2cpp/__init__.py | 13 +++++++++++++ tests/package/__init__.py | 13 +++++++++++++ tests/package/test_manager.py | 2 ++ tests/package/test_manifest.py | 2 +- tests/test_examples.py | 7 ++++--- tests/test_maintenance.py | 2 ++ tests/test_misc.py | 3 +++ tests/test_projectconf.py | 4 +++- tox.ini | 2 +- 22 files changed, 93 insertions(+), 13 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/commands/__init__.py create mode 100644 tests/ino2cpp/__init__.py create mode 100644 tests/package/__init__.py diff --git a/.pylintrc b/.pylintrc index 67ce4ef7..6ce74864 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,3 +1,6 @@ +[REPORTS] +output-format=colorized + [MESSAGES CONTROL] disable= bad-continuation, diff --git a/Makefile b/Makefile index 6b22d261..fa301e59 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ lint: - pylint --rcfile=./.pylintrc ./platformio + pylint -j 6 --rcfile=./.pylintrc ./platformio ./tests + pylint -j 6 --rcfile=./.pylintrc ./tests isort: isort -rc ./platformio diff --git a/platformio/platform/base.py b/platformio/platform/base.py index a5f3dd01..8e49288d 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -101,8 +101,6 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub if spec.name not in packages: packages[spec.name] = {} packages[spec.name].update(**options) - - print(13, packages) return packages @property diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/tests/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index b6d5b6d4..fc64db41 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=global-statement,unused-argument + import json import os import random @@ -30,6 +32,7 @@ pytestmark = pytest.mark.skipif( username = None email = None +splited_email = None firstname = None lastname = None password = None @@ -43,7 +46,7 @@ team_description = None def test_prepare(): - global username, splited_email, email, firstname, lastname + global username, email, splited_email, firstname, lastname global password, orgname, display_name, second_username, teamname, team_description username = "test-piocore-%s" % str(random.randint(0, 100000)) diff --git a/tests/commands/test_boards.py b/tests/commands/test_boards.py index bcb1f280..21142dd4 100644 --- a/tests/commands/test_boards.py +++ b/tests/commands/test_boards.py @@ -40,7 +40,7 @@ def test_board_options(clirunner, validate_cliresult): validate_cliresult(result) search_result = json.loads(result.output) assert isinstance(search_result, list) - assert len(search_result) + assert search_result platforms = [item["name"] for item in search_result] result = clirunner.invoke(cmd_boards, ["mbed", "--json-output"]) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 998d44cf..655449b0 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=redefined-outer-name + import json import sys from os.path import isfile, join diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 1880d671..332161e1 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import json import os diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index a71330db..dfb853f4 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=line-too-long + import json import re @@ -129,7 +131,9 @@ def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_c assert set(items1) >= set(items2) -def test_install_duplicates(clirunner, validate_cliresult, without_internet): +def test_install_duplicates( # pylint: disable=unused-argument + clirunner, validate_cliresult, without_internet +): # registry result = clirunner.invoke( cmd_lib, @@ -231,7 +235,7 @@ def test_global_lib_update_check(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"]) validate_cliresult(result) output = json.loads(result.output) - assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([lib["name"] for lib in output]) + assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set(lib["name"] for lib in output) def test_global_lib_update(clirunner, validate_cliresult): diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 74bb6f45..2c197ec3 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import json from platformio.commands import platform as cli_platform diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 16e0556c..9f072868 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -33,6 +33,7 @@ def test_local_env(): ) if result["returncode"] != 1: pytest.fail(str(result)) + # pylint: disable=unsupported-membership-test assert all([s in result["err"] for s in ("PASSED", "IGNORED", "FAILED")]), result[ "out" ] diff --git a/tests/commands/test_update.py b/tests/commands/test_update.py index 1817be33..b9ecb5c1 100644 --- a/tests/commands/test_update.py +++ b/tests/commands/test_update.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + from platformio.commands.update import cli as cmd_update diff --git a/tests/ino2cpp/__init__.py b/tests/ino2cpp/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/tests/ino2cpp/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/package/__init__.py b/tests/package/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/tests/package/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index c91924f4..f014a66b 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import os import time diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 35fdf367..9dd5b878 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -741,7 +741,7 @@ def test_examples_from_dir(tmpdir_factory): return re.sub(r"[\\/]+", "/", path) def _sort_examples(items): - for i, item in enumerate(items): + for i, _ in enumerate(items): items[i]["base"] = _to_unix_path(items[i]["base"]) items[i]["files"] = [_to_unix_path(f) for f in sorted(items[i]["files"])] return sorted(items, key=lambda item: item["name"]) diff --git a/tests/test_examples.py b/tests/test_examples.py index d0a580d4..ada20d35 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -21,7 +21,8 @@ import pytest from platformio import util from platformio.compat import PY2 -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -34,8 +35,8 @@ def pytest_generate_tests(metafunc): examples_dirs.append(normpath(join(dirname(__file__), "..", "examples"))) # dev/platforms - for manifest in PlatformManager().get_installed(): - p = PlatformFactory.new(manifest["__pkg_dir"]) + for pkg in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(pkg) examples_dir = join(p.get_dir(), "examples") assert isdir(examples_dir) examples_dirs.append(examples_dir) diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 4150bcdd..46bc82c8 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import json import os import re diff --git a/tests/test_misc.py b/tests/test_misc.py index d01cc46d..ae019f6b 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import pytest import requests @@ -21,6 +23,7 @@ from platformio import exception, util def test_platformio_cli(): result = util.exec_command(["pio", "--help"]) assert result["returncode"] == 0 + # pylint: disable=unsupported-membership-test assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result["out"] diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 4d832c97..832ceb71 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=redefined-outer-name + import os import pytest @@ -345,7 +347,7 @@ board = myboard ["check_types", [("float_option", 13.99), ("bool_option", True)]], ] ) - config.get("platformio", "extra_configs") == "extra.ini" + assert config.get("platformio", "extra_configs") == ["extra.ini"] config.remove_section("platformio") assert config.as_tuple() == [ ("env:myenv", [("board", "myboard"), ("framework", ["espidf", "arduino"])]), diff --git a/tox.ini b/tox.ini index fbe285e2..0faae46b 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ commands = [testenv:lint] commands = {envpython} --version - pylint --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./platformio ./tests [testenv:testcore] commands = From fb6e1fd33cf4bb23a99c4526a34f4cd01736a41a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Aug 2020 15:33:08 +0300 Subject: [PATCH 163/223] PyLint fixes --- Makefile | 2 +- tests/package/test_manager.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index fa301e59..57aa76c4 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ lint: - pylint -j 6 --rcfile=./.pylintrc ./platformio ./tests + pylint -j 6 --rcfile=./.pylintrc ./platformio pylint -j 6 --rcfile=./.pylintrc ./tests isort: diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index f014a66b..2c331dbe 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -349,7 +349,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert lm.uninstall(foo_1_0_0_pkg.path, silent=True) assert lm.uninstall(bar_pkg, silent=True) - assert len(lm.get_installed()) == 0 + assert not lm.get_installed() # test uninstall dependencies assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) @@ -360,7 +360,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): lm = LibraryPackageManager(str(storage_dir)) assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) assert lm.uninstall("AsyncMqttClient-esphome", silent=True) - assert len(lm.get_installed()) == 0 + assert not lm.get_installed() def test_registry(isolated_pio_core): From d9801946008f539d2a485fe359b3ba079e4e32d9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Aug 2020 15:34:02 +0300 Subject: [PATCH 164/223] Bump version to 4.4.0b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6ef8af69..3ecfb596 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -16,7 +16,7 @@ import sys DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) -VERSION = (4, 4, "0a8") +VERSION = (4, 4, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f79fb4190ec79bc31d3eaa3c46ce6a0672a8c308 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Aug 2020 14:25:59 +0300 Subject: [PATCH 165/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index e8ee370a..2f549805 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e8ee370a338270b453d4d97bd286f537d5f06456 +Subproject commit 2f5498050ae5602b92ab56e2ade0af33e088a944 From 49b70f44caa717e086706a42dbdaa85da165cfc9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 13:56:57 +0300 Subject: [PATCH 166/223] Ignore legacy tmp pkg folders --- platformio/package/manager/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index b307753c..cf359d13 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -196,6 +196,8 @@ class BasePackageManager( # pylint: disable=too-many-public-methods result = [] for name in sorted(os.listdir(self.package_dir)): + if name.startswith("_tmp_installing"): # legacy tmp folder + continue pkg_dir = os.path.join(self.package_dir, name) if not os.path.isdir(pkg_dir): continue From 70366d34b931603116b9760e3e202d28aeb08a3a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 13:57:18 +0300 Subject: [PATCH 167/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 2f549805..d01bbede 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2f5498050ae5602b92ab56e2ade0af33e088a944 +Subproject commit d01bbede6ca90420ed24736be4963a48228eff42 From aa186382a84db1af5e62c1faa0e487aaf712d09b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 14:22:37 +0300 Subject: [PATCH 168/223] Upgraded to SCons 4.0 --- HISTORY.rst | 32 +++++++++++++++++++------------- platformio/__init__.py | 2 +- platformio/platform/_run.py | 10 ++++++++-- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7f86e40f..17a8786b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -46,20 +46,26 @@ PlatformIO Core 4 - Command launcher with own arguments - Launch command with custom options declared in `"platformio.ini" `__ - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) + - List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) -* Display system-wide information using a new `platformio system info `__ command (`issue #3521 `_) -* List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) -* Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. -* Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment -* Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command -* Do not generate ".travis.yml" for a new project, let the user have a choice -* Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) -* Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) -* Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 -* Do not escape compiler arguments in VSCode template on Windows -* Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) -* Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) -* Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) +* **PlatformIO Build System** + + - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ + - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) + - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) + - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) + - Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) + +* **Miscellaneous** + + - Display system-wide information using a new `platformio system info `__ command (`issue #3521 `_) + - Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command + - Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. + - Do not generate ".travis.yml" for a new project, let the user have a choice + - Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 + - Do not escape compiler arguments in VSCode template on Windows + - Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 3ecfb596..b51a0be0 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -49,7 +49,7 @@ __core_packages__ = { "contrib-piohome": "~3.2.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~3.30102.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", "tool-cppcheck": "~1.190.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.7.0", diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 38cf232c..457983c4 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -20,7 +20,7 @@ import sys import click from platformio import app, fs, proc, telemetry -from platformio.compat import hashlib_encode_data, is_bytes +from platformio.compat import PY2, hashlib_encode_data, is_bytes from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound @@ -89,9 +89,15 @@ class PlatformRunMixin(object): telemetry.send_run_environment(topts, targets) def _run_scons(self, variables, targets, jobs): + scons_dir = get_core_package_dir("tool-scons") + script_path = ( + os.path.join(scons_dir, "script", "scons") + if PY2 + else os.path.join(scons_dir, "scons.py") + ) args = [ proc.get_pythonexe_path(), - os.path.join(get_core_package_dir("tool-scons"), "script", "scons"), + script_path, "-Q", "--warn=no-no-parallel-support", "--jobs", From d92c1d3442e7c93ba2d0a8510854349dcada6e3b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 17:48:49 +0300 Subject: [PATCH 169/223] Refactor HTTP related operations --- .pylintrc | 6 +- platformio/app.py | 7 +- platformio/clients/account.py | 14 +-- platformio/clients/http.py | 86 +++++++++++++- platformio/commands/debug/command.py | 4 +- platformio/commands/home/rpc/handlers/os.py | 3 +- platformio/commands/lib/command.py | 4 +- .../commands/remote/client/run_or_test.py | 4 +- platformio/commands/upgrade.py | 7 +- platformio/exception.py | 73 ------------ platformio/fs.py | 4 + platformio/maintenance.py | 9 +- platformio/package/download.py | 4 +- platformio/package/exception.py | 11 ++ platformio/package/lockfile.py | 10 +- platformio/package/manager/_update.py | 4 +- platformio/package/manager/core.py | 4 +- platformio/package/manager/platform.py | 4 +- platformio/package/manifest/parser.py | 3 +- platformio/package/manifest/schema.py | 4 +- platformio/package/unpack.py | 4 +- platformio/proc.py | 12 ++ platformio/telemetry.py | 2 +- platformio/util.py | 112 +++--------------- tests/commands/test_check.py | 6 +- tests/commands/test_test.py | 4 +- tests/conftest.py | 4 +- tests/test_examples.py | 8 +- tests/test_misc.py | 22 ++-- 29 files changed, 206 insertions(+), 233 deletions(-) diff --git a/.pylintrc b/.pylintrc index 6ce74864..e21dfef9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -15,4 +15,8 @@ disable= useless-object-inheritance, useless-import-alias, fixme, - bad-option-value + bad-option-value, + + ; PY2 Compat + super-with-arguments, + raise-missing-from diff --git a/platformio/app.py b/platformio/app.py index 21adba1c..0933892b 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -26,7 +26,7 @@ from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath from time import time -from platformio import __version__, exception, fs, proc, util +from platformio import __version__, exception, fs, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import ( @@ -394,6 +394,9 @@ def is_disabled_progressbar(): def get_cid(): + # pylint: disable=import-outside-toplevel + from platformio.clients.http import fetch_remote_content + cid = get_state_item("cid") if cid: return cid @@ -403,7 +406,7 @@ def get_cid(): elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): try: uid = json.loads( - util.fetch_remote_content( + fetch_remote_content( "{api}/user?token={token}".format( api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), token=getenv("USER_TOKEN"), diff --git a/platformio/clients/account.py b/platformio/clients/account.py index c29ef9f9..d492b65d 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -67,7 +67,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods token = self.fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.request_json_data(*args, **kwargs) + return self.fetch_json_data(*args, **kwargs) def login(self, username, password): try: @@ -79,7 +79,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods app.get_state_item("account", {}).get("email", "") ) - data = self.request_json_data( + data = self.fetch_json_data( "post", "/v1/login", data={"username": username, "password": password}, ) app.set_state_item("account", data) @@ -95,7 +95,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods app.get_state_item("account", {}).get("email", "") ) - result = self.request_json_data( + result = self.fetch_json_data( "post", "/v1/login/code", data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, @@ -107,7 +107,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods refresh_token = self.get_refresh_token() self.delete_local_session() try: - self.request_json_data( + self.fetch_json_data( "post", "/v1/logout", data={"refresh_token": refresh_token}, ) except AccountError: @@ -133,7 +133,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods app.get_state_item("account", {}).get("email", "") ) - return self.request_json_data( + return self.fetch_json_data( "post", "/v1/registration", data={ @@ -153,7 +153,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods ).get("auth_token") def forgot_password(self, username): - return self.request_json_data( + return self.fetch_json_data( "post", "/v1/forgot", data={"username": username}, ) @@ -278,7 +278,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods return auth.get("access_token") if auth.get("refresh_token"): try: - data = self.request_json_data( + data = self.fetch_json_data( "post", "/v1/login", headers={ diff --git a/platformio/clients/http.py b/platformio/clients/http.py index e18d2eed..0b4ca373 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -12,11 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os +import socket + import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util -from platformio.exception import PlatformioException +from platformio.exception import PlatformioException, UserSideException + + +PING_REMOTE_HOSTS = [ + "140.82.118.3", # Github.com + "35.231.145.151", # Gitlab.com + "88.198.170.159", # platformio.org + "github.com", + "platformio.org", +] class HTTPClientError(PlatformioException): @@ -29,6 +42,15 @@ class HTTPClientError(PlatformioException): return self.message +class InternetIsOffline(UserSideException): + + MESSAGE = ( + "You are not connected to the Internet.\n" + "PlatformIO needs the Internet connection to" + " download dependent packages or to work with PIO Account." + ) + + class HTTPClient(object): def __init__( self, base_url, @@ -57,7 +79,7 @@ class HTTPClient(object): def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout # print(self, method, path, kwargs) - util.internet_on(raise_exception=True) + ensure_internet_on(raise_exception=True) # set default timeout if "timeout" not in kwargs: @@ -68,9 +90,18 @@ class HTTPClient(object): except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: raise HTTPClientError(str(e)) - def request_json_data(self, *args, **kwargs): - response = self.send_request(*args, **kwargs) - return self.raise_error_from_response(response) + def fetch_json_data(self, *args, **kwargs): + cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None + if not cache_valid: + return self.raise_error_from_response(self.send_request(*args, **kwargs)) + cache_key = app.ContentCache.key_from_args(*args, kwargs) + with app.ContentCache() as cc: + result = cc.get(cache_key) + if result is not None: + return json.loads(result) + response = self.send_request(*args, **kwargs) + cc.set(cache_key, response.text, cache_valid) + return self.raise_error_from_response(response) @staticmethod def raise_error_from_response(response, expected_codes=(200, 201, 202)): @@ -84,3 +115,48 @@ class HTTPClient(object): except (KeyError, ValueError): message = response.text raise HTTPClientError(message, response) + + +# +# Helpers +# + + +@util.memoized(expire="10s") +def _internet_on(): + timeout = 2 + socket.setdefaulttimeout(timeout) + for host in PING_REMOTE_HOSTS: + try: + for var in ("HTTP_PROXY", "HTTPS_PROXY"): + if not os.getenv(var) and not os.getenv(var.lower()): + continue + requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) + return True + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((host, 80)) + s.close() + return True + except: # pylint: disable=bare-except + pass + return False + + +def ensure_internet_on(raise_exception=False): + result = _internet_on() + if raise_exception and not result: + raise InternetIsOffline() + return result + + +def fetch_remote_content(*args, **kwargs): + kwargs["headers"] = kwargs.get("headers", {}) + if "User-Agent" not in kwargs["headers"]: + kwargs["headers"]["User-Agent"] = app.get_user_agent() + + if "timeout" not in kwargs: + kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + + r = requests.get(*args, **kwargs) + r.raise_for_status() + return r.text diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 78a43eef..98115cbf 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -21,7 +21,7 @@ from os.path import isfile import click -from platformio import app, exception, fs, proc, util +from platformio import app, exception, fs, proc from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.package.manager.core import inject_contrib_pysite @@ -130,7 +130,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro nl=False, ) stream = helpers.GDBMIConsoleStream() - with util.capture_std_streams(stream): + with proc.capture_std_streams(stream): helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) stream.close() else: diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 2b1662f2..7c833180 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -23,6 +23,7 @@ import click from twisted.internet import defer # pylint: disable=import-error from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util +from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, glob_recursive @@ -47,7 +48,7 @@ class OSRPC(object): defer.returnValue(result) # check internet before and resolve issue with 60 seconds timeout - util.internet_on(raise_exception=True) + ensure_internet_on(raise_exception=True) session = helpers.requests_session() if data: diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 03463aab..6ca0ee77 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -28,7 +28,7 @@ from platformio.commands.lib.helpers import ( save_project_libdeps, ) from platformio.compat import dump_json_to_unicode -from platformio.package.exception import UnknownPackageError +from platformio.package.exception import NotGlobalLibDir, UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import is_ci @@ -97,7 +97,7 @@ def cli(ctx, **options): ) if not storage_dirs: - raise exception.NotGlobalLibDir( + raise NotGlobalLibDir( get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand ) diff --git a/platformio/commands/remote/client/run_or_test.py b/platformio/commands/remote/client/run_or_test.py index c986ad0a..10a9b008 100644 --- a/platformio/commands/remote/client/run_or_test.py +++ b/platformio/commands/remote/client/run_or_test.py @@ -20,7 +20,7 @@ from io import BytesIO from twisted.spread import pb # pylint: disable=import-error -from platformio import util +from platformio import fs from platformio.commands.remote.client.async_base import AsyncClientBase from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync from platformio.compat import hashlib_encode_data @@ -64,7 +64,7 @@ class RunOrTestClient(AsyncClientBase): return "%s-%s" % (os.path.basename(path), h.hexdigest()) def add_project_items(self, psync): - with util.cd(self.options["project_dir"]): + with fs.cd(self.options["project_dir"]): cfg = ProjectConfig.get_instance( os.path.join(self.options["project_dir"], "platformio.ini") ) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index c8c8b9fe..2411f49c 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -19,7 +19,8 @@ from zipfile import ZipFile import click -from platformio import VERSION, __version__, app, exception, util +from platformio import VERSION, __version__, app, exception +from platformio.clients.http import fetch_remote_content from platformio.compat import WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -130,7 +131,7 @@ def get_latest_version(): def get_develop_latest_version(): version = None - content = util.fetch_remote_content( + content = fetch_remote_content( "https://raw.githubusercontent.com/platformio/platformio" "/develop/platformio/__init__.py" ) @@ -150,5 +151,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - content = util.fetch_remote_content("https://pypi.org/pypi/platformio/json") + content = fetch_remote_content("https://pypi.org/pypi/platformio/json") return json.loads(content)["info"]["version"] diff --git a/platformio/exception.py b/platformio/exception.py index 91fd67cc..8ae549bc 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -30,10 +30,6 @@ class ReturnErrorCode(PlatformioException): MESSAGE = "{0}" -class LockFileTimeoutError(PlatformioException): - pass - - class MinitermException(PlatformioException): pass @@ -47,61 +43,6 @@ class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" -# Package Manager - - -class PlatformIOPackageException(PlatformioException): - pass - - -class UnknownPackage(UserSideException): - - MESSAGE = "Detected unknown package '{0}'" - - -class MissingPackageManifest(PlatformIOPackageException): - - MESSAGE = "Could not find one of '{0}' manifest files in the package" - - -class UndefinedPackageVersion(PlatformIOPackageException): - - MESSAGE = ( - "Could not find a version that satisfies the requirement '{0}'" - " for your system '{1}'" - ) - - -class PackageInstallError(PlatformIOPackageException): - - MESSAGE = ( - "Could not install '{0}' with version requirements '{1}' " - "for your system '{2}'.\n\n" - "Please try this solution -> http://bit.ly/faq-package-manager" - ) - - -# -# Library -# - - -class NotGlobalLibDir(UserSideException): - - MESSAGE = ( - "The `{0}` is not a PlatformIO project.\n\n" - "To manage libraries in global storage `{1}`,\n" - "please use `platformio lib --global {2}` or specify custom storage " - "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" - "Check `platformio lib --help` for details." - ) - - -class InvalidLibConfURL(UserSideException): - - MESSAGE = "Invalid library config URL '{0}'" - - # # UDEV Rules # @@ -143,20 +84,6 @@ class GetLatestVersionError(PlatformioException): MESSAGE = "Can not retrieve the latest PlatformIO version" -class APIRequestError(PlatformioException): - - MESSAGE = "[API] {0}" - - -class InternetIsOffline(UserSideException): - - MESSAGE = ( - "You are not connected to the Internet.\n" - "PlatformIO needs the Internet connection to" - " download dependent packages or to work with PIO Account." - ) - - class InvalidSettingName(UserSideException): MESSAGE = "Invalid setting with the name '{0}'" diff --git a/platformio/fs.py b/platformio/fs.py index 7a592746..a4dc6ee4 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -176,6 +176,10 @@ def expanduser(path): return os.environ["USERPROFILE"] + path[1:] +def change_filemtime(path, mtime): + os.utime(path, (mtime, mtime)) + + def rmtree(path): def _onerror(func, path, __): try: diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 54b0ad6d..d70c5ca8 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -20,6 +20,7 @@ import click import semantic_version from platformio import __version__, app, exception, fs, telemetry, util +from platformio.clients import http from platformio.commands import PlatformioCLI from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update @@ -53,9 +54,9 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_internal_updates(ctx, "platforms") check_internal_updates(ctx, "libraries") except ( - exception.InternetIsOffline, + http.HTTPClientError, + http.InternetIsOffline, exception.GetLatestVersionError, - exception.APIRequestError, ): click.secho( "Failed to check for PlatformIO upgrades. " @@ -221,7 +222,7 @@ def check_platformio_upgrade(): last_check["platformio_upgrade"] = int(time()) app.set_state_item("last_check", last_check) - util.internet_on(raise_exception=True) + http.ensure_internet_on(raise_exception=True) # Update PlatformIO's Core packages update_core_packages(silent=True) @@ -268,7 +269,7 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches last_check[what + "_update"] = int(time()) app.set_state_item("last_check", last_check) - util.internet_on(raise_exception=True) + http.ensure_internet_on(raise_exception=True) outdated_items = [] pm = PlatformPackageManager() if what == "platforms" else LibraryPackageManager() diff --git a/platformio/package/download.py b/platformio/package/download.py index 7f29e7ac..0f40fd0d 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -21,7 +21,7 @@ from time import mktime import click import requests -from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs from platformio.package.exception import PackageException @@ -134,7 +134,7 @@ class FileDownloader(object): def _preserve_filemtime(self, lmdate): timedata = parsedate_tz(lmdate) lmtime = mktime(timedata[:9]) - util.change_filemtime(self._destination, lmtime) + fs.change_filemtime(self._destination, lmtime) def __del__(self): if self._request: diff --git a/platformio/package/exception.py b/platformio/package/exception.py index f32c89ce..0f34592f 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -58,3 +58,14 @@ class UnknownPackageError(UserSideException): "Could not find a package with '{0}' requirements for your system '%s'" % util.get_systype() ) + + +class NotGlobalLibDir(UserSideException): + + MESSAGE = ( + "The `{0}` is not a PlatformIO project.\n\n" + "To manage libraries in global storage `{1}`,\n" + "please use `platformio lib --global {2}` or specify custom storage " + "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" + "Check `platformio lib --help` for details." + ) diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 44d2e4cf..db4b1d3f 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -15,7 +15,7 @@ import os from time import sleep, time -from platformio import exception +from platformio.exception import PlatformioException LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour LOCKFILE_DELAY = 0.2 @@ -36,7 +36,11 @@ except ImportError: LOCKFILE_CURRENT_INTERFACE = None -class LockFileExists(Exception): +class LockFileExists(PlatformioException): + pass + + +class LockFileTimeoutError(PlatformioException): pass @@ -88,7 +92,7 @@ class LockFile(object): sleep(self.delay) elapsed += self.delay - raise exception.LockFileTimeoutError() + raise LockFileTimeoutError() def release(self): self._unlock() diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index d3e8dbb1..10fdd978 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -16,10 +16,10 @@ import os import click -from platformio import util from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec from platformio.package.vcsclient import VCSBaseException, VCSClientFactory +from platformio.clients.http import ensure_internet_on class PackageManagerUpdateMixin(object): @@ -97,7 +97,7 @@ class PackageManagerUpdateMixin(object): ), nl=False, ) - if not util.internet_on(): + if not ensure_internet_on(): if not silent: click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) return pkg diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 2b872ab6..9f02b846 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -17,7 +17,7 @@ import os import subprocess import sys -from platformio import __core_packages__, exception, util +from platformio import __core_packages__, fs, exception, util from platformio.compat import PY2 from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec @@ -93,7 +93,7 @@ def inject_contrib_pysite(verify_openssl=False): def build_contrib_pysite_deps(target_dir): if os.path.isdir(target_dir): - util.rmtree_(target_dir) + fs.rmtree(target_dir) os.makedirs(target_dir) with open(os.path.join(target_dir, "package.json"), "w") as fp: json.dump( diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 91eabf6a..2172e672 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -13,7 +13,7 @@ # limitations under the License. from platformio import util -from platformio.exception import APIRequestError, InternetIsOffline +from platformio.clients.http import HTTPClientError, InternetIsOffline from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager from platformio.package.manager.tool import ToolPackageManager @@ -176,7 +176,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an key = "%s:%s" % (board["platform"], board["id"]) if key not in know_boards: boards.append(board) - except (APIRequestError, InternetIsOffline): + except (HTTPClientError, InternetIsOffline): pass return sorted(boards, key=lambda b: b["name"]) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index d453c83e..689de80b 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -20,6 +20,7 @@ import re import tarfile from platformio import util +from platformio.clients.http import fetch_remote_content from platformio.compat import get_object_members, string_types from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project @@ -106,7 +107,7 @@ class ManifestParserFactory(object): @staticmethod def new_from_url(remote_url): - content = util.fetch_remote_content(remote_url) + content = fetch_remote_content(remote_url) return ManifestParserFactory.new( content, ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 7dafaa23..39327f4a 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -21,7 +21,7 @@ import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates -from platformio import util +from platformio.clients.http import fetch_remote_content from platformio.package.exception import ManifestValidationError from platformio.util import memoized @@ -256,4 +256,4 @@ class ManifestSchema(BaseSchema): "https://raw.githubusercontent.com/spdx/license-list-data" "/v%s/json/licenses.json" % version ) - return json.loads(util.fetch_remote_content(spdx_data_url)) + return json.loads(fetch_remote_content(spdx_data_url)) diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index a00873cd..9956b46a 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -19,7 +19,7 @@ from zipfile import ZipFile import click -from platformio import util +from platformio import fs from platformio.package.exception import PackageException @@ -109,7 +109,7 @@ class ZIPArchiver(BaseArchiver): @staticmethod def preserve_mtime(item, dest_dir): - util.change_filemtime( + fs.change_filemtime( os.path.join(dest_dir, item.filename), mktime(tuple(item.date_time) + tuple([0, 0, 0])), ) diff --git a/platformio/proc.py b/platformio/proc.py index 04f15a57..82f5a9cf 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -15,6 +15,7 @@ import os import subprocess import sys +from contextlib import contextmanager from threading import Thread from platformio import exception @@ -137,6 +138,17 @@ def exec_command(*args, **kwargs): return result +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + def is_ci(): return os.getenv("CI", "").lower() == "true" diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 7435bdab..5e5878c8 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -124,7 +124,7 @@ class MeasurementProtocol(TelemetryBase): caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() self["cd4"] = ( - 1 if (not util.is_ci() and (caller_id or not is_container())) else 0 + 1 if (not is_ci() and (caller_id or not is_container())) else 0 ) if caller_id: self["cd5"] = caller_id.lower() diff --git a/platformio/util.py b/platformio/util.py index 982b0bab..a0686377 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -19,26 +19,17 @@ import math import os import platform import re -import socket import sys import time -from contextlib import contextmanager from functools import wraps from glob import glob import click -import requests -from platformio import DEFAULT_REQUESTS_TIMEOUT, __apiurl__, __version__, exception -from platformio.commands import PlatformioCLI +from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS -from platformio.fs import cd # pylint: disable=unused-import from platformio.fs import load_json # pylint: disable=unused-import -from platformio.fs import rmtree as rmtree_ # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import -from platformio.proc import is_ci # pylint: disable=unused-import - -# KEEP unused imports for backward compatibility with PIO Core 3.0 API class memoized(object): @@ -97,17 +88,6 @@ def singleton(cls): return get_instance -@contextmanager -def capture_std_streams(stdout, stderr=None): - _stdout = sys.stdout - _stderr = sys.stderr - sys.stdout = stdout - sys.stderr = stderr or stdout - yield - sys.stdout = _stdout - sys.stderr = _stderr - - def get_systype(): type_ = platform.system().lower() arch = platform.machine().lower() @@ -116,16 +96,6 @@ def get_systype(): return "%s_%s" % (type_, arch) if arch else type_ -def pioversion_to_intstr(): - vermatch = re.match(r"^([\d\.]+)", __version__) - assert vermatch - return [int(i) for i in vermatch.group(1).split(".")[:3]] - - -def change_filemtime(path, mtime): - os.utime(path, (mtime, mtime)) - - def get_serial_ports(filter_hwid=False): try: # pylint: disable=import-outside-toplevel @@ -164,7 +134,7 @@ def get_logical_devices(): items = [] if WINDOWS: try: - result = exec_command( + result = proc.exec_command( ["wmic", "logicaldisk", "get", "name,VolumeName"] ).get("out", "") devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") @@ -177,12 +147,12 @@ def get_logical_devices(): except WindowsError: # pylint: disable=undefined-variable pass # try "fsutil" - result = exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") + result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") for device in re.findall(r"[A-Z]:\\", result): items.append({"path": device, "name": None}) return items - result = exec_command(["df"]).get("out") + result = proc.exec_command(["df"]).get("out") devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) for line in result.split("\n"): match = devicenamere.match(line.strip()) @@ -370,60 +340,27 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): ) -PING_REMOTE_HOSTS = [ - "140.82.118.3", # Github.com - "35.231.145.151", # Gitlab.com - "88.198.170.159", # platformio.org - "github.com", - "platformio.org", -] - - -@memoized(expire="10s") -def _internet_on(): - timeout = 2 - socket.setdefaulttimeout(timeout) - for host in PING_REMOTE_HOSTS: - try: - for var in ("HTTP_PROXY", "HTTPS_PROXY"): - if not os.getenv(var, var.lower()): - continue - requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) - return True - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) - return True - except: # pylint: disable=bare-except - pass - return False - - -def internet_on(raise_exception=False): - result = _internet_on() - if raise_exception and not result: - raise exception.InternetIsOffline() - return result - - -def fetch_remote_content(*args, **kwargs): - # pylint: disable=import-outside-toplevel - from platformio.app import get_user_agent - - kwargs["headers"] = kwargs.get("headers", {}) - if "User-Agent" not in kwargs["headers"]: - kwargs["headers"]["User-Agent"] = get_user_agent() - - if "timeout" not in kwargs: - kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT - - r = requests.get(*args, **kwargs) - r.raise_for_status() - return r.text +def pioversion_to_intstr(): + vermatch = re.match(r"^([\d\.]+)", __version__) + assert vermatch + return [int(i) for i in vermatch.group(1).split(".")[:3]] def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) +def get_original_version(version): + if version.count(".") != 2: + return None + _, raw = version.split(".")[:2] + if int(raw) <= 99: + return None + if int(raw) <= 9999: + return "%s.%s" % (raw[:-2], int(raw[-2:])) + return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) + + def items_to_list(items): if isinstance(items, list): return items @@ -472,14 +409,3 @@ def humanize_duration_time(duration): tokens.append(int(round(duration) if multiplier == 1 else fraction)) duration -= fraction * multiplier return "{:02d}:{:02d}:{:02d}.{:03d}".format(*tokens) - - -def get_original_version(version): - if version.count(".") != 2: - return None - _, raw = version.split(".")[:2] - if int(raw) <= 99: - return None - if int(raw) <= 9999: - return "%s.%s" % (raw[:-2], int(raw[-2:])) - return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 655449b0..596c0f29 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -389,7 +389,7 @@ check_tool = pvs-studio assert style == 0 -def test_check_embedded_platform_all_tools(clirunner, tmpdir): +def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir): config = """ [env:test] platform = ststm32 @@ -422,11 +422,9 @@ int main() { for framework in frameworks: for tool in ("cppcheck", "clangtidy", "pvs-studio"): tmpdir.join("platformio.ini").write(config % (framework, tool)) - result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) - + validate_cliresult(result) defects = sum(count_defects(result.output)) - assert result.exit_code == 0 and defects > 0, "Failed %s with %s" % ( framework, tool, diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 9f072868..e0a64a8c 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -16,12 +16,12 @@ import os import pytest -from platformio import util +from platformio import proc from platformio.commands.test.command import cli as cmd_test def test_local_env(): - result = util.exec_command( + result = proc.exec_command( [ "platformio", "test", diff --git a/tests/conftest.py b/tests/conftest.py index 56a59cbd..d81f0e8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,7 +20,7 @@ import time import pytest from click.testing import CliRunner -from platformio import util +from platformio.clients import http def pytest_configure(config): @@ -74,7 +74,7 @@ def isolated_pio_core(request, tmpdir_factory): @pytest.fixture(scope="function") def without_internet(monkeypatch): - monkeypatch.setattr(util, "_internet_on", lambda: False) + monkeypatch.setattr(http, "_internet_on", lambda: False) @pytest.fixture diff --git a/tests/test_examples.py b/tests/test_examples.py index ada20d35..994eb8c0 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -19,7 +19,7 @@ from os.path import basename, dirname, getsize, isdir, isfile, join, normpath import pytest -from platformio import util +from platformio import fs, proc from platformio.compat import PY2 from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory @@ -64,14 +64,14 @@ def pytest_generate_tests(metafunc): def test_run(pioproject_dir): - with util.cd(pioproject_dir): + with fs.cd(pioproject_dir): config = ProjectConfig() build_dir = config.get_optional_dir("build") if isdir(build_dir): - util.rmtree_(build_dir) + fs.rmtree(build_dir) env_names = config.envs() - result = util.exec_command( + result = proc.exec_command( ["platformio", "run", "-e", random.choice(env_names)] ) if result["returncode"] != 0: diff --git a/tests/test_misc.py b/tests/test_misc.py index ae019f6b..f816fe6d 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -17,29 +17,33 @@ import pytest import requests -from platformio import exception, util +from platformio import proc +from platformio.clients import http +from platformio.clients.registry import RegistryClient def test_platformio_cli(): - result = util.exec_command(["pio", "--help"]) + result = proc.exec_command(["pio", "--help"]) assert result["returncode"] == 0 # pylint: disable=unsupported-membership-test assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result["out"] def test_ping_internet_ips(): - for host in util.PING_REMOTE_HOSTS: + for host in http.PING_REMOTE_HOSTS: requests.get("http://%s" % host, allow_redirects=False, timeout=2) def test_api_internet_offline(without_internet, isolated_pio_core): - with pytest.raises(exception.InternetIsOffline): - util.get_api_result("/stats") + regclient = RegistryClient() + with pytest.raises(http.InternetIsOffline): + regclient.fetch_json_data("get", "/v2/stats") def test_api_cache(monkeypatch, isolated_pio_core): - api_kwargs = {"url": "/stats", "cache_valid": "10s"} - result = util.get_api_result(**api_kwargs) + regclient = RegistryClient() + api_kwargs = {"method": "get", "path": "/v2/stats", "cache_valid": "10s"} + result = regclient.fetch_json_data(**api_kwargs) assert result and "boards" in result - monkeypatch.setattr(util, "_internet_on", lambda: False) - assert util.get_api_result(**api_kwargs) == result + monkeypatch.setattr(http, "_internet_on", lambda: False) + assert regclient.fetch_json_data(**api_kwargs) == result From 102aa5f22bec5c7c79de71f3db3b8b252e7123f8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 17:49:29 +0300 Subject: [PATCH 170/223] Port legacy API requests to the new registry client --- platformio/__init__.py | 2 - platformio/clients/registry.py | 15 ++-- platformio/commands/lib/command.py | 24 ++++--- platformio/commands/platform.py | 6 +- platformio/package/manager/platform.py | 7 +- platformio/util.py | 98 -------------------------- 6 files changed, 28 insertions(+), 124 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b51a0be0..0d25faf8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -39,8 +39,6 @@ __email__ = "contact@platformio.org" __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" -__apiurl__ = "https://api.platformio.org" - __accounts_api__ = "https://api.accounts.platformio.org" __registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f8130c60..6111846d 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -17,11 +17,6 @@ from platformio.clients.account import AccountClient from platformio.clients.http import HTTPClient, HTTPClientError from platformio.package.meta import PackageType -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - # pylint: disable=too-many-arguments @@ -35,7 +30,7 @@ class RegistryClient(HTTPClient): token = AccountClient().fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.request_json_data(*args, **kwargs) + return self.fetch_json_data(*args, **kwargs) def publish_package( self, archive_path, owner=None, released_at=None, private=False, notify=True @@ -123,17 +118,17 @@ class RegistryClient(HTTPClient): search_query.append('%s:"%s"' % (name[:-1], value)) if query: search_query.append(query) - params = dict(query=quote(" ".join(search_query))) + params = dict(query=" ".join(search_query)) if page: params["page"] = int(page) - return self.request_json_data("get", "/v3/packages", params=params) + return self.fetch_json_data("get", "/v3/packages", params=params) def get_package(self, type_, owner, name, version=None): try: - return self.request_json_data( + return self.fetch_json_data( "get", "/v3/packages/{owner}/{type}/{name}".format( - type=type_, owner=owner.lower(), name=quote(name.lower()) + type=type_, owner=owner.lower(), name=name.lower() ), params=dict(version=version) if version else None, ) diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 6ca0ee77..96d39814 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -347,6 +347,7 @@ def lib_list(ctx, json_output): help="Do not prompt, automatically paginate with delay", ) def lib_search(query, json_output, page, noninteractive, **filters): + regclient = LibraryPackageManager().get_registry_client_instance() if not query: query = [] if not isinstance(query, list): @@ -356,8 +357,11 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d" + result = regclient.fetch_json_data( + "get", + "/v2/lib/search", + params=dict(query=" ".join(query), page=page), + cache_valid="1d", ) if json_output: @@ -406,9 +410,10 @@ def lib_search(query, json_output, page, noninteractive, **filters): time.sleep(5) elif not click.confirm("Show next libraries?"): break - result = util.get_api_result( + result = regclient.fetch_json_data( + "get", "/v2/lib/search", - {"query": " ".join(query), "page": int(result["page"]) + 1}, + params=dict(query=" ".join(query), page=int(result["page"]) + 1), cache_valid="1d", ) @@ -438,10 +443,10 @@ def lib_builtin(storage, json_output): @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) def lib_show(library, json_output): - lib_id = LibraryPackageManager().reveal_registry_package_id( - library, silent=json_output - ) - lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") + lm = LibraryPackageManager() + lib_id = lm.reveal_registry_package_id(library, silent=json_output) + regclient = lm.get_registry_client_instance() + lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h") if json_output: return click.echo(dump_json_to_unicode(lib)) @@ -534,7 +539,8 @@ def lib_register(config_url): # pylint: disable=unused-argument @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): - result = util.get_api_result("/lib/stats", cache_valid="1h") + regclient = LibraryPackageManager().get_registry_client_instance() + result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h") if json_output: return click.echo(dump_json_to_unicode(result)) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index b996a16a..803149b1 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -57,7 +57,8 @@ def _print_platforms(platforms): def _get_registry_platforms(): - return util.get_api_result("/platforms", cache_valid="7d") + regclient = PlatformPackageManager().get_registry_client_instance() + return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d") def _get_platform_data(*args, **kwargs): @@ -188,8 +189,9 @@ def platform_search(query, json_output): @click.argument("query", required=False) @click.option("--json-output", is_flag=True) def platform_frameworks(query, json_output): + regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] - for framework in util.get_api_result("/frameworks", cache_valid="7d"): + for framework in regclient.fetch_json_data("get", "/v2/frameworks", cache_valid="1d"): if query == "all": query = "" search_data = dump_json_to_unicode(framework) diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 2172e672..71e8c5fb 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -164,9 +164,10 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an boards.append(board) return boards - @staticmethod - def get_registered_boards(): - return util.get_api_result("/boards", cache_valid="7d") + def get_registered_boards(self): + return self.get_registry_client_instance().fetch_json_data( + "get", "/v2/boards", cache_valid="1d" + ) def get_all_boards(self): boards = self.get_installed_boards() diff --git a/platformio/util.py b/platformio/util.py index a0686377..04576b24 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -242,104 +242,6 @@ def get_mdns_services(): return items -@memoized(expire="60s") -def _api_request_session(): - return requests.Session() - - -@throttle(500) -def _get_api_result( - url, params=None, data=None, auth=None # pylint: disable=too-many-branches -): - # pylint: disable=import-outside-toplevel - from platformio.app import get_user_agent, get_setting - - result = {} - r = None - verify_ssl = sys.version_info >= (2, 7, 9) - - if not url.startswith("http"): - url = __apiurl__ + url - if not get_setting("strict_ssl"): - url = url.replace("https://", "http://") - - headers = {"User-Agent": get_user_agent()} - try: - if data: - r = _api_request_session().post( - url, - params=params, - data=data, - headers=headers, - auth=auth, - verify=verify_ssl, - timeout=DEFAULT_REQUESTS_TIMEOUT, - ) - else: - r = _api_request_session().get( - url, - params=params, - headers=headers, - auth=auth, - verify=verify_ssl, - timeout=DEFAULT_REQUESTS_TIMEOUT, - ) - result = r.json() - r.raise_for_status() - return r.text - except requests.exceptions.HTTPError as e: - if result and "message" in result: - raise exception.APIRequestError(result["message"]) - if result and "errors" in result: - raise exception.APIRequestError(result["errors"][0]["title"]) - raise exception.APIRequestError(e) - except ValueError: - raise exception.APIRequestError("Invalid response: %s" % r.text.encode("utf-8")) - finally: - if r: - r.close() - return None - - -def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): - from platformio.app import ContentCache # pylint: disable=import-outside-toplevel - - total = 0 - max_retries = 5 - cache_key = ( - ContentCache.key_from_args(url, params, data, auth) if cache_valid else None - ) - while total < max_retries: - try: - with ContentCache() as cc: - if cache_key: - result = cc.get(cache_key) - if result is not None: - return json.loads(result) - - # check internet before and resolve issue with 60 seconds timeout - internet_on(raise_exception=True) - - result = _get_api_result(url, params, data) - if cache_valid: - with ContentCache() as cc: - cc.set(cache_key, result, cache_valid) - return json.loads(result) - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - total += 1 - if not PlatformioCLI.in_silence(): - click.secho( - "[API] ConnectionError: {0} (incremented retry: max={1}, " - "total={2})".format(e, max_retries, total), - fg="yellow", - ) - time.sleep(2 * total) - - raise exception.APIRequestError( - "Could not connect to PlatformIO API Service. Please try later." - ) - - def pioversion_to_intstr(): vermatch = re.match(r"^([\d\.]+)", __version__) assert vermatch From abae9c7e771492673efdabaed8f008daca390bc3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 17:52:12 +0300 Subject: [PATCH 171/223] Cache base registry requests --- platformio/clients/registry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 6111846d..e990a65f 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -121,7 +121,9 @@ class RegistryClient(HTTPClient): params = dict(query=" ".join(search_query)) if page: params["page"] = int(page) - return self.fetch_json_data("get", "/v3/packages", params=params) + return self.fetch_json_data( + "get", "/v3/packages", params=params, cache_valid="1h" + ) def get_package(self, type_, owner, name, version=None): try: @@ -131,6 +133,7 @@ class RegistryClient(HTTPClient): type=type_, owner=owner.lower(), name=name.lower() ), params=dict(version=version) if version else None, + cache_valid="1h", ) except HTTPClientError as e: if e.response.status_code == 404: From 7e4bfb1959b1876ed54da2c6d2e193de06d124b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 20:05:14 +0300 Subject: [PATCH 172/223] Move CacheContent API to "cache.py" module --- README.rst | 1 - platformio/app.py | 168 ++---------------- platformio/cache.py | 165 +++++++++++++++++ platformio/clients/account.py | 4 +- platformio/clients/http.py | 16 +- platformio/commands/debug/process/client.py | 9 +- platformio/commands/home/rpc/handlers/misc.py | 8 +- platformio/commands/home/rpc/handlers/os.py | 9 +- platformio/commands/platform.py | 10 +- platformio/commands/update.py | 4 +- platformio/maintenance.py | 3 +- platformio/package/manager/_download.py | 2 +- platformio/package/manager/_update.py | 2 +- platformio/package/manager/core.py | 2 +- platformio/telemetry.py | 4 +- 15 files changed, 215 insertions(+), 192 deletions(-) create mode 100644 platformio/cache.py diff --git a/README.rst b/README.rst index fcff06c8..c4ab3d5f 100644 --- a/README.rst +++ b/README.rst @@ -147,7 +147,6 @@ Share minimal diagnostics and usage information to help us make PlatformIO bette It is enabled by default. For more information see: * `Telemetry Setting `_ -* `SSL Setting `_ License ------- diff --git a/platformio/app.py b/platformio/app.py index 0933892b..59900500 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import codecs import getpass import hashlib import json @@ -22,18 +21,12 @@ import os import platform import socket import uuid -from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath -from time import time from platformio import __version__, exception, fs, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.package.lockfile import LockFile -from platformio.project.helpers import ( - get_default_projects_dir, - get_project_cache_dir, - get_project_core_dir, -) +from platformio.project.helpers import get_default_projects_dir, get_project_core_dir def projects_dir_validate(projects_dir): @@ -63,10 +56,9 @@ DEFAULT_SETTINGS = { "value": 7, }, "enable_cache": { - "description": "Enable caching for API requests and Library Manager", + "description": "Enable caching for HTTP API requests", "value": True, }, - "strict_ssl": {"description": "Strict SSL for PlatformIO Services", "value": False}, "enable_telemetry": { "description": ("Telemetry service (Yes/No)"), "value": True, @@ -173,146 +165,6 @@ class State(object): return item in self._storage -class ContentCache(object): - def __init__(self, cache_dir=None): - self.cache_dir = None - self._db_path = None - self._lockfile = None - - self.cache_dir = cache_dir or get_project_cache_dir() - self._db_path = join(self.cache_dir, "db.data") - - def __enter__(self): - self.delete() - return self - - def __exit__(self, type_, value, traceback): - pass - - def _lock_dbindex(self): - if not self.cache_dir: - os.makedirs(self.cache_dir) - self._lockfile = LockFile(self.cache_dir) - try: - self._lockfile.acquire() - except: # pylint: disable=bare-except - return False - - return True - - def _unlock_dbindex(self): - if self._lockfile: - self._lockfile.release() - return True - - def get_cache_path(self, key): - assert "/" not in key and "\\" not in key - key = str(key) - assert len(key) > 3 - return join(self.cache_dir, key[-2:], key) - - @staticmethod - def key_from_args(*args): - h = hashlib.md5() - for arg in args: - if arg: - h.update(hashlib_encode_data(arg)) - return h.hexdigest() - - def get(self, key): - cache_path = self.get_cache_path(key) - if not isfile(cache_path): - return None - with codecs.open(cache_path, "rb", encoding="utf8") as fp: - return fp.read() - - def set(self, key, data, valid): - if not get_setting("enable_cache"): - return False - cache_path = self.get_cache_path(key) - if isfile(cache_path): - self.delete(key) - if not data: - return False - if not isdir(self.cache_dir): - os.makedirs(self.cache_dir) - tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} - assert valid.endswith(tuple(tdmap)) - expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) - - if not self._lock_dbindex(): - return False - - if not isdir(dirname(cache_path)): - os.makedirs(dirname(cache_path)) - try: - with codecs.open(cache_path, "wb", encoding="utf8") as fp: - fp.write(data) - with open(self._db_path, "a") as fp: - fp.write("%s=%s\n" % (str(expire_time), cache_path)) - except UnicodeError: - if isfile(cache_path): - try: - remove(cache_path) - except OSError: - pass - - return self._unlock_dbindex() - - def delete(self, keys=None): - """ Keys=None, delete expired items """ - if not isfile(self._db_path): - return None - if not keys: - keys = [] - if not isinstance(keys, list): - keys = [keys] - paths_for_delete = [self.get_cache_path(k) for k in keys] - found = False - newlines = [] - with open(self._db_path) as fp: - for line in fp.readlines(): - line = line.strip() - if "=" not in line: - continue - expire, path = line.split("=") - try: - if ( - time() < int(expire) - and isfile(path) - and path not in paths_for_delete - ): - newlines.append(line) - continue - except ValueError: - pass - found = True - if isfile(path): - try: - remove(path) - if not listdir(dirname(path)): - fs.rmtree(dirname(path)) - except OSError: - pass - - if found and self._lock_dbindex(): - with open(self._db_path, "w") as fp: - fp.write("\n".join(newlines) + "\n") - self._unlock_dbindex() - - return True - - def clean(self): - if not self.cache_dir or not isdir(self.cache_dir): - return - fs.rmtree(self.cache_dir) - - -def clean_cache(): - with ContentCache() as cc: - cc.clean() - - def sanitize_setting(name, value): if name not in DEFAULT_SETTINGS: raise exception.InvalidSettingName(name) @@ -350,8 +202,8 @@ def delete_state_item(name): def get_setting(name): _env_name = "PLATFORMIO_SETTING_%s" % name.upper() - if _env_name in environ: - return sanitize_setting(name, getenv(_env_name)) + if _env_name in os.environ: + return sanitize_setting(name, os.getenv(_env_name)) with State() as state: if "settings" in state and name in state["settings"]: @@ -388,7 +240,7 @@ def is_disabled_progressbar(): [ get_session_var("force_option"), proc.is_ci(), - getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", + os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", ] ) @@ -401,15 +253,15 @@ def get_cid(): if cid: return cid uid = None - if getenv("C9_UID"): - uid = getenv("C9_UID") - elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): + if os.getenv("C9_UID"): + uid = os.getenv("C9_UID") + elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")): try: uid = json.loads( fetch_remote_content( "{api}/user?token={token}".format( - api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), - token=getenv("USER_TOKEN"), + api=os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")), + token=os.getenv("USER_TOKEN"), ) ) ).get("id") diff --git a/platformio/cache.py b/platformio/cache.py new file mode 100644 index 00000000..bc817f61 --- /dev/null +++ b/platformio/cache.py @@ -0,0 +1,165 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import codecs +import hashlib +import os +from time import time + +from platformio import app, fs +from platformio.compat import hashlib_encode_data +from platformio.package.lockfile import LockFile +from platformio.project.helpers import get_project_cache_dir + + +class ContentCache(object): + def __init__(self, namespace=None): + self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content") + self._db_path = os.path.join(self.cache_dir, "db.data") + self._lockfile = None + if not os.path.isdir(self.cache_dir): + os.makedirs(self.cache_dir) + + def __enter__(self): + # cleanup obsolete items + self.delete() + return self + + def __exit__(self, type_, value, traceback): + pass + + @staticmethod + def key_from_args(*args): + h = hashlib.sha1() + for arg in args: + if arg: + h.update(hashlib_encode_data(arg)) + return h.hexdigest() + + def get_cache_path(self, key): + assert "/" not in key and "\\" not in key + key = str(key) + assert len(key) > 3 + return os.path.join(self.cache_dir, key) + + def get(self, key): + cache_path = self.get_cache_path(key) + if not os.path.isfile(cache_path): + return None + with codecs.open(cache_path, "rb", encoding="utf8") as fp: + return fp.read() + + def set(self, key, data, valid): + if not app.get_setting("enable_cache"): + return False + cache_path = self.get_cache_path(key) + if os.path.isfile(cache_path): + self.delete(key) + if not data: + return False + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert valid.endswith(tuple(tdmap)) + expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) + + if not self._lock_dbindex(): + return False + + if not os.path.isdir(os.path.dirname(cache_path)): + os.makedirs(os.path.dirname(cache_path)) + try: + with codecs.open(cache_path, "wb", encoding="utf8") as fp: + fp.write(data) + with open(self._db_path, "a") as fp: + fp.write("%s=%s\n" % (str(expire_time), os.path.basename(cache_path))) + except UnicodeError: + if os.path.isfile(cache_path): + try: + os.remove(cache_path) + except OSError: + pass + + return self._unlock_dbindex() + + def delete(self, keys=None): + """ Keys=None, delete expired items """ + if not os.path.isfile(self._db_path): + return None + if not keys: + keys = [] + if not isinstance(keys, list): + keys = [keys] + paths_for_delete = [self.get_cache_path(k) for k in keys] + found = False + newlines = [] + with open(self._db_path) as fp: + for line in fp.readlines(): + line = line.strip() + if "=" not in line: + continue + expire, fname = line.split("=") + path = os.path.join(self.cache_dir, fname) + try: + if ( + time() < int(expire) + and os.path.isfile(path) + and path not in paths_for_delete + ): + newlines.append(line) + continue + except ValueError: + pass + found = True + if os.path.isfile(path): + try: + os.remove(path) + if not os.listdir(os.path.dirname(path)): + fs.rmtree(os.path.dirname(path)) + except OSError: + pass + + if found and self._lock_dbindex(): + with open(self._db_path, "w") as fp: + fp.write("\n".join(newlines) + "\n") + self._unlock_dbindex() + + return True + + def clean(self): + if not os.path.isdir(self.cache_dir): + return + fs.rmtree(self.cache_dir) + + def _lock_dbindex(self): + self._lockfile = LockFile(self.cache_dir) + try: + self._lockfile.acquire() + except: # pylint: disable=bare-except + return False + + return True + + def _unlock_dbindex(self): + if self._lockfile: + self._lockfile.release() + return True + + +# +# Helpers +# + + +def cleanup_content_cache(namespace=None): + with ContentCache(namespace) as cc: + cc.clean() diff --git a/platformio/clients/account.py b/platformio/clients/account.py index d492b65d..ba2c3451 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -153,9 +153,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods ).get("auth_token") def forgot_password(self, username): - return self.fetch_json_data( - "post", "/v1/forgot", data={"username": username}, - ) + return self.fetch_json_data("post", "/v1/forgot", data={"username": username},) def get_profile(self): return self.send_auth_request("get", "/v1/profile",) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 0b4ca373..b1330f2e 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -20,9 +20,9 @@ import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util +from platformio.cache import ContentCache from platformio.exception import PlatformioException, UserSideException - PING_REMOTE_HOSTS = [ "140.82.118.3", # Github.com "35.231.145.151", # Gitlab.com @@ -90,16 +90,20 @@ class HTTPClient(object): except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: raise HTTPClientError(str(e)) - def fetch_json_data(self, *args, **kwargs): + def fetch_json_data(self, method, path, **kwargs): cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None if not cache_valid: - return self.raise_error_from_response(self.send_request(*args, **kwargs)) - cache_key = app.ContentCache.key_from_args(*args, kwargs) - with app.ContentCache() as cc: + return self.raise_error_from_response( + self.send_request(method, path, **kwargs) + ) + cache_key = ContentCache.key_from_args( + method, path, kwargs.get("params"), kwargs.get("data") + ) + with ContentCache("http") as cc: result = cc.get(cache_key) if result is not None: return json.loads(result) - response = self.send_request(*args, **kwargs) + response = self.send_request(method, path, **kwargs) cc.set(cache_key, response.text, cache_valid) return self.raise_error_from_response(response) diff --git a/platformio/commands/debug/process/client.py b/platformio/commands/debug/process/client.py index a58438b7..45374727 100644 --- a/platformio/commands/debug/process/client.py +++ b/platformio/commands/debug/process/client.py @@ -26,7 +26,8 @@ from twisted.internet import reactor # pylint: disable=import-error from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error -from platformio import app, fs, proc, telemetry, util +from platformio import fs, proc, telemetry, util +from platformio.cache import ContentCache from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.debug.initcfgs import get_gdb_init_config @@ -252,7 +253,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def _kill_previous_session(self): assert self._session_id pid = None - with app.ContentCache() as cc: + with ContentCache() as cc: pid = cc.get(self._session_id) cc.delete(self._session_id) if not pid: @@ -269,11 +270,11 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def _lock_session(self, pid): if not self._session_id: return - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set(self._session_id, str(pid), "1h") def _unlock_session(self): if not self._session_id: return - with app.ContentCache() as cc: + with ContentCache() as cc: cc.delete(self._session_id) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index a216344e..a4bdc652 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -17,15 +17,15 @@ import time from twisted.internet import defer, reactor # pylint: disable=import-error -from platformio import app +from platformio.cache import ContentCache from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): def load_latest_tweets(self, data_url): - cache_key = app.ContentCache.key_from_args(data_url, "tweets") + cache_key = ContentCache.key_from_args(data_url, "tweets") cache_valid = "180d" - with app.ContentCache() as cc: + with ContentCache() as cc: cache_data = cc.get(cache_key) if cache_data: cache_data = json.loads(cache_data) @@ -43,7 +43,7 @@ class MiscRPC(object): @defer.inlineCallbacks def _preload_latest_tweets(data_url, cache_key, cache_valid): result = json.loads((yield OSRPC.fetch_content(data_url))) - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set( cache_key, json.dumps({"time": int(time.time()), "result": result}), diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 7c833180..38bdcb8a 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,8 @@ from functools import cmp_to_key import click from twisted.internet import defer # pylint: disable=import-error -from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, fs, util +from platformio.cache import ContentCache from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, glob_recursive @@ -40,8 +41,8 @@ class OSRPC(object): "Safari/603.3.8" ) } - cache_key = app.ContentCache.key_from_args(uri, data) if cache_valid else None - with app.ContentCache() as cc: + cache_key = ContentCache.key_from_args(uri, data) if cache_valid else None + with ContentCache() as cc: if cache_key: result = cc.get(cache_key) if result is not None: @@ -63,7 +64,7 @@ class OSRPC(object): r.raise_for_status() result = r.text if cache_valid: - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set(cache_key, result, cache_valid) defer.returnValue(result) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 803149b1..2bfe9ebb 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -16,7 +16,8 @@ import os import click -from platformio import app, util +from platformio import util +from platformio.cache import cleanup_content_cache from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager @@ -191,7 +192,9 @@ def platform_search(query, json_output): def platform_frameworks(query, json_output): regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] - for framework in regclient.fetch_json_data("get", "/v2/frameworks", cache_valid="1d"): + for framework in regclient.fetch_json_data( + "get", "/v2/frameworks", cache_valid="1d" + ): if query == "all": query = "" search_data = dump_json_to_unicode(framework) @@ -401,7 +404,8 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments return click.echo(dump_json_to_unicode(result)) # cleanup cached board and platform lists - app.clean_cache() + cleanup_content_cache("http") + for platform in platforms: click.echo( "Platform %s" diff --git a/platformio/commands/update.py b/platformio/commands/update.py index b1e15a43..ff88723e 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -14,7 +14,7 @@ import click -from platformio import app +from platformio.cache import cleanup_content_cache from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update @@ -38,7 +38,7 @@ from platformio.package.manager.library import LibraryPackageManager @click.pass_context def cli(ctx, core_packages, only_check, dry_run): # cleanup lib search results, cached board and platform lists - app.clean_cache() + cleanup_content_cache("http") only_check = dry_run or only_check diff --git a/platformio/maintenance.py b/platformio/maintenance.py index d70c5ca8..b8dd67fd 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -20,6 +20,7 @@ import click import semantic_version from platformio import __version__, app, exception, fs, telemetry, util +from platformio.cache import cleanup_content_cache from platformio.clients import http from platformio.commands import PlatformioCLI from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY @@ -160,7 +161,7 @@ def after_upgrade(ctx): else: click.secho("Please wait while upgrading PlatformIO...", fg="yellow") try: - app.clean_cache() + cleanup_content_cache("http") except: # pylint: disable=bare-except pass diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 34295287..4039568b 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -27,7 +27,7 @@ class PackageManagerDownloadMixin(object): DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month def compute_download_path(self, *args): - request_hash = hashlib.new("sha256") + request_hash = hashlib.new("sha1") for arg in args: request_hash.update(compat.hashlib_encode_data(arg)) dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest()) diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 10fdd978..3b6dd2d4 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -16,10 +16,10 @@ import os import click +from platformio.clients.http import ensure_internet_on from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec from platformio.package.vcsclient import VCSBaseException, VCSClientFactory -from platformio.clients.http import ensure_internet_on class PackageManagerUpdateMixin(object): diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 9f02b846..7eed9821 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -17,7 +17,7 @@ import os import subprocess import sys -from platformio import __core_packages__, fs, exception, util +from platformio import __core_packages__, exception, fs, util from platformio.compat import PY2 from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 5e5878c8..4c5a6706 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -123,9 +123,7 @@ class MeasurementProtocol(TelemetryBase): caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() - self["cd4"] = ( - 1 if (not is_ci() and (caller_id or not is_container())) else 0 - ) + self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0 if caller_id: self["cd5"] = caller_id.lower() From 95151062f5054203d0d4aa3abddd27e021910809 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 22:52:29 +0300 Subject: [PATCH 173/223] Implement mirroring for HTTP client --- platformio/__init__.py | 5 +- platformio/clients/account.py | 2 +- platformio/clients/http.py | 83 ++++++++++++++++++++----- platformio/clients/registry.py | 2 +- platformio/package/manager/_registry.py | 24 +++---- 5 files changed, 84 insertions(+), 32 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0d25faf8..b68f3371 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -40,7 +40,10 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" __accounts_api__ = "https://api.accounts.platformio.org" -__registry_api__ = "https://api.registry.platformio.org" +__registry_api__ = [ + "https://api.registry.platformio.org", + "https://api.registry.ns1.platformio.org", +] __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __core_packages__ = { diff --git a/platformio/clients/account.py b/platformio/clients/account.py index ba2c3451..e2abde17 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -40,7 +40,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__(self): - super(AccountClient, self).__init__(base_url=__accounts_api__) + super(AccountClient, self).__init__(__accounts_api__) @staticmethod def get_refresh_token(): diff --git a/platformio/clients/http.py b/platformio/clients/http.py index b1330f2e..8ee15b35 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import math import os import socket @@ -23,6 +24,12 @@ from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util from platformio.cache import ContentCache from platformio.exception import PlatformioException, UserSideException +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin + + PING_REMOTE_HOSTS = [ "140.82.118.3", # Github.com "35.231.145.151", # Gitlab.com @@ -51,23 +58,54 @@ class InternetIsOffline(UserSideException): ) -class HTTPClient(object): - def __init__( - self, base_url, - ): - if base_url.endswith("/"): - base_url = base_url[:-1] +class EndpointSession(requests.Session): + def __init__(self, base_url, *args, **kwargs): + super(EndpointSession, self).__init__(*args, **kwargs) self.base_url = base_url - self._session = requests.Session() - self._session.headers.update({"User-Agent": app.get_user_agent()}) - retry = Retry( - total=5, + + def request( # pylint: disable=signature-differs,arguments-differ + self, method, url, *args, **kwargs + ): + print(self.base_url, method, url, args, kwargs) + return super(EndpointSession, self).request( + method, urljoin(self.base_url, url), *args, **kwargs + ) + + +class EndpointSessionIterator(object): + def __init__(self, endpoints): + if not isinstance(endpoints, list): + endpoints = [endpoints] + self.endpoints = endpoints + self.endpoints_iter = iter(endpoints) + self.retry = Retry( + total=math.ceil(6 / len(self.endpoints)), backoff_factor=1, # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], status_forcelist=[413, 429, 500, 502, 503, 504], ) - adapter = requests.adapters.HTTPAdapter(max_retries=retry) - self._session.mount(base_url, adapter) + + def __iter__(self): # pylint: disable=non-iterator-returned + return self + + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + + def __next__(self): + base_url = next(self.endpoints_iter) + session = EndpointSession(base_url) + session.headers.update({"User-Agent": app.get_user_agent()}) + adapter = requests.adapters.HTTPAdapter(max_retries=self.retry) + session.mount(base_url, adapter) + return session + + +class HTTPClient(object): + def __init__(self, endpoints): + self._session_iter = EndpointSessionIterator(endpoints) + self._session = None + self._next_session() def __del__(self): if not self._session: @@ -75,20 +113,31 @@ class HTTPClient(object): self._session.close() self._session = None + def _next_session(self): + if self._session: + self._session.close() + self._session = next(self._session_iter) + @util.throttle(500) def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout - # print(self, method, path, kwargs) ensure_internet_on(raise_exception=True) # set default timeout if "timeout" not in kwargs: kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT - try: - return getattr(self._session, method)(self.base_url + path, **kwargs) - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - raise HTTPClientError(str(e)) + while True: + try: + return getattr(self._session, method)(path, **kwargs) + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + ) as e: + try: + self._next_session() + except: # pylint: disable=bare-except + raise HTTPClientError(str(e)) def fetch_json_data(self, method, path, **kwargs): cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index e990a65f..c8fbeeea 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -22,7 +22,7 @@ from platformio.package.meta import PackageType class RegistryClient(HTTPClient): def __init__(self): - super(RegistryClient, self).__init__(base_url=__registry_api__) + super(RegistryClient, self).__init__(__registry_api__) def send_auth_request(self, *args, **kwargs): headers = kwargs.get("headers", {}) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 72f189fb..415a9977 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -27,19 +27,23 @@ except ImportError: from urlparse import urlparse -class RegistryFileMirrorsIterator(object): +class RegistryFileMirrorIterator(object): HTTP_CLIENT_INSTANCES = {} def __init__(self, download_url): self.download_url = download_url self._url_parts = urlparse(download_url) - self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) + self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) self._visited_mirrors = [] def __iter__(self): # pylint: disable=non-iterator-returned return self + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + def __next__(self): http = self.get_http_client() response = http.send_request( @@ -64,16 +68,12 @@ class RegistryFileMirrorsIterator(object): response.headers.get("X-PIO-Content-SHA256"), ) - def next(self): - """ For Python 2 compatibility """ - return self.__next__() - def get_http_client(self): - if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES: - RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[ - self._base_url - ] = HTTPClient(self._base_url) - return RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[self._base_url] + if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: + RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( + self._mirror + ) + return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] class PackageManageRegistryMixin(object): @@ -98,7 +98,7 @@ class PackageManageRegistryMixin(object): if not pkgfile: raise UnknownPackageError(spec.humanize()) - for url, checksum in RegistryFileMirrorsIterator(pkgfile["download_url"]): + for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]): try: return self.install_from_url( url, From c2caf8b839981de390c435b60fe41f18d6aef9fc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 22:53:41 +0300 Subject: [PATCH 174/223] Bump version to 4.4.0b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b68f3371..edf20f6e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -16,7 +16,7 @@ import sys DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) -VERSION = (4, 4, "0b1") +VERSION = (4, 4, "0b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From dcf91c49acd267654ac2c38baf56b197638e517d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 22:56:26 +0300 Subject: [PATCH 175/223] Remove debug code --- platformio/clients/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 8ee15b35..318448c7 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -66,7 +66,7 @@ class EndpointSession(requests.Session): def request( # pylint: disable=signature-differs,arguments-differ self, method, url, *args, **kwargs ): - print(self.base_url, method, url, args, kwargs) + # print(self.base_url, method, url, args, kwargs) return super(EndpointSession, self).request( method, urljoin(self.base_url, url), *args, **kwargs ) From e2bb81bae4ec00887dd734136da262734c0eaec5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 13:22:11 +0300 Subject: [PATCH 176/223] Restore legacy util.cd API --- platformio/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/util.py b/platformio/util.py index 04576b24..f2c485a6 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -28,6 +28,7 @@ import click from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS +from platformio.fs import cd # pylint: disable=unused-import from platformio.fs import load_json # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import From 8ea10a18d3de95311f67397b4fc71850d33dbea0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 13:22:38 +0300 Subject: [PATCH 177/223] Bump version to 4.4.0b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index edf20f6e..f50be5fe 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -16,7 +16,7 @@ import sys DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) -VERSION = (4, 4, "0b2") +VERSION = (4, 4, "0b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From da179cb33fb615bf907f71dd1619fd70c9381df3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 14:29:31 +0300 Subject: [PATCH 178/223] Enhance configuration variables --- platformio/__init__.py | 12 ++++++++++-- platformio/clients/http.py | 17 ++++------------- platformio/commands/home/rpc/handlers/os.py | 6 +++--- platformio/package/download.py | 4 ++-- tests/test_misc.py | 4 ++-- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index f50be5fe..83f86b3b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,8 +14,6 @@ import sys -DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) - VERSION = (4, 4, "0b3") __version__ = ".".join([str(s) for s in VERSION]) @@ -46,6 +44,8 @@ __registry_api__ = [ ] __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" +__default_requests_timeout__ = (10, None) # (connect, read) + __core_packages__ = { "contrib-piohome": "~3.2.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), @@ -55,3 +55,11 @@ __core_packages__ = { "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.7.0", } + +__check_internet_hosts__ = [ + "140.82.118.3", # Github.com + "35.231.145.151", # Gitlab.com + "88.198.170.159", # platformio.org + "github.com", + "platformio.org", +] diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 318448c7..8e732958 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -20,7 +20,7 @@ import socket import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util +from platformio import __check_internet_hosts__, __default_requests_timeout__, app, util from platformio.cache import ContentCache from platformio.exception import PlatformioException, UserSideException @@ -30,15 +30,6 @@ except ImportError: from urlparse import urljoin -PING_REMOTE_HOSTS = [ - "140.82.118.3", # Github.com - "35.231.145.151", # Gitlab.com - "88.198.170.159", # platformio.org - "github.com", - "platformio.org", -] - - class HTTPClientError(PlatformioException): def __init__(self, message, response=None): super(HTTPClientError, self).__init__() @@ -125,7 +116,7 @@ class HTTPClient(object): # set default timeout if "timeout" not in kwargs: - kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + kwargs["timeout"] = __default_requests_timeout__ while True: try: @@ -179,7 +170,7 @@ class HTTPClient(object): def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout) - for host in PING_REMOTE_HOSTS: + for host in __check_internet_hosts__: try: for var in ("HTTP_PROXY", "HTTPS_PROXY"): if not os.getenv(var) and not os.getenv(var.lower()): @@ -208,7 +199,7 @@ def fetch_remote_content(*args, **kwargs): kwargs["headers"]["User-Agent"] = app.get_user_agent() if "timeout" not in kwargs: - kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + kwargs["timeout"] = __default_requests_timeout__ r = requests.get(*args, **kwargs) r.raise_for_status() diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 38bdcb8a..448c633a 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,7 @@ from functools import cmp_to_key import click from twisted.internet import defer # pylint: disable=import-error -from platformio import DEFAULT_REQUESTS_TIMEOUT, fs, util +from platformio import __default_requests_timeout__, fs, util from platformio.cache import ContentCache from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers @@ -54,11 +54,11 @@ class OSRPC(object): session = helpers.requests_session() if data: r = yield session.post( - uri, data=data, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + uri, data=data, headers=headers, timeout=__default_requests_timeout__ ) else: r = yield session.get( - uri, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + uri, headers=headers, timeout=__default_requests_timeout__ ) r.raise_for_status() diff --git a/platformio/package/download.py b/platformio/package/download.py index 0f40fd0d..bd425ac6 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -21,7 +21,7 @@ from time import mktime import click import requests -from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs +from platformio import __default_requests_timeout__, app, fs from platformio.package.exception import PackageException @@ -33,7 +33,7 @@ class FileDownloader(object): url, stream=True, headers={"User-Agent": app.get_user_agent()}, - timeout=DEFAULT_REQUESTS_TIMEOUT, + timeout=__default_requests_timeout__, ) if self._request.status_code != 200: raise PackageException( diff --git a/tests/test_misc.py b/tests/test_misc.py index f816fe6d..36574ee4 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -17,7 +17,7 @@ import pytest import requests -from platformio import proc +from platformio import __check_internet_hosts__, proc from platformio.clients import http from platformio.clients.registry import RegistryClient @@ -30,7 +30,7 @@ def test_platformio_cli(): def test_ping_internet_ips(): - for host in http.PING_REMOTE_HOSTS: + for host in __check_internet_hosts__: requests.get("http://%s" % host, allow_redirects=False, timeout=2) From 620241e067a26e17a35d2e95a2c9aeae266999f4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 15:24:31 +0300 Subject: [PATCH 179/223] Move package "version" related things to "platformio.package.version" module --- platformio/builder/tools/pioplatform.py | 3 +- platformio/builder/tools/platformio.py | 11 +++-- platformio/commands/platform.py | 4 +- platformio/maintenance.py | 19 +++------ platformio/package/meta.py | 21 +--------- platformio/package/version.py | 53 +++++++++++++++++++++++++ platformio/platform/base.py | 9 +++-- platformio/util.py | 16 +------- 8 files changed, 78 insertions(+), 58 deletions(-) create mode 100644 platformio/package/version.py diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 5ca7794f..ec5c6c4c 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -23,6 +23,7 @@ from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import fs, util from platformio.compat import WINDOWS from platformio.package.meta import PackageItem +from platformio.package.version import get_original_version from platformio.platform.exception import UnknownBoard from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions @@ -210,7 +211,7 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements def _get_packages_data(): data = [] for item in platform.dump_used_packages(): - original_version = util.get_original_version(item["version"]) + original_version = get_original_version(item["version"]) info = "%s %s" % (item["name"], item["version"]) extra = [] if original_version: diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 5d8f8e8b..aac47426 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -26,9 +26,9 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error -from platformio import fs +from platformio import __version__, fs from platformio.compat import string_types -from platformio.util import pioversion_to_intstr +from platformio.package.version import pepver_to_semver SRC_HEADER_EXT = ["h", "hpp"] SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] @@ -94,11 +94,16 @@ def BuildProgram(env): def ProcessProgramDeps(env): def _append_pio_macros(): + core_version = pepver_to_semver(__version__) env.AppendUnique( CPPDEFINES=[ ( "PLATFORMIO", - int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())), + int( + "{0:02d}{1:02d}{2:02d}".format( + core_version.major, core_version.minor, core_version.patch + ) + ), ) ] ) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 2bfe9ebb..7725be39 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -16,12 +16,12 @@ import os import click -from platformio import util from platformio.cache import cleanup_content_cache from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageItem, PackageSpec +from platformio.package.version import get_original_version from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory @@ -121,7 +121,7 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru continue item[key] = value if key == "version": - item["originalVersion"] = util.get_original_version(value) + item["originalVersion"] = get_original_version(value) data["packages"].append(item) return data diff --git a/platformio/maintenance.py b/platformio/maintenance.py index b8dd67fd..1900db49 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -19,7 +19,7 @@ from time import time import click import semantic_version -from platformio import __version__, app, exception, fs, telemetry, util +from platformio import __version__, app, exception, fs, telemetry from platformio.cache import cleanup_content_cache from platformio.clients import http from platformio.commands import PlatformioCLI @@ -32,6 +32,7 @@ from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec +from platformio.package.version import pepver_to_semver from platformio.platform.factory import PlatformFactory from platformio.proc import is_container @@ -87,12 +88,8 @@ def set_caller(caller=None): class Upgrader(object): def __init__(self, from_version, to_version): - self.from_version = semantic_version.Version.coerce( - util.pepver_to_semver(from_version) - ) - self.to_version = semantic_version.Version.coerce( - util.pepver_to_semver(to_version) - ) + self.from_version = pepver_to_semver(from_version) + self.to_version = pepver_to_semver(to_version) self._upgraders = [ (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), @@ -141,9 +138,7 @@ def after_upgrade(ctx): if last_version == "0.0.0": app.set_state_item("last_version", __version__) - elif semantic_version.Version.coerce( - util.pepver_to_semver(last_version) - ) > semantic_version.Version.coerce(util.pepver_to_semver(__version__)): + elif pepver_to_semver(last_version) > pepver_to_semver(__version__): click.secho("*" * terminal_width, fg="yellow") click.secho( "Obsolete PIO Core v%s is used (previous was %s)" @@ -229,9 +224,7 @@ def check_platformio_upgrade(): update_core_packages(silent=True) latest_version = get_latest_version() - if semantic_version.Version.coerce( - util.pepver_to_semver(latest_version) - ) <= semantic_version.Version.coerce(util.pepver_to_semver(__version__)): + if pepver_to_semver(latest_version) <= pepver_to_semver(__version__): return terminal_width, _ = click.get_terminal_size() diff --git a/platformio/package/meta.py b/platformio/package/meta.py index fa93780e..2715206f 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -357,28 +357,9 @@ class PackageMetaData(object): self._version = ( value if isinstance(value, semantic_version.Version) - else self.to_semver(value) + else cast_version_to_semver(value) ) - @staticmethod - def to_semver(value, force=True, raise_exception=False): - assert value - try: - return semantic_version.Version(value) - except ValueError: - pass - if force: - try: - return semantic_version.Version.coerce(value) - except ValueError: - pass - if raise_exception: - raise ValueError("Invalid SemVer version %s" % value) - # parse commit hash - if re.match(r"^[\da-f]+$", value, flags=re.I): - return semantic_version.Version("0.0.0+sha." + value) - return semantic_version.Version("0.0.0+" + value) - def as_dict(self): return dict( type=self.type, diff --git a/platformio/package/version.py b/platformio/package/version.py new file mode 100644 index 00000000..770be9e4 --- /dev/null +++ b/platformio/package/version.py @@ -0,0 +1,53 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import semantic_version + + +def cast_version_to_semver(value, force=True, raise_exception=False): + assert value + try: + return semantic_version.Version(value) + except ValueError: + pass + if force: + try: + return semantic_version.Version.coerce(value) + except ValueError: + pass + if raise_exception: + raise ValueError("Invalid SemVer version %s" % value) + # parse commit hash + if re.match(r"^[\da-f]+$", value, flags=re.I): + return semantic_version.Version("0.0.0+sha." + value) + return semantic_version.Version("0.0.0+" + value) + + +def pepver_to_semver(pepver): + return cast_version_to_semver( + re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) + ) + + +def get_original_version(version): + if version.count(".") != 2: + return None + _, raw = version.split(".")[:2] + if int(raw) <= 99: + return None + if int(raw) <= 9999: + return "%s.%s" % (raw[:-2], int(raw[-2:])) + return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 8e49288d..0c061a11 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -18,8 +18,9 @@ import subprocess import click import semantic_version -from platformio import __version__, fs, proc, util +from platformio import __version__, fs, proc from platformio.package.manager.tool import ToolPackageManager +from platformio.package.version import pepver_to_semver from platformio.platform._packages import PlatformPackagesMixin from platformio.platform._run import PlatformRunMixin from platformio.platform.board import PlatformBoardConfig @@ -31,7 +32,7 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub PlatformPackagesMixin, PlatformRunMixin ): - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) + CORE_SEMVER = pepver_to_semver(__version__) _BOARDS_CACHE = {} def __init__(self, manifest_path): @@ -110,10 +111,10 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub def ensure_engine_compatible(self): if not self.engines or "platformio" not in self.engines: return True - if self.PIO_VERSION in semantic_version.SimpleSpec(self.engines["platformio"]): + if self.CORE_SEMVER in semantic_version.SimpleSpec(self.engines["platformio"]): return True raise IncompatiblePlatform( - self.name, str(self.PIO_VERSION), self.engines["platformio"] + self.name, str(self.CORE_SEMVER), self.engines["platformio"] ) def get_dir(self): diff --git a/platformio/util.py b/platformio/util.py index f2c485a6..aeeaf55b 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -244,26 +244,12 @@ def get_mdns_services(): def pioversion_to_intstr(): + """ Legacy for framework-zephyr/scripts/platformio/platformio-build-pre.py""" vermatch = re.match(r"^([\d\.]+)", __version__) assert vermatch return [int(i) for i in vermatch.group(1).split(".")[:3]] -def pepver_to_semver(pepver): - return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) - - -def get_original_version(version): - if version.count(".") != 2: - return None - _, raw = version.split(".")[:2] - if int(raw) <= 99: - return None - if int(raw) <= 9999: - return "%s.%s" % (raw[:-2], int(raw[-2:])) - return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) - - def items_to_list(items): if isinstance(items, list): return items From 1c8aca2f6a63a18149fe5e0e5e9069f218f83a42 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 15:25:03 +0300 Subject: [PATCH 180/223] Check ALL possible version for the first matched package --- platformio/package/manager/_registry.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 415a9977..f8af2ece 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -19,7 +19,8 @@ import click from platformio.clients.http import HTTPClient from platformio.clients.registry import RegistryClient from platformio.package.exception import UnknownPackageError -from platformio.package.meta import PackageMetaData, PackageSpec +from platformio.package.meta import PackageSpec +from platformio.package.version import cast_version_to_semver try: from urllib.parse import urlparse @@ -185,17 +186,13 @@ class PackageManageRegistryMixin(object): ) def find_best_registry_version(self, packages, spec): - # find compatible version within the latest package versions for package in packages: + # find compatible version within the latest package versions version = self.pick_best_registry_version([package["version"]], spec) if version: return (package, version) - if not spec.requirements: - return (None, None) - - # if the custom version requirements, check ALL package versions - for package in packages: + # if the custom version requirements, check ALL package versions version = self.pick_best_registry_version( self.fetch_registry_package( PackageSpec( @@ -215,14 +212,14 @@ class PackageManageRegistryMixin(object): assert not spec or isinstance(spec, PackageSpec) best = None for version in versions: - semver = PackageMetaData.to_semver(version["name"]) + semver = cast_version_to_semver(version["name"]) if spec and spec.requirements and semver not in spec.requirements: continue if not any( self.is_system_compatible(f.get("system")) for f in version["files"] ): continue - if not best or (semver > PackageMetaData.to_semver(best["name"])): + if not best or (semver > cast_version_to_semver(best["name"])): best = version return best From a069bae1fbb99449b6ac975d95e8b188529bec4d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 15:26:58 +0300 Subject: [PATCH 181/223] Fix a bug with package updating when version is not in SemVer format // Resolve #3635 --- platformio/package/meta.py | 3 ++- tests/package/test_manager.py | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 2715206f..147a1faf 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -22,6 +22,7 @@ import semantic_version from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType +from platformio.package.version import cast_version_to_semver try: from urllib.parse import urlparse @@ -89,7 +90,7 @@ class PackageOutdatedResult(object): and name in ("current", "latest", "wanted") and not isinstance(value, semantic_version.Version) ): - value = semantic_version.Version(str(value)) + value = cast_version_to_semver(str(value)) return super(PackageOutdatedResult, self).__setattr__(name, value) def is_outdated(self, allow_incompatible=False): diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 2c331dbe..f5939f15 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -392,8 +392,14 @@ def test_registry(isolated_pio_core): def test_update_with_metadata(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) - pkg = lm.install("ArduinoJson @ 5.10.1", silent=True) + # test non SemVer in registry + pkg = lm.install("RadioHead @ <1.90", silent=True) + outdated = lm.outdated(pkg) + assert str(outdated.current) == "1.89.0" + assert outdated.latest > semantic_version.Version("1.100.0") + + pkg = lm.install("ArduinoJson @ 5.10.1", silent=True) # tesy latest outdated = lm.outdated(pkg) assert str(outdated.current) == "5.10.1" @@ -411,7 +417,7 @@ def test_update_with_metadata(isolated_pio_core, tmpdir_factory): new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5"), silent=True) assert str(new_pkg.metadata.version) == "5.13.4" # check that old version is removed - assert len(lm.get_installed()) == 1 + assert len(lm.get_installed()) == 2 # update to the latest lm = LibraryPackageManager(str(storage_dir)) @@ -422,7 +428,7 @@ def test_update_with_metadata(isolated_pio_core, tmpdir_factory): def test_update_without_metadata(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") storage_dir.join("legacy-package").mkdir().join("library.json").write( - '{"name": "AsyncMqttClient-esphome", "version": "0.8.2"}' + '{"name": "AsyncMqttClient-esphome", "version": "0.8"}' ) storage_dir.join("legacy-dep").mkdir().join("library.json").write( '{"name": "AsyncTCP-esphome", "version": "1.1.1"}' @@ -431,8 +437,8 @@ def test_update_without_metadata(isolated_pio_core, tmpdir_factory): pkg = lm.get_package("AsyncMqttClient-esphome") outdated = lm.outdated(pkg) assert len(lm.get_installed()) == 2 - assert str(pkg.metadata.version) == "0.8.2" - assert outdated.latest > semantic_version.Version("0.8.2") + assert str(pkg.metadata.version) == "0.8.0" + assert outdated.latest > semantic_version.Version("0.8.0") # update lm = LibraryPackageManager(str(storage_dir)) From 24f85a337fade368e2a218cb8e4cb7d5f05a4db2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 21:07:14 +0300 Subject: [PATCH 182/223] Fix "AttributeError: module 'platformio.exception' has no attribute 'InternetIsOffline'" --- platformio/builder/tools/piolib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 8cc1ad58..45371ece 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=no-member, no-self-use, unused-argument, too-many-lines +# pylint: disable=no-self-use, unused-argument, too-many-lines # pylint: disable=too-many-instance-attributes, too-many-public-methods # pylint: disable=assignment-from-no-return @@ -33,6 +33,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool +from platformio.clients.http import InternetIsOffline from platformio.compat import WINDOWS, hashlib_encode_data, string_types from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager @@ -882,7 +883,7 @@ class ProjectAsLibBuilder(LibBuilderBase): try: lm.install(spec) did_install = True - except (UnknownPackageError, exception.InternetIsOffline) as e: + except (UnknownPackageError, InternetIsOffline) as e: click.secho("Warning! %s" % e, fg="yellow") # reset cache From f39c9fb597facb92701afbfc687e37898e7bf43d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 21:07:40 +0300 Subject: [PATCH 183/223] Bump version to 4.4.0b4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 83f86b3b..4567a0ad 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (4, 4, "0b3") +VERSION = (4, 4, "0b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From b44bc80bd1ffe328b3d9dfe8130ba09e236487f5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 21:41:53 +0300 Subject: [PATCH 184/223] PyLint fix for PY2 --- platformio/builder/tools/piolib.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 45371ece..35d1462e 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -484,6 +484,7 @@ class ArduinoLibBuilder(LibBuilderBase): def src_filter(self): src_dir = join(self.path, "src") if isdir(src_dir): + # pylint: disable=no-member src_filter = LibBuilderBase.src_filter.fget(self) for root, _, files in os.walk(src_dir, followlinks=True): found = False @@ -518,6 +519,7 @@ class ArduinoLibBuilder(LibBuilderBase): @property def lib_ldf_mode(self): + # pylint: disable=no-member if not self._manifest.get("dependencies"): return LibBuilderBase.lib_ldf_mode.fget(self) missing = object() @@ -554,7 +556,7 @@ class MbedLibBuilder(LibBuilderBase): def src_dir(self): if isdir(join(self.path, "source")): return join(self.path, "source") - return LibBuilderBase.src_dir.fget(self) + return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) @@ -700,17 +702,18 @@ class PlatformIOLibBuilder(LibBuilderBase): if "includeDir" in self._manifest.get("build", {}): with fs.cd(self.path): return realpath(self._manifest.get("build").get("includeDir")) - return LibBuilderBase.include_dir.fget(self) + return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): with fs.cd(self.path): return realpath(self._manifest.get("build").get("srcDir")) - return LibBuilderBase.src_dir.fget(self) + return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member @property def src_filter(self): + # pylint: disable=no-member if "srcFilter" in self._manifest.get("build", {}): return self._manifest.get("build").get("srcFilter") if self.env["SRC_FILTER"]: @@ -723,19 +726,19 @@ class PlatformIOLibBuilder(LibBuilderBase): def build_flags(self): if "flags" in self._manifest.get("build", {}): return self._manifest.get("build").get("flags") - return LibBuilderBase.build_flags.fget(self) + return LibBuilderBase.build_flags.fget(self) # pylint: disable=no-member @property def build_unflags(self): if "unflags" in self._manifest.get("build", {}): return self._manifest.get("build").get("unflags") - return LibBuilderBase.build_unflags.fget(self) + return LibBuilderBase.build_unflags.fget(self) # pylint: disable=no-member @property def extra_script(self): if "extraScript" in self._manifest.get("build", {}): return self._manifest.get("build").get("extraScript") - return LibBuilderBase.extra_script.fget(self) + return LibBuilderBase.extra_script.fget(self) # pylint: disable=no-member @property def lib_archive(self): @@ -747,12 +750,14 @@ class PlatformIOLibBuilder(LibBuilderBase): return self.env.GetProjectConfig().get( "env:" + self.env["PIOENV"], "lib_archive" ) + # pylint: disable=no-member return self._manifest.get("build", {}).get( "libArchive", LibBuilderBase.lib_archive.fget(self) ) @property def lib_ldf_mode(self): + # pylint: disable=no-member return self.validate_ldf_mode( self._manifest.get("build", {}).get( "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self) @@ -761,6 +766,7 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def lib_compat_mode(self): + # pylint: disable=no-member return self.validate_compat_mode( self._manifest.get("build", {}).get( "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self) @@ -835,7 +841,7 @@ class ProjectAsLibBuilder(LibBuilderBase): @property def lib_ldf_mode(self): - mode = LibBuilderBase.lib_ldf_mode.fget(self) + mode = LibBuilderBase.lib_ldf_mode.fget(self) # pylint: disable=no-member if not mode.startswith("chain"): return mode # parse all project files @@ -843,6 +849,7 @@ class ProjectAsLibBuilder(LibBuilderBase): @property def src_filter(self): + # pylint: disable=no-member return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self) @property @@ -1037,7 +1044,7 @@ def ConfigureProjectLibBuilder(env): _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") - ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) + ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") click.echo( From d6d95e05e8991d635f072be07f5b8c2e32c96747 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 15:09:37 +0300 Subject: [PATCH 185/223] Rename "fs.format_filesize" to "fs.humanize_file_size" --- platformio/builder/tools/pioplatform.py | 3 ++- platformio/commands/boards.py | 4 ++-- platformio/fs.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index ec5c6c4c..7dd36a64 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -179,7 +179,8 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") data.append( - "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash)) + "%s RAM, %s Flash" + % (fs.humanize_file_size(ram), fs.humanize_file_size(flash)) ) return data diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 21614b13..962ab504 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -59,8 +59,8 @@ def print_boards(boards): click.style(b["id"], fg="cyan"), b["mcu"], "%dMHz" % (b["fcpu"] / 1000000), - fs.format_filesize(b["rom"]), - fs.format_filesize(b["ram"]), + fs.humanize_file_size(b["rom"]), + fs.humanize_file_size(b["ram"]), b["name"], ) for b in boards diff --git a/platformio/fs.py b/platformio/fs.py index a4dc6ee4..75bf959c 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -57,7 +57,7 @@ def load_json(file_path): raise exception.InvalidJSONFile(file_path) -def format_filesize(filesize): +def humanize_file_size(filesize): base = 1024 unit = 0 suffix = "B" From 13db51a5560d8c7dfd6dd87622fd671b9aa8fa3e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 15:10:38 +0300 Subject: [PATCH 186/223] Install/Uninstall dependencies only for library-type packages // Resolve #3637 --- platformio/package/manager/_install.py | 27 ++------------ platformio/package/manager/_uninstall.py | 18 ++-------- platformio/package/manager/library.py | 45 +++++++++++++++++++++++- 3 files changed, 48 insertions(+), 42 deletions(-) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 003c2386..1bafcf8c 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,7 @@ import click from platformio import app, compat, fs, util from platformio.package.exception import PackageException -from platformio.package.meta import PackageItem, PackageSpec +from platformio.package.meta import PackageItem from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -119,30 +119,7 @@ class PackageManagerInstallMixin(object): return pkg def install_dependencies(self, pkg, silent=False): - assert isinstance(pkg, PackageItem) - manifest = self.load_manifest(pkg) - if not manifest.get("dependencies"): - return - if not silent: - self.print_message("Installing dependencies...") - for dependency in manifest.get("dependencies"): - if not self._install_dependency(dependency, silent) and not silent: - self.print_message( - "Warning! Could not install dependency %s for package '%s'" - % (dependency, pkg.metadata.name), - fg="yellow", - ) - - def _install_dependency(self, dependency, silent=False): - spec = PackageSpec( - name=dependency.get("name"), requirements=dependency.get("version") - ) - search_filters = { - key: value - for key, value in dependency.items() - if key in ("authors", "platforms", "frameworks") - } - return self._install(spec, search_filters=search_filters or None, silent=silent) + pass def install_from_url(self, url, spec, checksum=None, silent=False): spec = self.ensure_spec(spec) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 2cca8505..322eced6 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -19,7 +19,7 @@ import click from platformio import fs from platformio.package.exception import UnknownPackageError -from platformio.package.meta import PackageItem, PackageSpec +from platformio.package.meta import PackageSpec class PackageManagerUninstallMixin(object): @@ -73,18 +73,4 @@ class PackageManagerUninstallMixin(object): return pkg def uninstall_dependencies(self, pkg, silent=False): - assert isinstance(pkg, PackageItem) - manifest = self.load_manifest(pkg) - if not manifest.get("dependencies"): - return - if not silent: - self.print_message("Removing dependencies...", fg="yellow") - for dependency in manifest.get("dependencies"): - pkg = self.get_package( - PackageSpec( - name=dependency.get("name"), requirements=dependency.get("version") - ) - ) - if not pkg: - continue - self._uninstall(pkg, silent=silent) + pass diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 1375e84e..a0d1407f 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -17,7 +17,7 @@ import os from platformio.package.exception import MissingPackageManifestError from platformio.package.manager.base import BasePackageManager -from platformio.package.meta import PackageSpec, PackageType +from platformio.package.meta import PackageItem, PackageSpec, PackageType from platformio.project.helpers import get_project_global_lib_dir @@ -62,3 +62,46 @@ class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-anc return os.path.dirname(root) return root return path + + def install_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message("Installing dependencies...") + for dependency in manifest.get("dependencies"): + if not self._install_dependency(dependency, silent) and not silent: + self.print_message( + "Warning! Could not install dependency %s for package '%s'" + % (dependency, pkg.metadata.name), + fg="yellow", + ) + + def _install_dependency(self, dependency, silent=False): + spec = PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + search_filters = { + key: value + for key, value in dependency.items() + if key in ("authors", "platforms", "frameworks") + } + return self._install(spec, search_filters=search_filters or None, silent=silent) + + def uninstall_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message("Removing dependencies...", fg="yellow") + for dependency in manifest.get("dependencies"): + pkg = self.get_package( + PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + ) + if not pkg: + continue + self._uninstall(pkg, silent=silent) From 3e7e9e2b3d1b75cd76fc623b6adf9927f35f4225 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 15:22:05 +0300 Subject: [PATCH 187/223] Remove unused data using a new `pio system prune // Resolve #3522 --- HISTORY.rst | 17 +++++++++-------- docs | 2 +- platformio/commands/system/command.py | 25 ++++++++++++++++++++++++- platformio/fs.py | 11 +++++++++++ 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 17a8786b..538649fa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,8 +30,8 @@ PlatformIO Core 4 - Built-in fine-grained access control (role based, teams, organizations) - Command Line Interface: - * `platformio package publish `__ – publish a personal or organization package - * `platformio package unpublish `__ – remove a pushed package from the registry + * `pio package publish `__ – publish a personal or organization package + * `pio package unpublish `__ – remove a pushed package from the registry * Grant package access to the team members or maintainers * New **Package Management System** @@ -46,7 +46,7 @@ PlatformIO Core 4 - Command launcher with own arguments - Launch command with custom options declared in `"platformio.ini" `__ - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) - - List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) + - List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) * **PlatformIO Build System** @@ -58,12 +58,13 @@ PlatformIO Core 4 * **Miscellaneous** - - Display system-wide information using a new `platformio system info `__ command (`issue #3521 `_) - - Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command - - Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment - - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. - - Do not generate ".travis.yml" for a new project, let the user have a choice + - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) + - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) + - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command + - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. - Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 + - Do not generate ".travis.yml" for a new project, let the user have a choice - Do not escape compiler arguments in VSCode template on Windows - Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) diff --git a/docs b/docs index d01bbede..be04ee45 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d01bbede6ca90420ed24736be4963a48228eff42 +Subproject commit be04ee45c8d037c8eecbbe9a178926593fd620a0 diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 2fee5471..cb311205 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import os import platform import subprocess import sys @@ -20,7 +21,7 @@ import sys import click from tabulate import tabulate -from platformio import __version__, compat, proc, util +from platformio import __version__, compat, fs, proc, util from platformio.commands.system.completion import ( get_completion_install_path, install_completion_code, @@ -30,6 +31,7 @@ from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_cache_dir @click.group("system", short_help="Miscellaneous system commands") @@ -95,6 +97,27 @@ def system_info(json_output): ) +@cli.command("prune", short_help="Remove unused data") +@click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation") +def system_prune(force): + click.secho("WARNING! This will remove:", fg="yellow") + click.echo(" - cached API requests") + click.echo(" - cached package downloads") + click.echo(" - temporary data") + if not force: + click.confirm("Do you want to continue?", abort=True) + + reclaimed_total = 0 + cache_dir = get_project_cache_dir() + if os.path.isdir(cache_dir): + reclaimed_total += fs.calculate_folder_size(cache_dir) + fs.rmtree(cache_dir) + + click.secho( + "Total reclaimed space: %s" % fs.humanize_file_size(reclaimed_total), fg="green" + ) + + @cli.group("completion", short_help="Shell completion support") def completion(): # pylint: disable=import-error,import-outside-toplevel diff --git a/platformio/fs.py b/platformio/fs.py index 75bf959c..da2101c5 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -85,6 +85,17 @@ def calculate_file_hashsum(algorithm, path): return h.hexdigest() +def calculate_folder_size(path): + assert os.path.isdir(path) + result = 0 + for root, __, files in os.walk(path): + for f in files: + file_path = os.path.join(root, f) + if not os.path.islink(file_path): + result += os.path.getsize(file_path) + return result + + def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel From 6af2bad12320bc264a8ea3f070d25d5c09f4b75a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 22:56:31 +0300 Subject: [PATCH 188/223] Make PIO Core 4.0 automatically compatible with dev-platforms for PIO Core 2.0 & 3.0 // Resolve #3638 --- platformio/platform/base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 0c061a11..ffafd61c 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -111,11 +111,13 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub def ensure_engine_compatible(self): if not self.engines or "platformio" not in self.engines: return True - if self.CORE_SEMVER in semantic_version.SimpleSpec(self.engines["platformio"]): + core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) + if self.CORE_SEMVER in core_spec: return True - raise IncompatiblePlatform( - self.name, str(self.CORE_SEMVER), self.engines["platformio"] - ) + # PIO Core 4 is compatible with dev-platforms for PIO Core 2.0 & 3.0 + if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3)): + return True + raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) def get_dir(self): return os.path.dirname(self.manifest_path) From c6a37ef88061c11b6362ff4acc575b7bfa361f8e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 23:04:17 +0300 Subject: [PATCH 189/223] Get real path of just installed core-package --- platformio/package/manager/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 7eed9821..62b884a8 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -34,9 +34,9 @@ def get_core_package_dir(name): pkg = pm.get_package(spec) if pkg: return pkg.path - pkg = pm.install(spec).path + assert pm.install(spec).path _remove_unnecessary_packages() - return pkg + return pm.get_package(spec) def update_core_packages(only_check=False, silent=False): From 655e2856d1c93412162d1de6677e251ce9a50e7b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 23:05:01 +0300 Subject: [PATCH 190/223] Bump version to 4.4.0b5 --- platformio/__init__.py | 2 +- platformio/package/manager/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 4567a0ad..735d4276 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (4, 4, "0b4") +VERSION = (4, 4, "0b5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 62b884a8..098bfdea 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -34,7 +34,7 @@ def get_core_package_dir(name): pkg = pm.get_package(spec) if pkg: return pkg.path - assert pm.install(spec).path + assert pm.install(spec) _remove_unnecessary_packages() return pm.get_package(spec) From e43176e33a5fceac98e7ebf03b03e04ad232fd3c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 23:11:24 +0300 Subject: [PATCH 191/223] Typo fix --- platformio/package/manager/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 098bfdea..2d01b155 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -36,7 +36,7 @@ def get_core_package_dir(name): return pkg.path assert pm.install(spec) _remove_unnecessary_packages() - return pm.get_package(spec) + return pm.get_package(spec).path def update_core_packages(only_check=False, silent=False): From 091ba4346dbe928887fb5eec4ce3a2e48f946c07 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 23:11:43 +0300 Subject: [PATCH 192/223] Bump version to 4.4.0b6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 735d4276..071e87bf 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (4, 4, "0b5") +VERSION = (4, 4, "0b6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From ff19109787fdb5430a16b1590cb0718bac200522 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 14:34:03 +0300 Subject: [PATCH 193/223] Fix test --- tests/commands/test_platform.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 2c197ec3..088f4b38 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -51,12 +51,19 @@ def test_install_unknown_from_registry(clirunner): assert isinstance(result.exception, UnknownPackageError) -def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): +# def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): +# result = clirunner.invoke( +# cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], +# ) +# assert result.exit_code != 0 +# assert isinstance(result.exception, IncompatiblePlatform) + + +def test_install_core_3_dev_platform(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], ) - assert result.exit_code != 0 - assert isinstance(result.exception, IncompatiblePlatform) + assert result.exit_code == 0 def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): @@ -120,7 +127,7 @@ def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cli_platform.platform_uninstall, ["atmelavr", "espressif8266"] + cli_platform.platform_uninstall, ["atmelavr@1.2.0", "atmelavr", "espressif8266"] ) validate_cliresult(result) assert not isolated_pio_core.join("platforms").listdir() From fa9025171415e8b23ee8dcf98946748db631dbd5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 14:35:01 +0300 Subject: [PATCH 194/223] Fixed an issue when Unit Testing engine fails with a custom project configuration file // Resolve #3583 --- HISTORY.rst | 3 ++- platformio/commands/test/processor.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 538649fa..4328fe11 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -66,7 +66,8 @@ PlatformIO Core 4 - Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 - Do not generate ".travis.yml" for a new project, let the user have a choice - Do not escape compiler arguments in VSCode template on Windows - - Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) + - Fixed an issue with Unit Testing engine when running multiple environments (`issue #3523 < https://github.com/platformio/platformio-core/issues/3523>`_) + - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 334db858..de09b5f9 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -138,6 +138,7 @@ class TestProcessorBase(object): return self.cmd_ctx.invoke( cmd_run, project_dir=self.options["project_dir"], + project_conf=self.options["project_config"].path, upload_port=self.options["upload_port"], verbose=self.options["verbose"], silent=self.options["silent"], From 2ea80d91f8eb2cdbd0048583b2d96a2441c5ff33 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 15:55:17 +0300 Subject: [PATCH 195/223] Minor fixes --- HISTORY.rst | 6 +++--- tests/commands/test_platform.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4328fe11..e1ddddc4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -62,11 +62,11 @@ PlatformIO Core 4 - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment - - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. - - Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required + - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6 - Do not generate ".travis.yml" for a new project, let the user have a choice - Do not escape compiler arguments in VSCode template on Windows - - Fixed an issue with Unit Testing engine when running multiple environments (`issue #3523 < https://github.com/platformio/platformio-core/issues/3523>`_) + - Fixed an issue with Unit Testing engine when running multiple environments (`issue #3523 `_) - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) 4.3.4 (2020-05-23) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 088f4b38..39afbeb5 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -18,7 +18,6 @@ import json from platformio.commands import platform as cli_platform from platformio.package.exception import UnknownPackageError -from platformio.platform.exception import IncompatiblePlatform def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): From 79bfac29baa3000d0254f407feb6fe042f29bc59 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 18:57:20 +0300 Subject: [PATCH 196/223] Update history and sync docs --- HISTORY.rst | 49 ++++++++++++++++++++++++++++--------------------- docs | 2 +- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e1ddddc4..108cc259 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,12 +11,6 @@ PlatformIO Core 4 **A professional collaborative platform for embedded development** -* Integration with the new `Account Management System `__ - - - Manage own organizations - - Manage organization teams - - Manage resource access - * Integration with the new **PlatformIO Trusted Registry** - Enterprise-grade package storage with high availability (multi replicas) @@ -34,40 +28,53 @@ PlatformIO Core 4 * `pio package unpublish `__ – remove a pushed package from the registry * Grant package access to the team members or maintainers +* Integration with the new `Account Management System `__ + + - Manage own organizations + - Manage organization teams + - Manage resource access + * New **Package Management System** - Integrated PlatformIO Core with the new PlatformIO Trusted Registry - Strict dependency declaration using owner name (resolves name conflicts) (`issue #1824 `_) - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) -* New `Custom Targets `__ - - - Pre/Post processing based on a dependent sources (other target, source file, etc.) - - Command launcher with own arguments - - Launch command with custom options declared in `"platformio.ini" `__ - - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) - - List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) - * **PlatformIO Build System** + - New `Custom Targets `__ + + * Pre/Post processing based on a dependent sources (other target, source file, etc.) + * Command launcher with own arguments + * Launch command with custom options declared in `"platformio.ini" `__ + * Python callback as a target (use the power of Python interpreter and PlatformIO Build API) + * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) + - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) - Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) +* **Project Management** + + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required + - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment + - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command + - Do not generate ".travis.yml" for a new project, let the user have a choice + +* **Unit Testing** + + - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6 + - Fixed an issue when running multiple test environments (`issue #3523 `_) + - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) + * **Miscellaneous** - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) - - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command - - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment - - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required - - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6 - - Do not generate ".travis.yml" for a new project, let the user have a choice - Do not escape compiler arguments in VSCode template on Windows - - Fixed an issue with Unit Testing engine when running multiple environments (`issue #3523 `_) - - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) + 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index be04ee45..dfa6701b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit be04ee45c8d037c8eecbbe9a178926593fd620a0 +Subproject commit dfa6701b70dac8b8a1449cdff99879f79151589e From b9fe4933365a5f2b9807bbc9baa5f4a4a4b67671 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 19:18:26 +0300 Subject: [PATCH 197/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index dfa6701b..cc09d981 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit dfa6701b70dac8b8a1449cdff99879f79151589e +Subproject commit cc09d981358a438ad474485a89ee5ddbc1842c65 From 3e72f098fe6e2c52ba1e1094e2b11dd39a9a02c1 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 25 Aug 2020 21:19:21 +0300 Subject: [PATCH 198/223] Updates for PIO Check (#3640) * Update check tools to the latest versions * Use language standard when exporting defines to check tools * Buffer Cppcheck output to detect multiline messages * Add new test for PIO Check * Pass include paths to Clang-Tidy as individual compiler arguments Clang-tidy doesn't support response files which can exceed command length limitations on Windows * Simplify tests for PIO Check * Update history * Sync changelog --- HISTORY.rst | 1 + platformio/__init__.py | 4 +- platformio/commands/check/tools/base.py | 4 +- platformio/commands/check/tools/clangtidy.py | 7 +- platformio/commands/check/tools/cppcheck.py | 18 ++- tests/commands/test_check.py | 117 +++++++++++++------ 6 files changed, 101 insertions(+), 50 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 108cc259..24963d3e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -82,6 +82,7 @@ PlatformIO Core 4 * Added `PlatformIO CLI Shell Completion `__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 `_) * Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 `_) * Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 `_, `issue #3328 `_) +* Fixed an issue with PIO Check when a defect with multiline error message is not reported in verbose mode (`issue #3631 `_) 4.3.3 (2020-04-28) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 071e87bf..0d8d3f17 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -51,9 +51,9 @@ __core_packages__ = { "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", - "tool-cppcheck": "~1.190.0", + "tool-cppcheck": "~1.210.0", "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.7.0", + "tool-pvs-studio": "~7.8.0", } __check_internet_hosts__ = [ diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index d873810d..dc9f476f 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -83,7 +83,9 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes cmd = "echo | %s -x %s %s %s -dM -E -" % ( self.cc_path, language, - " ".join([f for f in build_flags if f.startswith(("-m", "-f"))]), + " ".join( + [f for f in build_flags if f.startswith(("-m", "-f", "-std"))] + ), includes_file, ) result = proc.exec_command(cmd, shell=True) diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 05be67b4..06f3ff76 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -63,10 +63,7 @@ class ClangtidyCheckTool(CheckToolBase): for scope in project_files: src_files.extend(project_files[scope]) - cmd.extend(flags) - cmd.extend(src_files) - cmd.append("--") - + cmd.extend(flags + src_files + ["--"]) cmd.extend( ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines["c++"]] ) @@ -79,6 +76,6 @@ class ClangtidyCheckTool(CheckToolBase): continue includes.append(inc) - cmd.append("--extra-arg=" + self._long_includes_hook(includes)) + cmd.extend(["-I%s" % inc for inc in includes]) return cmd diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 931b16ed..b38bb8d6 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -24,6 +24,8 @@ from platformio.package.manager.core import get_core_package_dir class CppcheckCheckTool(CheckToolBase): def __init__(self, *args, **kwargs): + self._field_delimiter = "<&PIO&>" + self._buffer = "" self.defect_fields = [ "severity", "message", @@ -55,13 +57,15 @@ class CppcheckCheckTool(CheckToolBase): return line def parse_defect(self, raw_line): - if "<&PIO&>" not in raw_line or any( - f not in raw_line for f in self.defect_fields - ): + if self._field_delimiter not in raw_line: + return None + + self._buffer += raw_line + if any(f not in self._buffer for f in self.defect_fields): return None args = dict() - for field in raw_line.split("<&PIO&>"): + for field in self._buffer.split(self._field_delimiter): field = field.strip().replace('"', "") name, value = field.split("=", 1) args[name] = value @@ -94,6 +98,7 @@ class CppcheckCheckTool(CheckToolBase): self._bad_input = True return None + self._buffer = "" return DefectItem(**args) def configure_command( @@ -103,13 +108,16 @@ class CppcheckCheckTool(CheckToolBase): cmd = [ tool_path, + "--addon-python=%s" % proc.get_pythonexe_path(), "--error-exitcode=1", "--verbose" if self.options.get("verbose") else "--quiet", ] cmd.append( '--template="%s"' - % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) + % self._field_delimiter.join( + ["{0}={{{0}}}".format(f) for f in self.defect_fields] + ) ) flags = self.get_flags("cppcheck") diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 596c0f29..6d4c6878 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -61,6 +61,12 @@ int main() { } """ + +PVS_STUDIO_FREE_LICENSE_HEADER = """ +// This is an open source non-commercial project. Dear PVS-Studio, please check it. +// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com +""" + EXPECTED_ERRORS = 4 EXPECTED_WARNINGS = 1 EXPECTED_STYLE = 1 @@ -87,19 +93,21 @@ def count_defects(output): return error, warning, style -def test_check_cli_output(clirunner, check_dir): +def test_check_cli_output(clirunner, validate_cliresult, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir)]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS -def test_check_json_output(clirunner, check_dir): +def test_check_json_output(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--json-output"] ) + validate_cliresult(result) + output = json.loads(result.stdout.strip()) assert isinstance(output, list) @@ -114,14 +122,24 @@ def test_check_tool_defines_passed(clirunner, check_dir): assert "__GNUC__" in output -def test_check_severity_threshold(clirunner, check_dir): +def test_check_language_standard_definition_passed(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\nbuild_flags = -std=c++17" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "__cplusplus=201703L" in result.output + assert "--std=c++17" in result.output + + +def test_check_severity_threshold(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--severity=high"] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 @@ -129,10 +147,9 @@ def test_check_severity_threshold(clirunner, check_dir): def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) - output = result.output inc_count = 0 - for l in output.split("\n"): + for l in result.output.split("\n"): if l.startswith("Includes:"): inc_count = l.count("-I") @@ -140,18 +157,18 @@ def test_check_includes_passed(clirunner, check_dir): assert inc_count > 1 -def test_check_silent_mode(clirunner, check_dir): +def test_check_silent_mode(clirunner, validate_cliresult, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--silent"]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 -def test_check_custom_pattern_absolute_path(clirunner, tmpdir_factory): +def test_check_custom_pattern_absolute_path(clirunner, validate_cliresult, tmpdir_factory): project_dir = tmpdir_factory.mktemp("project") project_dir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -161,16 +178,16 @@ def test_check_custom_pattern_absolute_path(clirunner, tmpdir_factory): result = clirunner.invoke( cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == EXPECTED_WARNINGS assert style == EXPECTED_STYLE -def test_check_custom_pattern_relative_path(clirunner, tmpdir_factory): +def test_check_custom_pattern_relative_path(clirunner, validate_cliresult, tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -180,10 +197,10 @@ def test_check_custom_pattern_relative_path(clirunner, tmpdir_factory): result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--pattern=app", "--pattern=prj"] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS * 2 @@ -214,7 +231,7 @@ def test_check_bad_flag_passed(clirunner, check_dir): assert style == 0 -def test_check_success_if_no_errors(clirunner, tmpdir): +def test_check_success_if_no_errors(clirunner, validate_cliresult, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src").join("main.c").write( """ @@ -232,26 +249,28 @@ int main() { ) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert "[PASSED]" in result.output - assert result.exit_code == 0 assert errors == 0 assert warnings == 1 assert style == 1 -def test_check_individual_flags_passed(clirunner, tmpdir): +def test_check_individual_flags_passed(clirunner, validate_cliresult, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy, pvs-studio" config += """\ncheck_flags = cppcheck: --std=c++11 clangtidy: --fix-errors pvs-studio: --analysis-mode=4 """ + tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + tmpdir.mkdir("src").join("main.cpp").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + validate_cliresult(result) clang_flags_found = cppcheck_flags_found = pvs_flags_found = False for l in result.output.split("\n"): @@ -269,7 +288,7 @@ def test_check_individual_flags_passed(clirunner, tmpdir): assert pvs_flags_found -def test_check_cppcheck_misra_addon(clirunner, check_dir): +def test_check_cppcheck_misra_addon(clirunner, validate_cliresult, check_dir): check_dir.join("misra.json").write( """ { @@ -309,12 +328,12 @@ R21.4 text. cmd_check, ["--project-dir", str(check_dir), "--flags=--addon=misra.json"] ) - assert result.exit_code == 0 + validate_cliresult(result) assert "R21.3 Found MISRA defect" in result.output assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) -def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): +def test_check_fails_on_defects_only_with_flag(clirunner, validate_cliresult, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) @@ -325,11 +344,13 @@ def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) - assert default_result.exit_code == 0 + validate_cliresult(default_result) assert result_with_flag.exit_code != 0 -def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): +def test_check_fails_on_defects_only_on_specified_level( + clirunner, validate_cliresult, tmpdir +): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write( @@ -350,12 +371,12 @@ int main() { high_result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) + validate_cliresult(high_result) low_result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=low"] ) - assert high_result.exit_code == 0 assert low_result.exit_code != 0 @@ -367,15 +388,9 @@ board = teensy35 framework = arduino check_tool = pvs-studio """ - code = ( - """// This is an open source non-commercial project. Dear PVS-Studio, please check it. -// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com -""" - + TEST_CODE - ) tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.c").write(code) + tmpdir.mkdir("src").join("main.c").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high", "-v"] @@ -399,8 +414,7 @@ check_tool = %s """ # tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write( - """// This is an open source non-commercial project. Dear PVS-Studio, please check it. -// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com + PVS_STUDIO_FREE_LICENSE_HEADER + """ #include void unused_function(int val){ @@ -425,13 +439,13 @@ int main() { result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) defects = sum(count_defects(result.output)) - assert result.exit_code == 0 and defects > 0, "Failed %s with %s" % ( + assert defects > 0, "Failed %s with %s" % ( framework, tool, ) -def test_check_skip_includes_from_packages(clirunner, tmpdir): +def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir): config = """ [env:test] platform = nordicnrf52 @@ -445,13 +459,42 @@ framework = arduino result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--skip-packages", "-v"] ) - - output = result.output + validate_cliresult(result) project_path = fs.to_unix_path(str(tmpdir)) - for l in output.split("\n"): + for l in result.output.split("\n"): if not l.startswith("Includes:"): continue for inc in l.split(" "): if inc.startswith("-I") and project_path not in inc: pytest.fail("Detected an include path from packages: " + inc) + + +def test_check_multiline_error(clirunner, tmpdir_factory): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + project_dir.mkdir("include").join("main.h").write( + """ +#error This is a multiline error message \\ +that should be correctly reported \\ +in both default and verbose modes. +""" + ) + + project_dir.mkdir("src").join("main.c").write( + """ +#include +#include "main.h" + +int main() {} +""" + ) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir)]) + errors, _, _ = count_defects(result.output) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir), "-v"]) + verbose_errors, _, _ = count_defects(result.output) + + assert verbose_errors == errors == 1 From f77978a295b8208d36a5162fd40063fc6b53dedd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 22:01:08 +0300 Subject: [PATCH 199/223] Apply formatting --- tests/commands/test_check.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 6d4c6878..fa33af68 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -168,7 +168,9 @@ def test_check_silent_mode(clirunner, validate_cliresult, check_dir): assert style == 0 -def test_check_custom_pattern_absolute_path(clirunner, validate_cliresult, tmpdir_factory): +def test_check_custom_pattern_absolute_path( + clirunner, validate_cliresult, tmpdir_factory +): project_dir = tmpdir_factory.mktemp("project") project_dir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -187,7 +189,9 @@ def test_check_custom_pattern_absolute_path(clirunner, validate_cliresult, tmpdi assert style == EXPECTED_STYLE -def test_check_custom_pattern_relative_path(clirunner, validate_cliresult, tmpdir_factory): +def test_check_custom_pattern_relative_path( + clirunner, validate_cliresult, tmpdir_factory +): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -268,7 +272,9 @@ def test_check_individual_flags_passed(clirunner, validate_cliresult, tmpdir): """ tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.cpp").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) + tmpdir.mkdir("src").join("main.cpp").write( + PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE + ) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) validate_cliresult(result) @@ -414,7 +420,8 @@ check_tool = %s """ # tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write( - PVS_STUDIO_FREE_LICENSE_HEADER + """ + PVS_STUDIO_FREE_LICENSE_HEADER + + """ #include void unused_function(int val){ @@ -439,10 +446,7 @@ int main() { result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) defects = sum(count_defects(result.output)) - assert defects > 0, "Failed %s with %s" % ( - framework, - tool, - ) + assert defects > 0, "Failed %s with %s" % (framework, tool,) def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir): From 210cd760424bbb94bc888d5f65b0fa8f90962c39 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 22:01:22 +0300 Subject: [PATCH 200/223] Rename "idedata" sub-command to "data" --- docs | 2 +- platformio/commands/project.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs b/docs index cc09d981..ccce6f04 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit cc09d981358a438ad474485a89ee5ddbc1842c65 +Subproject commit ccce6f04e2e54dd3e08c83f23480a49823c8f5c7 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 6900ce74..bd37175a 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -69,7 +69,7 @@ def project_config(project_dir, json_output): return None -@cli.command("idedata", short_help="Dump data intended for IDE extensions/plugins") +@cli.command("data", short_help="Dump data intended for IDE extensions/plugins") @click.option( "-d", "--project-dir", @@ -78,7 +78,7 @@ def project_config(project_dir, json_output): ) @click.option("-e", "--environment", multiple=True) @click.option("--json-output", is_flag=True) -def project_idedata(project_dir, environment, json_output): +def project_data(project_dir, environment, json_output): if not is_platformio_project(project_dir): raise NotPlatformIOProjectError(project_dir) with fs.cd(project_dir): From 5086b96edea9ad67e909d39d28194ae14f083c94 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 22:22:35 +0300 Subject: [PATCH 201/223] Bump version to 5.0.0b1 --- HISTORY.rst | 18 ++++++++++++------ docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 24963d3e..4f82d8d8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,15 +1,16 @@ Release Notes ============= -.. _release_notes_4: +.. _release_notes_5: -PlatformIO Core 4 +PlatformIO Core 5 ----------------- -4.4.0 (2020-??-??) +**A professional collaborative platform for embedded development** + +5.0.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ -**A professional collaborative platform for embedded development** * Integration with the new **PlatformIO Trusted Registry** @@ -42,6 +43,7 @@ PlatformIO Core 4 * **PlatformIO Build System** + - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ - New `Custom Targets `__ * Pre/Post processing based on a dependent sources (other target, source file, etc.) @@ -50,7 +52,6 @@ PlatformIO Core 4 * Python callback as a target (use the power of Python interpreter and PlatformIO Build API) * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) - - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) @@ -60,7 +61,7 @@ PlatformIO Core 4 - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment - - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command + - Dump build system data intended for IDE extensions/plugins using a new `pio project data `__ command - Do not generate ".travis.yml" for a new project, let the user have a choice * **Unit Testing** @@ -76,6 +77,11 @@ PlatformIO Core 4 - Do not escape compiler arguments in VSCode template on Windows +.. _release_notes_4: + +PlatformIO Core 4 +----------------- + 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index ccce6f04..c536bff8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ccce6f04e2e54dd3e08c83f23480a49823c8f5c7 +Subproject commit c536bff8352fc0a26366d1d9ca73b8e60af8d205 diff --git a/platformio/__init__.py b/platformio/__init__.py index 0d8d3f17..9e5c7d31 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (4, 4, "0b6") +VERSION = (5, 0, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 0db39ccfbd853f4ba9863aeffbc2976ee8b72f87 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 06:40:22 +0300 Subject: [PATCH 202/223] Automatically accept PIO. Core 4.0 compatible dev-platforms --- platformio/platform/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index ffafd61c..d5bcbcc2 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -114,8 +114,8 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) if self.CORE_SEMVER in core_spec: return True - # PIO Core 4 is compatible with dev-platforms for PIO Core 2.0 & 3.0 - if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3)): + # PIO Core 4 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 + if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3, 4)): return True raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) From 1560fb724c0c3471a994eab47c7e80c24f1e2f83 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 06:40:46 +0300 Subject: [PATCH 203/223] Bump version to 5.0.0b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 9e5c7d31..82133bd4 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "0b1") +VERSION = (5, 0, "0b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 3c91e3c1e13b9fa9bcfc22c416a37fef7e849d53 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 14:51:01 +0300 Subject: [PATCH 204/223] Move build dir to the disk root (should fix issue with long path for Zephyr RTOS on WIndows) --- .github/workflows/examples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index f1db5c38..e3bb201f 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -49,6 +49,7 @@ jobs: if: startsWith(matrix.os, 'windows') env: PLATFORMIO_CORE_DIR: C:/pio + PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap" run: | tox -e testexamples From 8625fdc571748c7cc873c3c16a406eb73b80dced Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 14:51:53 +0300 Subject: [PATCH 205/223] Minor imperovements --- platformio/package/manager/_registry.py | 7 +- platformio/platform/base.py | 2 +- scripts/docspregen.py | 558 +++++++++++++++--------- 3 files changed, 345 insertions(+), 222 deletions(-) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index f8af2ece..6d243d76 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -139,14 +139,13 @@ class PackageManageRegistryMixin(object): def fetch_registry_package(self, spec): assert isinstance(spec, PackageSpec) result = None + regclient = self.get_registry_client_instance() if spec.owner and spec.name: - result = self.get_registry_client_instance().get_package( - self.pkg_type, spec.owner, spec.name - ) + result = regclient.get_package(self.pkg_type, spec.owner, spec.name) if not result and (spec.id or (spec.name and not spec.owner)): packages = self.search_registry_packages(spec) if packages: - result = self.get_registry_client_instance().get_package( + result = regclient.get_package( self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"] ) if not result: diff --git a/platformio/platform/base.py b/platformio/platform/base.py index d5bcbcc2..b29a9d7b 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -114,7 +114,7 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) if self.CORE_SEMVER in core_spec: return True - # PIO Core 4 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 + # PIO Core 5 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3, 4)): return True raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) diff --git a/scripts/docspregen.py b/scripts/docspregen.py index f2dc2757..3698aa0b 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -22,7 +22,8 @@ path.append("..") import click from platformio import fs, util -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory try: from urlparse import ParseResult, urlparse, urlunparse @@ -41,15 +42,16 @@ RST_COPYRIGHT = """.. Copyright (c) 2014-present PlatformIO `__ - - {description}""".format(name=name, - url=campaign_url(API_PACKAGES[name]['url']), - description=API_PACKAGES[name]['description'])) + - {description}""".format( + name=name, + url=campaign_url(API_PACKAGES[name]["url"]), + description=API_PACKAGES[name]["description"], + ) + ) if is_embedded: - lines.append(""" + lines.append( + """ .. warning:: **Linux Users**: * Install "udev" rules :ref:`faq_udev_rules` * Raspberry Pi users, please read this article `Enable serial port on Raspberry Pi `__. -""") +""" + ) if platform == "teensy": - lines.append(""" + lines.append( + """ **Windows Users:** Teensy programming uses only Windows built-in HID @@ -278,14 +321,17 @@ Packages `_ is needed to access the COM port your program uses. No special driver installation is necessary on Windows 10. -""") +""" + ) else: - lines.append(""" + lines.append( + """ **Windows Users:** Please check that you have a correctly installed USB driver from board manufacturer -""") +""" + ) return "\n".join(lines) @@ -293,14 +339,12 @@ Packages def generate_platform(name, rst_dir): print("Processing platform: %s" % name) - compatible_boards = [ - board for board in BOARDS if name == board['platform'] - ] + compatible_boards = [board for board in BOARDS if name == board["platform"]] lines = [] lines.append(RST_COPYRIGHT) - p = PlatformFactory.newPlatform(name) + p = PlatformFactory.new(name) assert p.repository_url.endswith(".git") github_url = p.repository_url[:-4] @@ -314,14 +358,18 @@ def generate_platform(name, rst_dir): lines.append(" :ref:`projectconf_env_platform` = ``%s``" % p.name) lines.append("") lines.append(p.description) - lines.append(""" -For more detailed information please visit `vendor site <%s>`_.""" % - campaign_url(p.homepage)) - lines.append(""" + lines.append( + """ +For more detailed information please visit `vendor site <%s>`_.""" + % campaign_url(p.homepage) + ) + lines.append( + """ .. contents:: Contents :local: :depth: 1 -""") +""" + ) # # Extra @@ -332,12 +380,15 @@ For more detailed information please visit `vendor site <%s>`_.""" % # # Examples # - lines.append(""" + lines.append( + """ Examples -------- Examples are listed from `%s development platform repository <%s>`_: -""" % (p.title, campaign_url("%s/tree/master/examples" % github_url))) +""" + % (p.title, campaign_url("%s/tree/master/examples" % github_url)) + ) examples_dir = join(p.get_dir(), "examples") if isdir(examples_dir): for eitem in os.listdir(examples_dir): @@ -355,14 +406,17 @@ Examples are listed from `%s development platform repository <%s>`_: generate_debug_contents( compatible_boards, skip_board_columns=["Platform"], - extra_rst="%s_debug.rst" % - name if isfile(join(rst_dir, "%s_debug.rst" % - name)) else None)) + extra_rst="%s_debug.rst" % name + if isfile(join(rst_dir, "%s_debug.rst" % name)) + else None, + ) + ) # # Development version of dev/platform # - lines.append(""" + lines.append( + """ Stable and upstream versions ---------------------------- @@ -393,13 +447,15 @@ Upstream [env:upstream_develop] platform = {github_url}.git board = ... -""".format(name=p.name, title=p.title, github_url=github_url)) +""".format( + name=p.name, title=p.title, github_url=github_url + ) + ) # # Packages # - _packages_content = generate_packages(name, p.packages.keys(), - p.is_embedded()) + _packages_content = generate_packages(name, p.packages.keys(), p.is_embedded()) if _packages_content: lines.append(_packages_content) @@ -408,8 +464,8 @@ Upstream # compatible_frameworks = [] for framework in API_FRAMEWORKS: - if is_compat_platform_and_framework(name, framework['name']): - compatible_frameworks.append(framework['name']) + if is_compat_platform_and_framework(name, framework["name"]): + compatible_frameworks.append(framework["name"]) lines.extend(generate_frameworks_contents(compatible_frameworks)) # @@ -418,11 +474,12 @@ Upstream if compatible_boards: vendors = {} for board in compatible_boards: - if board['vendor'] not in vendors: - vendors[board['vendor']] = [] - vendors[board['vendor']].append(board) + if board["vendor"] not in vendors: + vendors[board["vendor"]] = [] + vendors[board["vendor"]].append(board) - lines.append(""" + lines.append( + """ Boards ------ @@ -431,20 +488,20 @@ Boards `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll the tables below by horizontally. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) - lines.extend( - generate_boards_table(boards, skip_columns=["Platform"])) + lines.extend(generate_boards_table(boards, skip_columns=["Platform"])) return "\n".join(lines) def update_platform_docs(): for manifest in PLATFORM_MANIFESTS: - name = manifest['name'] + name = manifest["name"] platforms_dir = join(DOCS_ROOT_DIR, "platforms") rst_path = join(platforms_dir, "%s.rst" % name) with open(rst_path, "w") as f: @@ -455,12 +512,11 @@ def generate_framework(type_, data, rst_dir=None): print("Processing framework: %s" % type_) compatible_platforms = [ - m for m in PLATFORM_MANIFESTS - if is_compat_platform_and_framework(m['name'], type_) - ] - compatible_boards = [ - board for board in BOARDS if type_ in board['frameworks'] + m + for m in PLATFORM_MANIFESTS + if is_compat_platform_and_framework(m["name"], type_) ] + compatible_boards = [board for board in BOARDS if type_ in board["frameworks"]] lines = [] @@ -468,21 +524,26 @@ def generate_framework(type_, data, rst_dir=None): lines.append(".. _framework_%s:" % type_) lines.append("") - lines.append(data['title']) - lines.append("=" * len(data['title'])) + lines.append(data["title"]) + lines.append("=" * len(data["title"])) lines.append("") lines.append(":Configuration:") lines.append(" :ref:`projectconf_env_framework` = ``%s``" % type_) lines.append("") - lines.append(data['description']) - lines.append(""" + lines.append(data["description"]) + lines.append( + """ For more detailed information please visit `vendor site <%s>`_. -""" % campaign_url(data['url'])) +""" + % campaign_url(data["url"]) + ) - lines.append(""" + lines.append( + """ .. contents:: Contents :local: - :depth: 1""") + :depth: 1""" + ) # Extra if isfile(join(rst_dir, "%s_extra.rst" % type_)): @@ -495,27 +556,37 @@ For more detailed information please visit `vendor site <%s>`_. lines.extend( generate_debug_contents( compatible_boards, - extra_rst="%s_debug.rst" % - type_ if isfile(join(rst_dir, "%s_debug.rst" % - type_)) else None)) + extra_rst="%s_debug.rst" % type_ + if isfile(join(rst_dir, "%s_debug.rst" % type_)) + else None, + ) + ) if compatible_platforms: # examples - lines.append(""" + lines.append( + """ Examples -------- -""") +""" + ) for manifest in compatible_platforms: - p = PlatformFactory.newPlatform(manifest['name']) - lines.append("* `%s for %s <%s>`_" % - (data['title'], manifest['title'], - campaign_url("%s/tree/master/examples" % - p.repository_url[:-4]))) + p = PlatformFactory.new(manifest["name"]) + lines.append( + "* `%s for %s <%s>`_" + % ( + data["title"], + manifest["title"], + campaign_url("%s/tree/master/examples" % p.repository_url[:-4]), + ) + ) # Platforms lines.extend( generate_platforms_contents( - [manifest['name'] for manifest in compatible_platforms])) + [manifest["name"] for manifest in compatible_platforms] + ) + ) # # Boards @@ -523,10 +594,11 @@ Examples if compatible_boards: vendors = {} for board in compatible_boards: - if board['vendor'] not in vendors: - vendors[board['vendor']] = [] - vendors[board['vendor']].append(board) - lines.append(""" + if board["vendor"] not in vendors: + vendors[board["vendor"]] = [] + vendors[board["vendor"]].append(board) + lines.append( + """ Boards ------ @@ -534,7 +606,8 @@ Boards * You can list pre-configured boards by :ref:`cmd_boards` command or `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll the tables below by horizontally. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) @@ -544,7 +617,7 @@ Boards def update_framework_docs(): for framework in API_FRAMEWORKS: - name = framework['name'] + name = framework["name"] frameworks_dir = join(DOCS_ROOT_DIR, "frameworks") rst_path = join(frameworks_dir, "%s.rst" % name) with open(rst_path, "w") as f: @@ -561,7 +634,8 @@ def update_boards(): lines.append("Boards") lines.append("======") - lines.append(""" + lines.append( + """ Rapid Embedded Development, Continuous and IDE integration in a few steps with PlatformIO thanks to built-in project generator for the most popular embedded boards and IDE. @@ -570,25 +644,28 @@ popular embedded boards and IDE. * You can list pre-configured boards by :ref:`cmd_boards` command or `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) platforms = {} for data in BOARDS: - platform = data['platform'] + platform = data["platform"] if platform in platforms: platforms[platform].append(data) else: platforms[platform] = [data] for platform, boards in sorted(platforms.items()): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) lines.append(p.title) lines.append("-" * len(p.title)) - lines.append(""" + lines.append( + """ .. toctree:: :maxdepth: 1 - """) - for board in sorted(boards, key=lambda item: item['name']): + """ + ) + for board in sorted(boards, key=lambda item: item["name"]): lines.append(" %s/%s" % (platform, board["id"])) lines.append("") @@ -600,44 +677,48 @@ popular embedded boards and IDE. for data in BOARDS: # if data['id'] != "m5stack-core-esp32": # continue - rst_path = join(DOCS_ROOT_DIR, "boards", data["platform"], - "%s.rst" % data["id"]) + rst_path = join( + DOCS_ROOT_DIR, "boards", data["platform"], "%s.rst" % data["id"] + ) if not isdir(dirname(rst_path)): os.makedirs(dirname(rst_path)) update_embedded_board(rst_path, data) def update_embedded_board(rst_path, board): - platform = PlatformFactory.newPlatform(board['platform']) - board_config = platform.board_config(board['id']) + platform = PlatformFactory.new(board["platform"]) + board_config = platform.board_config(board["id"]) board_manifest_url = platform.repository_url assert board_manifest_url if board_manifest_url.endswith(".git"): board_manifest_url = board_manifest_url[:-4] - board_manifest_url += "/blob/master/boards/%s.json" % board['id'] + board_manifest_url += "/blob/master/boards/%s.json" % board["id"] - variables = dict(id=board['id'], - name=board['name'], - platform=board['platform'], - platform_description=platform.description, - url=campaign_url(board['url']), - mcu=board_config.get("build", {}).get("mcu", ""), - mcu_upper=board['mcu'].upper(), - f_cpu=board['fcpu'], - f_cpu_mhz=int(int(board['fcpu']) / 1000000), - ram=fs.format_filesize(board['ram']), - rom=fs.format_filesize(board['rom']), - vendor=board['vendor'], - board_manifest_url=board_manifest_url, - upload_protocol=board_config.get("upload.protocol", "")) + variables = dict( + id=board["id"], + name=board["name"], + platform=board["platform"], + platform_description=platform.description, + url=campaign_url(board["url"]), + mcu=board_config.get("build", {}).get("mcu", ""), + mcu_upper=board["mcu"].upper(), + f_cpu=board["fcpu"], + f_cpu_mhz=int(int(board["fcpu"]) / 1000000), + ram=fs.humanize_file_size(board["ram"]), + rom=fs.humanize_file_size(board["rom"]), + vendor=board["vendor"], + board_manifest_url=board_manifest_url, + upload_protocol=board_config.get("upload.protocol", ""), + ) lines = [RST_COPYRIGHT] lines.append(".. _board_{platform}_{id}:".format(**variables)) lines.append("") - lines.append(board['name']) - lines.append("=" * len(board['name'])) - lines.append(""" + lines.append(board["name"]) + lines.append("=" * len(board["name"])) + lines.append( + """ .. contents:: Hardware @@ -657,12 +738,16 @@ Platform :ref:`platform_{platform}`: {platform_description} - {ram} * - **Vendor** - `{vendor} <{url}>`__ -""".format(**variables)) +""".format( + **variables + ) + ) # # Configuration # - lines.append(""" + lines.append( + """ Configuration ------------- @@ -690,23 +775,33 @@ board manifest `{id}.json <{board_manifest_url}>`_. For example, ; change MCU frequency board_build.f_cpu = {f_cpu}L -""".format(**variables)) +""".format( + **variables + ) + ) # # Uploading # upload_protocols = board_config.get("upload.protocols", []) if len(upload_protocols) > 1: - lines.append(""" + lines.append( + """ Uploading --------- %s supports the next uploading protocols: -""" % board['name']) +""" + % board["name"] + ) for protocol in sorted(upload_protocols): lines.append("* ``%s``" % protocol) - lines.append(""" -Default protocol is ``%s``""" % variables['upload_protocol']) - lines.append(""" + lines.append( + """ +Default protocol is ``%s``""" + % variables["upload_protocol"] + ) + lines.append( + """ You can change upload protocol using :ref:`projectconf_upload_protocol` option: .. code-block:: ini @@ -716,22 +811,29 @@ You can change upload protocol using :ref:`projectconf_upload_protocol` option: board = {id} upload_protocol = {upload_protocol} -""".format(**variables)) +""".format( + **variables + ) + ) # # Debugging # lines.append("Debugging") lines.append("---------") - if not board.get('debug'): + if not board.get("debug"): lines.append( ":ref:`piodebug` currently does not support {name} board.".format( - **variables)) + **variables + ) + ) else: default_debug_tool = board_config.get_debug_tool_name() has_onboard_debug = any( - t.get("onboard") for (_, t) in board['debug']['tools'].items()) - lines.append(""" + t.get("onboard") for (_, t) in board["debug"]["tools"].items() + ) + lines.append( + """ :ref:`piodebug` - "1-click" solution for debugging with a zero configuration. .. warning:: @@ -741,34 +843,43 @@ You can change upload protocol using :ref:`projectconf_upload_protocol` option: You can switch between debugging :ref:`debugging_tools` using :ref:`projectconf_debug_tool` option in :ref:`projectconf`. -""") +""" + ) if has_onboard_debug: lines.append( "{name} has on-board debug probe and **IS READY** for " - "debugging. You don't need to use/buy external debug probe.". - format(**variables)) + "debugging. You don't need to use/buy external debug probe.".format( + **variables + ) + ) else: lines.append( "{name} does not have on-board debug probe and **IS NOT " "READY** for debugging. You will need to use/buy one of " - "external probe listed below.".format(**variables)) - lines.append(""" + "external probe listed below.".format(**variables) + ) + lines.append( + """ .. list-table:: :header-rows: 1 * - Compatible Tools - On-board - - Default""") - for (tool_name, tool_data) in sorted(board['debug']['tools'].items()): - lines.append(""" * - :ref:`debugging_tool_{name}` + - Default""" + ) + for (tool_name, tool_data) in sorted(board["debug"]["tools"].items()): + lines.append( + """ * - :ref:`debugging_tool_{name}` - {onboard} - {default}""".format( - name=tool_name, - onboard="Yes" if tool_data.get("onboard") else "", - default="Yes" if tool_name == default_debug_tool else "")) + name=tool_name, + onboard="Yes" if tool_data.get("onboard") else "", + default="Yes" if tool_name == default_debug_tool else "", + ) + ) - if board['frameworks']: - lines.extend(generate_frameworks_contents(board['frameworks'])) + if board["frameworks"]: + lines.extend(generate_frameworks_contents(board["frameworks"])) with open(rst_path, "w") as f: f.write("\n".join(lines)) @@ -781,21 +892,21 @@ def update_debugging(): platforms = [] frameworks = [] for data in BOARDS: - if not data.get('debug'): + if not data.get("debug"): continue - for tool in data['debug']['tools']: + for tool in data["debug"]["tools"]: tool = str(tool) if tool not in tool_to_platforms: tool_to_platforms[tool] = [] - tool_to_platforms[tool].append(data['platform']) + tool_to_platforms[tool].append(data["platform"]) if tool not in tool_to_boards: tool_to_boards[tool] = [] - tool_to_boards[tool].append(data['id']) + tool_to_boards[tool].append(data["id"]) - platforms.append(data['platform']) - frameworks.extend(data['frameworks']) - vendor = data['vendor'] + platforms.append(data["platform"]) + frameworks.extend(data["frameworks"]) + vendor = data["vendor"] if vendor in vendors: vendors[vendor].append(data) else: @@ -809,26 +920,30 @@ def update_debugging(): lines.extend(generate_frameworks_contents(frameworks)) # Boards - lines.append(""" + lines.append( + """ Boards ------ .. note:: For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) lines.extend(generate_boards_table(boards)) # save - with open(join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), - "r+") as fp: + with open( + join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+" + ) as fp: content = fp.read() fp.seek(0) fp.truncate() - fp.write(content[:content.index(".. _debugging_platforms:")] + - "\n".join(lines)) + fp.write( + content[: content.index(".. _debugging_platforms:")] + "\n".join(lines) + ) # Debug tools for tool, platforms in tool_to_platforms.items(): @@ -847,24 +962,27 @@ Boards tool_frameworks.append(framework) lines.extend(generate_frameworks_contents(tool_frameworks)) - lines.append(""" + lines.append( + """ Boards ------ .. note:: For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) lines.extend( generate_boards_table( - [b for b in BOARDS if b['id'] in tool_to_boards[tool]], - skip_columns=None)) + [b for b in BOARDS if b["id"] in tool_to_boards[tool]], + skip_columns=None, + ) + ) with open(tool_path, "r+") as fp: content = fp.read() fp.seek(0) fp.truncate() - fp.write(content[:content.index(".. begin_platforms")] + - "\n".join(lines)) + fp.write(content[: content.index(".. begin_platforms")] + "\n".join(lines)) def update_project_examples(): @@ -899,7 +1017,7 @@ def update_project_examples(): desktop = [] for manifest in PLATFORM_MANIFESTS: - p = PlatformFactory.newPlatform(manifest['name']) + p = PlatformFactory.new(manifest["name"]) github_url = p.repository_url[:-4] # Platform README @@ -922,19 +1040,21 @@ def update_project_examples(): name=p.name, title=p.title, description=p.description, - examples="\n".join(examples_md_lines))) + examples="\n".join(examples_md_lines), + ) + ) # Framework README for framework in API_FRAMEWORKS: - if not is_compat_platform_and_framework(p.name, framework['name']): + if not is_compat_platform_and_framework(p.name, framework["name"]): continue - if framework['name'] not in framework_examples_md_lines: - framework_examples_md_lines[framework['name']] = [] + if framework["name"] not in framework_examples_md_lines: + framework_examples_md_lines[framework["name"]] = [] lines = [] lines.append("- [%s](%s)" % (p.title, github_url)) lines.extend(" %s" % l for l in examples_md_lines) lines.append("") - framework_examples_md_lines[framework['name']].extend(lines) + framework_examples_md_lines[framework["name"]].extend(lines) # Root README line = "* [%s](%s)" % (p.title, "%s/tree/master/examples" % github_url) @@ -946,27 +1066,29 @@ def update_project_examples(): # Frameworks frameworks = [] for framework in API_FRAMEWORKS: - readme_dir = join(project_examples_dir, "frameworks", - framework['name']) + readme_dir = join(project_examples_dir, "frameworks", framework["name"]) if not isdir(readme_dir): os.makedirs(readme_dir) with open(join(readme_dir, "README.md"), "w") as fp: fp.write( framework_readme_tpl.format( - name=framework['name'], - title=framework['title'], - description=framework['description'], - examples="\n".join( - framework_examples_md_lines[framework['name']]))) + name=framework["name"], + title=framework["title"], + description=framework["description"], + examples="\n".join(framework_examples_md_lines[framework["name"]]), + ) + ) url = campaign_url( "https://docs.platformio.org/en/latest/frameworks/%s.html#examples" - % framework['name'], + % framework["name"], source="github", - medium="examples") - frameworks.append("* [%s](%s)" % (framework['title'], url)) + medium="examples", + ) + frameworks.append("* [%s](%s)" % (framework["title"], url)) with open(join(project_examples_dir, "README.md"), "w") as fp: - fp.write("""# PlatformIO Project Examples + fp.write( + """# PlatformIO Project Examples - [Development platforms](#development-platforms): - [Embedded](#embedded) @@ -986,7 +1108,9 @@ def update_project_examples(): ## Frameworks %s -""" % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks))) +""" + % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks)) + ) def main(): From d59416431dae2d4b22f0116c5057e5ec78b0e7d2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 15:40:03 +0300 Subject: [PATCH 206/223] Parse npm-like "repository" data from a package manifest // Resolve #3637 --- platformio/package/manifest/parser.py | 12 ++++++++++++ platformio/util.py | 4 ++-- tests/package/test_manifest.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 689de80b..8949f43e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -662,6 +662,7 @@ class PackageJsonManifestParser(BaseManifestParser): data["keywords"] = self.str_to_list(data["keywords"], sep=",") data = self._parse_system(data) data = self._parse_homepage(data) + data = self._parse_repository(data) return data @staticmethod @@ -682,3 +683,14 @@ class PackageJsonManifestParser(BaseManifestParser): data["homepage"] = data["url"] del data["url"] return data + + @staticmethod + def _parse_repository(data): + if isinstance(data.get("repository", {}), dict): + return data + data["repository"] = dict(type="git", url=str(data["repository"])) + if data["repository"]["url"].startswith(("github:", "gitlab:", "bitbucket:")): + data["repository"]["url"] = "https://{0}.com/{1}".format( + *(data["repository"]["url"].split(":", 1)) + ) + return data diff --git a/platformio/util.py b/platformio/util.py index aeeaf55b..f950a364 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -28,8 +28,8 @@ import click from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS -from platformio.fs import cd # pylint: disable=unused-import -from platformio.fs import load_json # pylint: disable=unused-import +from platformio.fs import cd, load_json # pylint: disable=unused-import +from platformio.package.version import pepver_to_semver # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 9dd5b878..426cbdf1 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -672,6 +672,20 @@ def test_package_json_schema(): ) assert mp.as_dict()["system"] == ["darwin_x86_64"] + # shortcut repository syntax (npm-style) + contents = """ +{ + "name": "tool-github", + "version": "1.2.0", + "repository": "github:user/repo" +} +""" + raw_data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.PACKAGE_JSON + ).as_dict() + data = ManifestSchema().load_manifest(raw_data) + assert data["repository"]["url"] == "https://github.com/user/repo.git" + def test_parser_from_dir(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") From 4a7f578649781252f1466f0725bf31b239a9fdaa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 15:40:24 +0300 Subject: [PATCH 207/223] Sync docs and history --- HISTORY.rst | 2010 +-------------------------------------------------- docs | 2 +- examples | 2 +- 3 files changed, 7 insertions(+), 2007 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4f82d8d8..53a933d3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -82,2024 +82,24 @@ PlatformIO Core 5 PlatformIO Core 4 ----------------- -4.3.4 (2020-05-23) -~~~~~~~~~~~~~~~~~~ - -* Added `PlatformIO CLI Shell Completion `__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 `_) -* Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 `_) -* Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 `_, `issue #3328 `_) -* Fixed an issue with PIO Check when a defect with multiline error message is not reported in verbose mode (`issue #3631 `_) - -4.3.3 (2020-04-28) -~~~~~~~~~~~~~~~~~~ - -* Fixed "UnicodeDecodeError: 'utf-8' codec can't decode byte" when non-Latin chars are used in project path (`issue #3481 `_) - -4.3.2 (2020-04-28) -~~~~~~~~~~~~~~~~~~ - -* New `Account Management System `__ (preview) -* Open source `PIO Remote `__ client -* Improved `PIO Check `__ with more accurate project processing -* Echo what is typed when ``send_on_enter`` `device monitor filter `__ is used (`issue #3452 `_) -* Fixed PIO Unit Testing for Zephyr RTOS -* Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) -* Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) -* Fixed an incorrect node path used for pattern matching when processing middleware nodes -* Fixed an issue with missing ``lib_extra_dirs`` option in SRC_LIST for CLion (`issue #3460 `_) - -4.3.1 (2020-03-20) -~~~~~~~~~~~~~~~~~~ - -* Fixed a SyntaxError "'return' with argument inside generator" for PIO Unified Debugger when Python 2.7 is used -* Fixed an issue when ``lib_archive = no`` was not honored in `"platformio.ini" `__ -* Fixed a TypeError "super(type, obj): obj must be an instance or subtype of type" when device monitor is used with a custom dev-platform filter (`issue #3431 `_) - -4.3.0 (2020-03-19) -~~~~~~~~~~~~~~~~~~ - -* Initial support for an official `PlatformIO for CLion IDE `__ plugin: - - - Smart C and C++ editor - - Code refactoring - - On-the-fly code analysis - - "New PlatformIO Project" wizard - - Building, Uploading, Testing - - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.) - -* `Device Monitor 2.0 `__ - - - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 `_) - - Configure project device monitor with `monitor_filters `__ option - - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) - - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) - - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 `_) - - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter - -* New standalone (1-script) `PlatformIO Core Installer `_ -* Initial support for `Renode `__ simulation framework (`issue #3401 `_) -* Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) -* Improved support for Arduino "library.properties" ``depends`` field -* Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server -* Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) -* Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 `_) -* Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 `_) -* Fixed an issue when not all commands in `compilation database "compile_commands.json" `__ use absolute paths (`pull #3415 `_) -* Fixed an issue when unknown transport is used for `PIO Unit Testing `__ engine (`issue #3422 `_) - -4.2.1 (2020-02-17) -~~~~~~~~~~~~~~~~~~ - -* Improved VSCode template with special ``forceInclude`` field for direct includes via ``-include`` flag (`issue #3379 `_) -* Improved support of PIO Home on card-sized PC (Raspberry Pi, etc.) (`issue #3313 `_) -* Froze "marshmallow" dependency to 2.X for Python 2 (`issue #3380 `_) -* Fixed "TypeError: unsupported operand type(s)" when system environment variable is used by project configuration parser (`issue #3377 `_) -* Fixed an issue when Library Dependency Finder (LDF) ignores custom "libLDFMode" and "libCompatMode" options in `library.json `__ -* Fixed an issue when generating of compilation database "compile_commands.json" does not work with Python 2.7 (`issue #3378 `_) - - -4.2.0 (2020-02-12) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home 3.1 `__: - - - Project Manager - - Project Configuration UI for `"platformio.ini" `__ - -* `PIO Check `__ – automated code analysis without hassle: - - - Added support for `PVS-Studio `__ static code analyzer - -* Initial support for `Project Manager `_ CLI: - - - Show computed project configuration with a new `platformio project config `_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 `_) - - Moved ``platformio init`` command to `platformio project init `_ - -* Generate `compilation database "compile_commands.json" `__ (`issue #2990 `_) -* Control debug flags and optimization level with a new `debug_build_flags `__ option -* Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) -* Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) -* Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) -* Added support for Arduino's library.properties ``depends`` field (`issue #2781 `_) -* Autodetect monitor port for boards with specified HWIDs (`issue #3349 `_) -* Updated SCons tool to 3.1.2 -* Updated Unity tool to 2.5.0 -* Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) -* Warn about broken library manifest when scanning dependencies (`issue #3268 `_) -* Do not overwrite custom items in VSCode's "extensions.json" (`issue #3374 `_) -* Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) -* Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) -* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ -* Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" (`issue #3311 `_) -* Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 `_) -* Fixed an issue when invalid CLI command does not return non-zero exit code -* Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 `_) -* Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 (`issue #3370 `_) -* Fixed an issue with improperly handled compiler flags with space symbols in VSCode template (`issue #3364 `_) -* Fixed an issue when no error is raised if referred parameter (interpolation) is missing in a project configuration file (`issue #3279 `_) - - -4.1.0 (2019-11-07) -~~~~~~~~~~~~~~~~~~ - -* `PIO Check `__ – automated code analysis without hassle: - - - Potential NULL pointer dereferences - - Possible indexing beyond array bounds - - Suspicious assignments - - Reads of potentially uninitialized objects - - Unused variables or functions - - Out of scope memory usage. - -* `PlatformIO Home 3.0 `__: - - - Project Inspection - - Static Code Analysis - - Firmware File Explorer - - Firmware Memory Inspection - - Firmware Sections & Symbols Viewer. - -* Added support for `Build Middlewares `__: configure custom build flags per specific file, skip any build nodes from a framework, replace build file with another on-the-fly, etc. -* Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) -* Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting -* Added ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters -* Added ``--shutdown-timeout`` option to `PIO Home Server `__ -* Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) -* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ -* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) -* Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 `_) -* Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path -* Fixed security issue when extracting items from TAR archive (`issue #2995 `_) -* Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 `_) -* Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 `_) -* Fixed an issue with invalid encoding when generating project for Visual Studio (`issue #3183 `_) -* Fixed an issue when Project Config Parser does not remove in-line comments when Python 3 is used (`issue #3213 `_) -* Fixed an issue with a GCC Linter for PlatformIO IDE for Atom (`issue #3218 `_) - -4.0.3 (2019-08-30) -~~~~~~~~~~~~~~~~~~ - -* Added support for multi-environment PlatformIO project for `CLion IDE `__ (`issue #2824 `_) -* Generate ``.ccls`` LSP file for `Vim `__ cross references, hierarchies, completion and semantic highlighting (`issue #2952 `_) -* Added support for `PLATFORMIO_DISABLE_COLOR `__ system environment variable which disables color ANSI-codes in a terminal output (`issue #2956 `_) -* Updated SCons tool to 3.1.1 -* Remove ProjectConfig cache when "platformio.ini" was modified outside -* Fixed an issue with PIO Unified Debugger on Windows OS when debug server is piped -* Fixed an issue when `--upload-port `__ CLI flag does not override declared `upload_port `__ option in `"platformio.ini" (Project Configuration File) `__ - -4.0.2 (2019-08-23) -~~~~~~~~~~~~~~~~~~ - -* Fixed an issue with a broken `LDF `__ when checking for framework compatibility (`issue #2940 `_) - -4.0.1 (2019-08-22) -~~~~~~~~~~~~~~~~~~ - -* Print `debug tool `__ name for the active debugging session -* Do not shutdown PIO Home Server for "upgrade" operations (`issue #2784 `_) -* Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding -* Improved printing of tabulated results -* Automatically normalize file system paths to UNIX-style for Project Generator (`issue #2857 `_) -* Ability to set "databaseFilename" for VSCode and C/C++ extension (`issue #2825 `_) -* Renamed "enable_ssl" setting to `strict_ssl `__ -* Fixed an issue with incorrect escaping of Windows slashes when using `PIO Unified Debugger `__ and "piped" openOCD -* Fixed an issue when "debug", "home", "run", and "test" commands were not shown in "platformio --help" CLI -* Fixed an issue with PIO Home's "No JSON object could be decoded" (`issue #2823 `_) -* Fixed an issue when `library.json `__ had priority over project configuration for `LDF `__ (`issue #2867 `_) - -4.0.0 (2019-07-10) -~~~~~~~~~~~~~~~~~~ - -`Migration Guide from 3.0 to 4.0 `__. - -* `PlatformIO Plus Goes Open Source `__ - - - Built-in `PIO Unified Debugger `__ - - Built-in `PIO Unit Testing `__ - -* **Project Configuration** - - - New project configuration parser with a strict options typing (`API `__) - - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) - - Share common (global) options between project environments using `[env] `__ section (`issue #1643 `_) - - Include external configuration files with `extra_configs `__ option (`issue #1590 `_) - - Custom project ``***_dir`` options declared in `platformio `__ section have higher priority than `Environment variables `__ - - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) - - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) - - Added support for `PLATFORMIO_DEFAULT_ENVS `__ system environment variable (`issue #1967 `_) - - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a `PIO Remote `__ machine - -* **Library Management** - - - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` - - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) - - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - - Use isolated library dependency storage per project build environment (`issue #1696 `_) - - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 `_) - - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) - - Fixed an issue when `lib_compat_mode = strict `__ does not ignore libraries incompatible with a project framework - -* **Build System** - - - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` - - Switch between `Build Configurations `__ (``release`` and ``debug``) with a new project configuration option `build_type `__ - - Custom `platform_packages `__ per a build environment with an option to override default (`issue #1367 `_) - - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) - - Control a number of parallel build jobs with a new `-j, --jobs `__ option - - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) - - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir `__ option (`issue #2674 `_) - - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) - -* **Infrastructure** - - - Python 3 support (`issue #895 `_) - - Significantly speedup back-end for PIO Home. It works super fast now! - - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) - - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) - - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 `_, `issue #1342 `_, `issue #1959 `_, `issue #2100 `_) - -* **Integration** - - - Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file - - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 `_) - - Fixed an issue with Project Generator when an include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) - - Fixed an issue when generating invalid "Eclipse CDT Cross GCC Built-in Compiler Settings" if a custom `PLATFORMIO_CORE_DIR `__ is used (`issue #806 `_) - -* **Miscellaneous** - - - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead - - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) - - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 `_) +See `PlatformIO Core 4.0 history `__. PlatformIO Core 3 ----------------- -3.6.7 (2019-04-23) -~~~~~~~~~~~~~~~~~~ - -* `PIO Unified Debugger `__: improved debugging in ``debug_load_mode = modified`` and fixed an issue with useless project rebuilding -* Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist -* Fixed an "IndexError: list index out of range" for Arduino sketch preprocessor - (`issue #2268 `_) -* Fixed an issue when invalid "env_default" in `"platformio.ini" (Project Configuration File) `__ results into unhandled errors - (`issue #2265 `_) - -3.6.6 (2019-03-29) -~~~~~~~~~~~~~~~~~~ - -* Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode - (`issue #2243 `_) -* Fixed an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries - (`issue #2164 `_) -* Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) -* Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option -* Fixed an issue with incorrect order of project "include" and "src" paths in ``CPPPATH`` - (`issue #1914 `_) - -3.6.5 (2019-03-07) -~~~~~~~~~~~~~~~~~~ - -* Project Generator: added new targets for CLion IDE "BUILD_VERBOSE" and "MONITOR" (serial port monitor) - (`issue #359 `_) -* Fixed an issue with slow updating of PlatformIO Core packages on Windows -* Fixed an issue when `platformio ci `__ recompiles project if ``--keep-build-dir`` option is passed - (`issue #2109 `_) -* Fixed an issue when ``$PROJECT_HASH`` template was not expanded for the other directory ``***_dir`` options in `"platformio.ini" (Project Configuration File) `__ - (`issue #2170 `_) - -3.6.4 (2019-01-23) -~~~~~~~~~~~~~~~~~~ - -* Improved Project Generator for IDEs: - - - Use full path to PlatformIO CLI when generating a project - (`issue #1674 `_) - - CLion: Improved project portability using "${CMAKE_CURRENT_LIST_DIR}" instead of full path - - Eclipse: Provide language standard to a project C/C++ indexer - (`issue #1010 `_) - -* Fixed an issue with incorrect detecting of compatibility (LDF) between generic library and Arduino or ARM mbed frameworks -* Fixed "Runtime Error: Dictionary size changed during iteration" - (`issue #2003 `_) -* Fixed an error "Could not extract item..." when extracting TAR archive with symbolic items on Windows platform - (`issue #2015 `_) - -3.6.3 (2018-12-12) -~~~~~~~~~~~~~~~~~~ - -* Ignore ``*.asm`` and ``*.ASM`` files when building Arduino-based library (compatibility with Arduino builder) -* Fixed spurious project's "Problems" for `PlatformIO IDE for VSCode `__ when ARM mbed framework is used -* Fixed an issue with a broken headers list when generating ".clang_complete" for `Emacs `__ - (`issue #1960 `_) - -3.6.2 (2018-11-29) -~~~~~~~~~~~~~~~~~~ - -* Improved IntelliSense for `PlatformIO IDE for VSCode `__ via passing extra compiler information for C/C++ Code Parser (resolves issues with spurious project's "Problems") -* Fixed an issue with VSCode IntelliSense warning about the missed headers located in `include `__ folder -* Fixed incorrect wording when initializing/updating project -* Fixed an issue with incorrect order for library dependencies ``CPPPATH`` - (`issue #1914 `_) -* Fixed an issue when Library Dependency Finder (LDF) does not handle project `src_filter `__ - (`issue #1905 `_) -* Fixed an issue when Library Dependency Finder (LDF) finds spurious dependencies in ``chain+`` and ``deep+`` modes - (`issue #1930 `_) - -3.6.1 (2018-10-29) -~~~~~~~~~~~~~~~~~~ - -* Generate an `include `__ and `test `__ directories with a README file when initializing a new project -* Support in-line comments for multi-line value (``lib_deps``, ``build_flags``, etc) in `"platformio.ini" (Project Configuration File) `__ -* Added ``$PROJECT_HASH`` template variable for `build_dir `__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR `__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``$[sysenv.TEMP}/pio-build/$PROJECT_HASH`` (Windows) -* Improved a loading speed of PIO Home "Recent News" -* Improved `PIO Unified Debugger `__ for "mbed" framework and fixed issue with missed local variables -* Introduced `"Release" and "Debug" Build Configurations `__ -* Build project in "Debug Mode" including debugging information with a new ``debug`` target using `platformio run `__ command or `targets `__ option in ``platformio.ini``. The last option allows avoiding project rebuilding between "Run/Debug" modes. - (`issue #1833 `_) -* Process ``build_unflags`` for the cloned environment when building a static library -* Report on outdated `99-platformio-udev.rules `__ - (`issue #1823 `_) -* Show a valid error when the Internet is off-line while initializing a new project - (`issue #1784 `_) -* Do not re-create ".gitignore" and ".travis.yml" files if they were removed from a project -* Fixed an issue when dynamic build flags were not handled correctly - (`issue #1799 `_) -* Fixed an issue when ``pio run -t monitor`` always uses the first ``monitor_port`` even with multiple environments - (`issue #1841 `_) -* Fixed an issue with broken includes when generating ``.clang_complete`` and space is used in a path - (`issue #1873 `_) -* Fixed an issue with incorrect handling of a custom package name when using `platformio lib install `__ or `platformio platform install `__ commands - -3.6.0 (2018-08-06) -~~~~~~~~~~~~~~~~~~ - -* `Program Memory Usage `_ - - - Print human-readable memory usage information after a build and before uploading - - Print detailed memory usage information with "sections" and "addresses" - in `verbose mode `__ - - Check maximum allowed "program" and "data" sizes before uploading/programming - (`issue #1412 `_) - -* `PIO Unit Testing `__: - - - Documented `Project Shared Code `__ - - Force building of project source code using `test_build_project_src `__ option - - Fixed missed ``UNIT_TEST`` macro for unit test components/libraries - -* Check package structure after unpacking and raise error when antivirus tool - blocks PlatformIO package manager - (`issue #1462 `_) -* Lock interprocess requests to PlatformIO Package Manager for - install/uninstall operations - (`issue #1594 `_) -* Fixed an issue with `PIO Remote `__ - when upload process depends on the source code of a project framework -* Fixed an issue when ``srcFilter`` field in `library.json `__ - breaks a library build - (`issue #1735 `_) - -3.5.4 (2018-07-03) -~~~~~~~~~~~~~~~~~~ - -* Improved removing of default build flags using `build_unflags `__ option - (`issue #1712 `_) -* Export ``LIBS``, ``LIBPATH``, and ``LINKFLAGS`` data from project dependent - libraries to the global build environment -* Don't export ``CPPPATH`` data of project dependent libraries to framework's - build environment - (`issue #1665 `_) -* Handle "architectures" data from "library.properties" manifest in - `lib_compat_mode = strict `__ -* Added workaround for Python SemVer package's `issue #61 `_ with caret range and pre-releases -* Replaced conflicted "env" pattern by "sysenv" for `"platformio.ini" Dynamic Variables" `__ - (`issue #1705 `_) -* Removed "date&time" when processing project with `platformio run `__ command - (`issue #1343 `_) -* Fixed issue with invalid LD script if path contains space -* Fixed preprocessor for Arduino sketch when function returns certain type - (`issue #1683 `_) -* Fixed issue when `platformio lib uninstall `__ - removes initial source code - (`issue #1023 `_) - -3.5.3 (2018-06-01) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - "Recent News" block on "Welcome" page - - Direct import of development platform's example - -* Simplify configuration for `PIO Unit Testing `__: separate main program from a test build process, drop - requirement for ``#ifdef UNIT_TEST`` guard -* Override any option from board manifest in `"platformio.ini" (Project Configuration File) `__ - (`issue #1612 `_) -* Configure a custom path to SVD file using `debug_svd_path `__ - option -* Custom project `description `_ - which will be used by `PlatformIO Home `_ -* Updated Unity tool to 2.4.3 -* Improved support for Black Magic Probe in "uploader" mode -* Renamed "monitor_baud" option to "monitor_speed" -* Fixed issue when a custom `lib_dir `__ - was not handled correctly - (`issue #1473 `_) -* Fixed issue with useless project rebuilding for case insensitive file - systems (Windows) -* Fixed issue with ``build_unflags`` option when a macro contains value - (e.g., ``-DNAME=VALUE``) -* Fixed issue which did not allow to override runtime build environment using - extra POST script -* Fixed "RuntimeError: maximum recursion depth exceeded" for library manager - (`issue #1528 `_) - -3.5.2 (2018-03-13) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - Multiple themes (Dark & Light) - - Ability to specify a name for new project - -* Control `PIO Unified Debugger `__ - and its firmware loading mode using - `debug_load_mode `__ option -* Added aliases (off, light, strict) for - `LDF Compatibility Mode `__ -* Search for a library using PIO Library Registry ID ``id:X`` (e.g. ``pio lib search id:13``) -* Show device system information (MCU, Frequency, RAM, Flash, Debugging tools) - in a build log -* Show all available upload protocols before firmware uploading in a build log -* Handle "os.mbed.com" URL as a Mercurial (hg) repository -* Improved support for old mbed libraries without manifest -* Fixed project generator for Qt Creator IDE - (`issue #1303 `_, - `issue #1323 `_) -* Mark project source and library directories for CLion IDE - (`issue #1359 `_, - `issue #1345 `_, - `issue #897 `_) -* Fixed issue with duplicated "include" records when generating data for IDE - (`issue #1301 `_) - -3.5.1 (2018-01-18) -~~~~~~~~~~~~~~~~~~ - -* New ``test_speed`` option to control a communication baudrate/speed between - `PIO Unit Testing `__ - engine and a target device - (`issue #1273 `_) -* Show full library version in "Library Dependency Graph" including VCS - information - (`issue #1274 `_) -* Configure a custom firmware/program name in build directory (`example `__) -* Renamed ``envs_dir`` option to ``build_dir`` - in `"platformio.ini" (Project Configuration File) `__ -* Refactored code without "arrow" dependency (resolve issue with "ImportError: - No module named backports.functools_lru_cache") -* Improved support of PIO Unified Debugger for Eclipse Oxygen -* Improved a work in off-line mode -* Fixed project generator for CLion and Qt Creator IDE - (`issue #1299 `_) -* Fixed PIO Unified Debugger for mbed framework -* Fixed library updates when a version is declared in VCS format (not SemVer) - -3.5.0 (2017-12-28) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - Library Manager: - - * Search for new libraries in PlatformIO Registry - * "1-click" library installation, per-project libraries, extra storages - * List installed libraries in multiple storages - * List built-in libraries (by frameworks) - * Updates for installed libraries - * Multiple examples, trending libraries, and more. - - - PlatformIO Projects - - PIO Account - - Development platforms, frameworks and board explorer - - Device Manager: serial, logical, and multicast DNS services - -* Integration with `Jenkins CI `_ -* New `include `__ - folder for project's header files - (`issue #1107 `_) -* Depend on development platform using VCS URL (Git, Mercurial and Subversion) - instead of a name in `"platformio.ini" (Project Configuration File) `__. - Drop support for ``*_stage`` dev/platform names (use VCS URL instead). -* Reinstall/redownload package with a new ``-f, --force`` option for - `platformio lib install `__ - and `platformio platform install `__ - commands - (`issue #778 `_) -* Handle missed dependencies and provide a solution based on PlatformIO Library - Registry - (`issue #781 `_) -* New setting `projects_dir `__ - that allows to override a default PIO Home Projects location - (`issue #1161 `_) - -* `Library Dependency Finder (LDF) `__: - - - Search for dependencies used in `PIO Unit Testing `__ - (`issue #953 `_) - - Parse library source file in pair with a header when they have the same name - (`issue #1175 `_) - - Handle library dependencies defined as VCS or SemVer in - `"platformio.ini" (Project Configuration File) `__ - (`issue #1155 `_) - - Added option to configure library `Compatible Mode `__ - using `library.json `__ - -* New options for `platformio device list `__ - command: - - - ``--serial`` list available serial ports (default) - - ``--logical`` list logical devices - - ``--mdns`` discover multicast DNS services - (`issue #463 `_) - -* Fixed platforms, packages, and libraries updating behind proxy - (`issue #1061 `_) -* Fixed missing toolchain include paths for project generator - (`issue #1154 `_) -* Fixed "Super-Quick (Mac / Linux)" installation in "get-platformio.py" script - (`issue #1017 `_) -* Fixed "get-platformio.py" script which hangs on Windows 10 - (`issue #1118 `_) -* Other bug fixes and performance improvements - -3.4.1 (2017-08-02) -~~~~~~~~~~~~~~~~~~ - -* Pre/Post extra scripting for advanced control of PIO Build System - (`issue #891 `_) -* New `lib_archive `_ - option to control library archiving and linking behavior - (`issue #993 `_) -* Add "inc" folder automatically to CPPPATH when "src" is available (works for project and library) - (`issue #1003 `_) -* Use a root of library when filtering source code using - `library.json `__ - and ``srcFilter`` field -* Added ``monitor_*`` options to white-list for `"platformio.ini" (Project Configuration File) `__ - (`issue #982 `_) -* Do not ask for board ID when initialize project for desktop platform -* Handle broken PIO Core state and create new one -* Fixed an issue with a custom transport for `PIO Unit Testing `__ - when multiple tests are present -* Fixed an issue when can not upload firmware to SAM-BA based board (Due) - -3.4.0 (2017-06-26) -~~~~~~~~~~~~~~~~~~ - -* `PIO Unified Debugger `__ - - - "1-click" solution, zero configuration - - Support for 100+ embedded boards - - Multiple architectures and development platforms - - Windows, MacOS, Linux (+ARMv6-8) - - Built-in into `PlatformIO IDE for Atom `__ and `PlatformIO IDE for VScode `__ - - Integration with `Eclipse `__ and `Sublime Text `__ - -* Filter `PIO Unit Testing `__ - tests using a new ``test_filter`` option in `"platformio.ini" (Project Configuration File) `__ - or `platformio test --filter `__ command - (`issue #934 `_) -* Custom ``test_transport`` for `PIO Unit Testing `__ Engine -* Configure Serial Port Monitor in `"platformio.ini" (Project Configuration File) `__ - (`issue #787 `_) -* New `monitor `__ - target which allows to launch Serial Monitor automatically after successful - "build" or "upload" operations - (`issue #788 `_) -* Project generator for `VIM `__ -* Multi-line support for the different options in `"platformio.ini" (Project Configuration File) `__, - such as: ``build_flags``, ``build_unflags``, etc. - (`issue #889 `_) -* Handle dynamic ``SRC_FILTER`` environment variable from - `library.json extra script `__ -* Notify about multiple installations of PIO Core - (`issue #961 `_) -* Improved auto-detecting of mbed-enabled media disks -* Automatically update Git-submodules for development platforms and libraries - that were installed from repository -* Add support for ``.*cc`` extension - (`issue #939 `_) -* Handle ``env_default`` in `"platformio.ini" (Project Configuration File) `__ - when re-initializing a project - (`issue #950 `_) -* Use root directory for PIO Home when path contains non-ascii characters - (`issue #951 `_, - `issue #952 `_) -* Don't warn about known ``boards_dir`` option - (`pull #949 `_) -* Escape non-valid file name characters when installing a new package (library) - (`issue #985 `_) -* Fixed infinite dependency installing when repository consists of multiple - libraries - (`issue #935 `_) -* Fixed linter error "unity.h does not exist" for Unit Testing - (`issue #947 `_) -* Fixed issue when `Library Dependency Finder (LDF) `__ - does not handle custom ``src_dir`` - (`issue #942 `_) -* Fixed cloning a package (library) from a private Git repository with - custom user name and SSH port - (`issue #925 `_) - -3.3.1 (2017-05-27) -~~~~~~~~~~~~~~~~~~ - -* Hotfix for recently updated Python Requests package (2.16.0) - -3.3.0 (2017-03-27) -~~~~~~~~~~~~~~~~~~ - -* PlatformIO Library Registry statistics with new - `pio lib stats `__ command - - - Recently updated and added libraries - - Recent and popular keywords - - Featured libraries (today, week, month) - -* List built-in libraries based on development platforms with a new - `pio lib builtin `__ command -* Show detailed info about a library using `pio lib show `__ - command - (`issue #430 `_) -* List supported frameworks, SDKs with a new - `pio platform frameworks `__ command -* Visual Studio Code extension for PlatformIO - (`issue #619 `_) -* Added new options ``--no-reset``, ``--monitor-rts`` and ``--monitor-dtr`` - to `pio test `__ - command (allows to avoid automatic board's auto-reset when gathering test results) -* Added support for templated methods in ``*.ino to *.cpp`` converter - (`pull #858 `_) -* Package version as "Repository URL" in manifest of development version - (``"version": "https://github.com/user/repo.git"``) -* Produce less noisy output when ``-s/--silent`` options are used for - `platformio init `__ - and `platformio run `__ - commands - (`issue #850 `_) -* Use C++11 by default for CLion IDE based projects - (`pull #873 `_) -* Escape project path when Glob matching is used -* Do not overwrite project configuration variables when system environment - variables are set -* Handle dependencies when installing non-registry package/library (VCS, archive, local folder) - (`issue #913 `_) -* Fixed package installing with VCS branch for Python 2.7.3 - (`issue #885 `_) - -3.2.1 (2016-12-07) -~~~~~~~~~~~~~~~~~~ - -* Changed default `LDF Mode `__ - from ``chain+`` to ``chain`` - -3.2.0 (2016-12-07) -~~~~~~~~~~~~~~~~~~ - -* `PIO Remote™ `__. - **Your devices are always with you!** - - + Over-The-Air (OTA) Device Manager - + OTA Serial Port Monitor - + OTA Firmware Updates - + Continuous Deployment - + Continuous Delivery - -* Integration with `Cloud IDEs `__ - - + Cloud9 - + Codeanywhere - + Eclipse Che - -* `PIO Account `__ - and `PLATFORMIO_AUTH_TOKEN `__ - environment variable for CI systems - (`issue #808 `_, - `issue #467 `_) -* Inject system environment variables to configuration settings in - `"platformio.ini" (Project Configuration File) `__ - (`issue #792 `_) -* Custom boards per project with ``boards_dir`` option in - `"platformio.ini" (Project Configuration File) `__ - (`issue #515 `_) -* Unix shell-style wildcards for `upload_port `_ - (`issue #839 `_) -* Refactored `Library Dependency Finder (LDF) `__ - C/C++ Preprocessor for conditional syntax (``#ifdef``, ``#if``, ``#else``, - ``#elif``, ``#define``, etc.) - (`issue #837 `_) -* Added new `LDF Modes `__: - ``chain+`` and ``deep+`` and set ``chain+`` as default -* Added global ``lib_extra_dirs`` option to ``[platformio]`` section for - `"platformio.ini" (Project Configuration File) `__ - (`issue #842 `_) -* Enabled caching by default for API requests and Library Manager (see `enable_cache `__ setting) -* Native integration with VIM/Neovim using `neomake-platformio `__ plugin -* Changed a default exit combination for Device Monitor from ``Ctrl+]`` to ``Ctrl+C`` -* Improved detecting of ARM mbed media disk for uploading -* Improved Project Generator for CLion IDE when source folder contains nested items -* Improved handling of library dependencies specified in ``library.json`` manifest - (`issue #814 `_) -* Improved `Library Dependency Finder (LDF) `__ - for circular dependencies -* Show vendor version of a package for `platformio platform show `__ command - (`issue #838 `_) -* Fixed unable to include SSH user in ``lib_deps`` repository url - (`issue #830 `_) -* Fixed merging of ".gitignore" files when re-initialize project - (`issue #848 `_) -* Fixed issue with ``PATH`` auto-configuring for upload tools -* Fixed ``99-platformio-udev.rules`` checker for Linux OS - -3.1.0 (2016-09-19) -~~~~~~~~~~~~~~~~~~ - -* New! Dynamic variables/templates for `"platformio.ini" (Project Configuration File) `__ - (`issue #705 `_) -* Summary about processed environments - (`issue #777 `_) -* Implemented LocalCache system for API and improved a work in off-line mode -* Improved Project Generator when custom ``--project-option`` is passed to - `platformio init `__ - command -* Deprecated ``lib_force`` option, please use `lib_deps `__ instead -* Return valid exit code from ``plaformio test`` command -* Fixed Project Generator for CLion IDE using Windows OS - (`issue #785 `_) -* Fixed SSL Server-Name-Indication for Python < 2.7.9 - (`issue #774 `_) - -3.0.1 (2016-09-08) -~~~~~~~~~~~~~~~~~~ - -* Disabled temporary SSL for PlatformIO services - (`issue #772 `_) - -3.0.0 (2016-09-07) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Plus `__ - - + Local and Embedded `Unit Testing `__ - (`issue #408 `_, - `issue #519 `_) - -* Decentralized Development Platforms - - + Development platform manifest "platform.json" and - `open source development platforms `__ - + `Semantic Versioning `__ for platform commands, - development platforms and dependent packages - + Custom package repositories - + External embedded board configuration files, isolated build scripts - (`issue #479 `_) - + Embedded Board compatibility with more than one development platform - (`issue #456 `_) - -* Library Manager 3.0 - - + Project dependencies per build environment using `lib_deps `__ option - (`issue #413 `_) - + `Semantic Versioning `__ for library commands and - dependencies - (`issue #410 `_) - + Multiple library storages: Project's Local, PlatformIO's Global or Custom - (`issue #475 `_) - + Install library by name - (`issue #414 `_) - + Depend on a library using VCS URL (GitHub, Git, ARM mbed code registry, Hg, SVN) - (`issue #498 `_) - + Strict search for library dependencies - (`issue #588 `_) - + Allowed ``library.json`` to specify sources other than PlatformIO's Repository - (`issue #461 `_) - + Search libraries by headers/includes with ``platformio lib search --header`` option - -* New Intelligent Library Build System - - + `Library Dependency Finder `__ - that interprets C/C++ Preprocessor conditional macros with deep search behavior - + Check library compatibility with project environment before building - (`issue #415 `_) - + Control Library Dependency Finder for compatibility using - `lib_compat_mode `__ - option - + Custom library storages/directories with - `lib_extra_dirs `__ option - (`issue #537 `_) - + Handle extra build flags, source filters and build script from - `library.json `__ - (`issue #289 `_) - + Allowed to disable library archiving (``*.ar``) - (`issue #719 `_) - + Show detailed build information about dependent libraries - (`issue #617 `_) - + Support for the 3rd party manifests (Arduino IDE "library.properties" - and ARM mbed "module.json") - -* Removed ``enable_prompts`` setting. Now, all PlatformIO CLI is non-blocking! -* Switched to SSL PlatformIO API -* Renamed ``platformio serialports`` command to ``platformio device`` -* Build System: Attach custom Before/Pre and After/Post actions for targets - (`issue #542 `_) -* Allowed passing custom project configuration options to ``platformio ci`` - and ``platformio init`` commands using ``-O, --project-option``. -* Print human-readable information when processing environments without - ``-v, --verbose`` option - (`issue #721 `_) -* Improved INO to CPP converter - (`issue #659 `_, - `issue #765 `_) -* Added ``license`` field to `library.json `__ - (`issue #522 `_) -* Warn about unknown options in project configuration file ``platformio.ini`` - (`issue #740 `_) -* Fixed wrong line number for INO file when ``#warning`` directive is used - (`issue #742 `_) -* Stopped supporting Python 2.6 +See `PlatformIO Core 3.0 history `__. PlatformIO Core 2 ----------------- -2.11.2 (2016-08-02) -~~~~~~~~~~~~~~~~~~~ - -* Improved support for `Microchip PIC32 `__ development platform and ChipKIT boards - (`issue #438 `_) -* Added support for Pinoccio Scout board - (`issue #52 `_) -* Added support for `Teensy USB Features `__ - (HID, SERIAL_HID, DISK, DISK_SDFLASH, MIDI, etc.) - (`issue #722 `_) -* Switched to built-in GCC LwIP library for Espressif development platform -* Added support for local ``--echo`` for Serial Port Monitor - (`issue #733 `_) -* Updated ``udev`` rules for the new STM32F407DISCOVERY boards - (`issue #731 `_) -* Implemented firmware merging with base firmware for Nordic nRF51 development platform - (`issue #500 `_, - `issue #533 `_) -* Fixed Project Generator for ESP8266 and ARM mbed based projects - (resolves incorrect linter errors) -* Fixed broken LD Script for Element14 chipKIT Pi board - (`issue #725 `_) -* Fixed firmware uploading to Atmel SAMD21-XPRO board using ARM mbed framework - (`issue #732 `_) - -2.11.1 (2016-07-12) -~~~~~~~~~~~~~~~~~~~ - -* Added support for Arduino M0, M0 Pro and Tian boards - (`issue #472 `_) -* Added support for Microchip chipKIT Lenny board -* Updated Microchip PIC32 Arduino framework to v1.2.1 -* Documented `uploading of EEPROM data `__ - (from EEMEM directive) -* Added ``Rebuild C/C++ Project Index`` target to CLion and Eclipse IDEs -* Improved project generator for `CLion IDE `__ -* Added ``udev`` rules for OpenOCD CMSIS-DAP adapters - (`issue #718 `_) -* Auto-remove project cache when PlatformIO is upgraded -* Keep user changes for ``.gitignore`` file when re-generate/update project data -* Ignore ``[platformio]`` section from custom project configuration file when - `platformio ci --project-conf `__ - command is used -* Fixed missed ``--boot`` flag for the firmware uploader for ATSAM3X8E - Cortex-M3 MCU based boards (Arduino Due, etc) - (`issue #710 `_) -* Fixed missing trailing ``\`` for the source files list when generate project - for `Qt Creator IDE `__ - (`issue #711 `_) -* Split source files to ``HEADERS`` and ``SOURCES`` when generate project - for `Qt Creator IDE `__ - (`issue #713 `_) - -2.11.0 (2016-06-28) -~~~~~~~~~~~~~~~~~~~ - -* New ESP8266-based boards: Generic ESP8285 Module, Phoenix 1.0 & 2.0, WifInfo -* Added support for Arduino M0 Pro board - (`issue #472 `_) -* Added support for Arduino MKR1000 board - (`issue #620 `_) -* Added support for Adafruit Feather M0, SparkFun SAMD21 and SparkFun SAMD21 - Mini Breakout boards - (`issue #520 `_) -* Updated Arduino ESP8266 core for Espressif platform to 2.3.0 -* Better removing unnecessary flags using ``build_unflags`` option - (`issue #698 `_) -* Fixed issue with ``platformio init --ide`` command for Python 2.6 - -2.10.3 (2016-06-15) -~~~~~~~~~~~~~~~~~~~ - -* Fixed issue with ``platformio init --ide`` command - -2.10.2 (2016-06-15) -~~~~~~~~~~~~~~~~~~~ - -* Added support for ST Nucleo L031K6 board to ARM mbed framework -* Process ``build_unflags`` option for ARM mbed framework -* Updated Intel ARC32 Arduino framework to v1.0.6 - (`issue #695 `_) -* Improved a check of program size before uploading to the board -* Fixed issue with ARM mbed framework ``-u _printf_float`` and - ``-u _scanf_float`` when parsing ``$LINKFLAGS`` -* Fixed issue with ARM mbed framework and extra includes for the custom boards, - such as Seeeduino Arch Pro - -2.10.1 (2016-06-13) -~~~~~~~~~~~~~~~~~~~ - -* Re-submit a package to PyPI - -2.10.0 (2016-06-13) -~~~~~~~~~~~~~~~~~~~ - -* Added support for `emonPi `__, - the OpenEnergyMonitor system - (`issue #687 `_) -* Added support for `SPL `__ - framework for STM32F0 boards - (`issue #683 `_) -* Added support for `Arduboy DevKit `__, the game system - the size of a credit card -* Updated ARM mbed framework package to v121 -* Check program size before uploading to the board - (`issue #689 `_) -* Improved firmware uploading to Arduino Leonardo based boards - (`issue #691 `_) -* Fixed issue with ``-L relative/path`` when parsing ``build_flags`` - (`issue #688 `_) - -2.9.4 (2016-06-04) -~~~~~~~~~~~~~~~~~~ - -* Show ``udev`` warning only for the Linux OS while uploading firmware - -2.9.3 (2016-06-03) -~~~~~~~~~~~~~~~~~~ - -* Added support for `Arduboy `__, the game system - the size of a credit card -* Updated `99-platformio-udev.rules `__ for Linux OS -* Refactored firmware uploading to the embedded boards with SAM-BA bootloader - -2.9.2 (2016-06-02) -~~~~~~~~~~~~~~~~~~ - -* Simplified `Continuous Integration with AppVeyor `__ - (`issue #671 `_) -* Automatically add source directory to ``CPPPATH`` of Build System -* Added support for Silicon Labs SLSTK3401A (Pearl Gecko) and - MultiTech mDot F411 ARM mbed based boards -* Added support for MightyCore ATmega8535 board - (`issue #585 `_) -* Added ``stlink`` as the default uploader for STM32 Discovery boards - (`issue #665 `_) -* Use HTTP mirror for Package Manager in a case with SSL errors - (`issue #645 `_) -* Improved firmware uploading to Arduino Leonardo/Due based boards -* Fixed bug with ``env_default`` when ``pio run -e`` is used -* Fixed issue with ``src_filter`` option for Windows OS - (`issue #652 `_) -* Fixed configuration data for TI LaunchPads based on msp430fr4133 and - msp430fr6989 MCUs - (`issue #676 `_) -* Fixed issue with ARM mbed framework and multiple definition errors - on FRDM-KL46Z board - (`issue #641 `_) -* Fixed issue with ARM mbed framework when abstract class breaks compile - for LPC1768 - (`issue #666 `_) - -2.9.1 (2016-04-30) -~~~~~~~~~~~~~~~~~~ - -* Handle prototype pointers while converting ``*.ino`` to ``.cpp`` - (`issue #639 `_) - -2.9.0 (2016-04-28) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `CodeBlocks IDE `__ - (`issue #600 `_) -* New `Lattice iCE40 FPGA `__ - development platform with support for Lattice iCEstick FPGA Evaluation - Kit and BQ IceZUM Alhambra FPGA - (`issue #480 `_) -* New `Intel ARC 32-bit `_ - development platform with support for Arduino/Genuino 101 board - (`issue #535 `_) -* New `Microchip PIC32 `__ - development platform with support for 20+ different PIC32 based boards - (`issue #438 `_) -* New RTOS and build Framework named `Simba `__ - (`issue #412 `_) -* New boards for `ARM mbed `__ - framework: ST Nucleo F410RB, ST Nucleo L073RZ and BBC micro:bit -* Added support for Arduino.Org boards: Arduino Leonardo ETH, Arduino Yun Mini, - Arduino Industrial 101 and Linino One - (`issue #472 `_) -* Added support for Generic ATTiny boards: ATTiny13, ATTiny24, ATTiny25, - ATTiny45 and ATTiny85 - (`issue #636 `_) -* Added support for MightyCore boards: ATmega1284, ATmega644, ATmega324, - ATmega164, ATmega32, ATmega16 and ATmega8535 - (`issue #585 `_) -* Added support for `TI MSP430 `__ - boards: TI LaunchPad w/ msp430fr4133 and TI LaunchPad w/ msp430fr6989 -* Updated Arduino core for Espressif platform to 2.2.0 - (`issue #627 `_) -* Updated native SDK for ESP8266 to 1.5 - (`issue #366 `_) -* PlatformIO Library Registry in JSON format! Implemented - ``--json-output`` and ``--page`` options for - `platformio lib search `__ - command - (`issue #604 `_) -* Allowed to specify default environments `env_default `__ - which should be processed by default with ``platformio run`` command - (`issue #576 `_) -* Allowed to unflag(remove) base/initial flags using - `build_unflags `__ - option - (`issue #559 `_) -* Allowed multiple VID/PID pairs when detecting serial ports - (`issue #632 `_) -* Automatically add ``-DUSB_MANUFACTURER`` with vendor's name - (`issue #631 `_) -* Automatically reboot Teensy board after upload when Teensy Loader GUI is used - (`issue #609 `_) -* Refactored source code converter from ``*.ino`` to ``*.cpp`` - (`issue #610 `_) -* Forced ``-std=gnu++11`` for Atmel SAM development platform - (`issue #601 `_) -* Don't check OS type for ARM mbed-enabled boards and ST STM32 development - platform before uploading to disk - (`issue #596 `_) -* Fixed broken compilation for Atmel SAMD based boards except Arduino Due - (`issue #598 `_) -* Fixed firmware uploading using serial port with spaces in the path -* Fixed cache system when project's root directory is used as ``src_dir`` - (`issue #635 `_) - -2.8.6 (2016-03-22) -~~~~~~~~~~~~~~~~~~ - -* Launched `PlatformIO Community Forums `_ - (`issue #530 `_) -* Added support for ARM mbed-enabled board Seed Arch Max (STM32F407VET6) - (`issue #572 `_) -* Improved DNS lookup for PlatformIO API -* Updated Arduino Wiring-based framework to the latest version for - Atmel AVR/SAM development platforms -* Updated "Teensy Loader CLI" and fixed uploading of large .hex files - (`issue #568 `_) -* Updated the support for Sanguino Boards - (`issue #586 `_) -* Better handling of used boards when re-initialize/update project -* Improved support for non-Unicode user profiles for Windows OS -* Disabled progress bar for download operations when prompts are disabled -* Fixed multiple definition errors for ST STM32 development platform and - ARM mbed framework - (`issue #571 `_) -* Fixed invalid board parameters (reset method and baudrate) for a few - ESP8266 based boards -* Fixed "KeyError: 'content-length'" in PlatformIO Download Manager - (`issue #591 `_) - - -2.8.5 (2016-03-07) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `NetBeans IDE `__ - (`issue #541 `_) -* Created package for Homebrew Mac OS X Package Manager: ``brew install - platformio`` - (`issue #395 `_) -* Updated Arduino core for Espressif platform to 2.1.0 - (`issue #544 `_) -* Added support for the ESP8266 ESP-07 board to - `Espressif `__ - (`issue #527 `_) -* Improved handling of String-based ``CPPDEFINES`` passed to extra ``build_flags`` - (`issue #526 `_) -* Generate appropriate project for CLion IDE and CVS - (`issue #523 `_) -* Use ``src_dir`` directory from `Project Configuration File platformio.ini `__ - when initializing project otherwise create base ``src`` directory - (`issue #536 `_) -* Fixed issue with incorrect handling of user's build flags where the base flags - were passed after user's flags to GCC compiler - (`issue #528 `_) -* Fixed issue with Project Generator when optional build flags were passed using - system environment variables: `PLATFORMIO_BUILD_FLAGS `__ - or `PLATFORMIO_BUILD_SRC_FLAGS `__ -* Fixed invalid detecting of compiler type - (`issue #550 `_) -* Fixed issue with updating package which was deleted manually by user - (`issue #555 `_) -* Fixed incorrect parsing of GCC ``-include`` flag - (`issue #552 `_) - -2.8.4 (2016-02-17) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new ESP8266-based boards (ESPDuino, ESP-WROOM-02, - ESPresso Lite 1.0 & 2.0, SparkFun ESP8266 Thing Dev, ThaiEasyElec ESPino) to - `Espressif `__ - development platform -* Added ``board_f_flash`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom flash chip frequency `_ - for Espressif development platform - (`issue #501 `_) -* Added ``board_flash_mode`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom flash chip mode `_ - for Espressif development platform -* Handle new environment variables - `PLATFORMIO_UPLOAD_PORT `_ - and `PLATFORMIO_UPLOAD_FLAGS `_ - (`issue #518 `_) -* Fixed issue with ``CPPDEFINES`` which contain space and break PlatformIO - IDE Linter - (`IDE issue #34 `_) -* Fixed unable to link C++ standard library to Espressif platform build - (`issue #503 `_) -* Fixed issue with pointer (``char* myfunc()``) while converting from ``*.ino`` - to ``*.cpp`` - (`issue #506 `_) - -2.8.3 (2016-02-02) -~~~~~~~~~~~~~~~~~~ - -* Better integration of PlatformIO Builder with PlatformIO IDE Linter -* Fixed issue with removing temporary file while converting ``*.ino`` to - ``*.cpp`` -* Fixed missing dependency (mbed framework) for Atmel SAM development platform - (`issue #487 `_) - -2.8.2 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* Corrected RAM size for NXP LPC1768 based boards - (`issue #484 `_) -* Exclude only ``test`` and ``tests`` folders from build process -* Reverted ``-Wl,-whole-archive`` hook for ST STM32 and mbed - -2.8.1 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* Fixed a bug with Project Initialization in PlatformIO IDE - -2.8.0 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO IDE `_ for - Atom - (`issue #470 `_) -* Added ``pio`` command line alias for ``platformio`` command - (`issue #447 `_) -* Added SPL-Framework support for Nucleo F401RE board - (`issue #453 `_) -* Added ``upload_resetmethod`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom upload reset method `_ - for Espressif development platform - (`issue #444 `_) -* Allowed to force output of color ANSI-codes or to disable progress bar even - if the output is a ``pipe`` (not a ``tty``) using `Environment variables `__ - (`issue #465 `_) -* Set 1Mb SPIFFS for Espressif boards by default - (`issue #458 `_) -* Exclude ``test*`` folder by default from build process -* Generate project for IDEs with information about installed libraries -* Fixed builder for mbed framework and ST STM32 platform - - -2.7.1 (2016-01-06) -~~~~~~~~~~~~~~~~~~ - -* Initial support for Arduino Zero board - (`issue #356 `_) -* Added support for completions to Atom text editor using ``.clang_complete`` -* Generate default targets for `supported IDE `__ - (CLion, Eclipse IDE, Emacs, Sublime Text, VIM): Build, - Clean, Upload, Upload SPIFFS image, Upload using Programmer, Update installed - platforms and libraries - (`issue #427 `_) -* Updated Teensy Arduino Framework to 1.27 - (`issue #434 `_) -* Fixed uploading of EEPROM data using ``uploadeep`` target for Atmel AVR - development platform -* Fixed project generator for CLion IDE - (`issue #422 `_) -* Fixed package ``shasum`` validation on Mac OS X 10.11.2 - (`issue #429 `_) -* Fixed CMakeLists.txt ``add_executable`` has only one source file - (`issue #421 `_) - -2.7.0 (2015-12-30) -~~~~~~~~~~~~~~~~~~ - -**Happy New Year!** - -* Moved SCons to PlatformIO packages. PlatformIO does not require SCons to be - installed in your system. Significantly simplified installation process of - PlatformIO. ``pip install platformio`` rocks! -* Implemented uploading files to file system of ESP8266 SPIFFS (including OTA) - (`issue #382 `_) -* Added support for the new Adafruit boards Bluefruit Micro and Feather - (`issue #403 `_) -* Added support for RFDuino - (`issue #319 `_) -* Project generator for `Emacs `__ - text editor - (`pull #404 `_) -* Updated Arduino framework for Atmel AVR development platform to 1.6.7 -* Documented `firmware uploading for Atmel AVR development platform using - Programmers `_: - AVR ISP, AVRISP mkII, USBtinyISP, USBasp, Parallel Programmer and Arduino as ISP -* Fixed issue with current Python interpreter for Python-based tools - (`issue #417 `_) - -2.6.3 (2015-12-21) -~~~~~~~~~~~~~~~~~~ - -* Restored support for Espressif ESP8266 ESP-01 1MB board (ready for OTA) -* Fixed invalid ROM size for ESP8266-based boards - (`issue #396 `_) - -2.6.2 (2015-12-21) -~~~~~~~~~~~~~~~~~~ - -* Removed ``SCons`` from requirements list. PlatformIO will try to install it - automatically, otherwise users need to install it manually -* Fixed ``ChunkedEncodingError`` when SF connection is broken - (`issue #356 `_) - -2.6.1 (2015-12-18) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new ESP8266-based boards (SparkFun ESP8266 Thing, - NodeMCU 0.9 & 1.0, Olimex MOD-WIFI-ESP8266(-DEV), Adafruit HUZZAH ESP8266, - ESPino, SweetPea ESP-210, WeMos D1, WeMos D1 mini) to - `Espressif `__ - development platform -* Created public `platformio-pkg-ldscripts `_ - repository for LD scripts. Moved common configuration for ESP8266 MCU to - ``esp8266.flash.common.ld`` - (`issue #379 `_) -* Improved documentation for `Espressif `__ - development platform: OTA update, custom Flash Size, Upload Speed and CPU - frequency -* Fixed reset method for Espressif NodeMCU (ESP-12E Module) - (`issue #380 `_) -* Fixed issue with code builder when build path contains spaces - (`issue #387 `_) -* Fixed project generator for Eclipse IDE and "duplicate path entries found - in project path" - (`issue #383 `_) - - -2.6.0 (2015-12-15) -~~~~~~~~~~~~~~~~~~ - -* Install only required packages depending on build environment - (`issue #308 `_) -* Added support for Raspberry Pi `WiringPi `__ - framework - (`issue #372 `_) -* Implemented Over The Air (OTA) upgrades for `Espressif `__ - development platform. - (`issue #365 `_) -* Updated `CMSIS framework `__ - and added CMSIS support for Nucleo F401RE board - (`issue #373 `_) -* Added support for Espressif ESP8266 ESP-01-1MB board (ready for OTA) -* Handle ``upload_flags`` option in `platformio.ini `__ - (`issue #368 `_) -* Improved PlatformIO installation on the Mac OS X El Capitan - -2.5.0 (2015-12-08) -~~~~~~~~~~~~~~~~~~ - -* Improved code builder for parallel builds (up to 4 times faster than before) -* Generate `.travis.yml `__ - CI and `.gitignore` files for embedded projects by default - (`issue #354 `_) -* Removed prompt with "auto-uploading" from `platformio init `__ - command and added ``--enable-auto-uploading`` option - (`issue #352 `_) -* Fixed incorrect behaviour of `platformio serialports monitor `__ - in pair with PySerial 3.0 - -2.4.1 (2015-12-01) -~~~~~~~~~~~~~~~~~~ - -* Restored ``PLATFORMIO`` macros with the current version - -2.4.0 (2015-12-01) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new boards: Atmel ATSAMR21-XPRO, Atmel SAML21-XPRO-B, - Atmel SAMD21-XPRO, ST 32F469IDISCOVERY, ST 32L476GDISCOVERY, ST Nucleo F031K6, - ST Nucleo F042K6, ST Nucleo F303K8 and ST Nucleo L476RG -* Updated Arduino core for Espressif platform to 2.0.0 - (`issue #345 `_) -* Added to FAQ explanation of `Can not compile a library that compiles without issue - with Arduino IDE `_ - (`issue #331 `_) -* Fixed ESP-12E flash size - (`pull #333 `_) -* Fixed configuration for LowPowerLab MoteinoMEGA board - (`issue #335 `_) -* Fixed "LockFailed: failed to create appstate.json.lock" error for Windows -* Fixed relative include path for preprocessor using ``build_flags`` - (`issue #271 `_) - -2.3.5 (2015-11-18) -~~~~~~~~~~~~~~~~~~ - -* Added support of `libOpenCM3 `_ - framework for Nucleo F103RB board - (`issue #309 `_) -* Added support for Espressif ESP8266 ESP-12E board (NodeMCU) - (`issue #310 `_) -* Added support for pySerial 3.0 - (`issue #307 `_) -* Updated Arduino AVR/SAM frameworks to 1.6.6 - (`issue #321 `_) -* Upload firmware using external programmer via `platformio run --target program `__ - target - (`issue #311 `_) -* Fixed handling of upload port when ``board`` option is not specified in - `platformio.ini `__ - (`issue #313 `_) -* Fixed firmware uploading for `nordicrf51 `__ - development platform - (`issue #316 `_) -* Fixed installation on Mac OS X El Capitan - (`issue #312 `_) -* Fixed project generator for CLion IDE under Windows OS with invalid path to - executable - (`issue #326 `_) -* Fixed empty list with serial ports on Mac OS X - (`isge #294 `_) -* Fixed compilation error ``TWI_Disable not declared`` for Arduino Due board - (`issue #329 `_) - -2.3.4 (2015-10-13) -~~~~~~~~~~~~~~~~~~ - -* Full support of `CLion IDE `_ - including code auto-completion - (`issue #132 `_) -* PlatformIO `command completion in Terminal `_ for ``bash`` and ``zsh`` -* Added support for ubIQio Ardhat board - (`pull #302 `_) -* Install SCons automatically and avoid ``error: option --single-version-externally-managed not recognized`` - (`issue #279 `_) -* Use Teensy CLI Loader for upload of .hex files on Mac OS X - (`issue #306 `_) -* Fixed missing `framework-mbed `_ - package for `teensy `_ - platform - (`issue #305 `_) - -2.3.3 (2015-10-02) -~~~~~~~~~~~~~~~~~~ - -* Added support for LightBlue Bean board - (`pull #292 `_) -* Added support for ST Nucleo F446RE board - (`pull #293 `_) -* Fixed broken lock file for "appstate" storage - (`issue #288 `_) -* Fixed ESP8266 compile errors about RAM size when adding 1 library - (`issue #296 `_) - -2.3.2 (2015-09-10) -~~~~~~~~~~~~~~~~~~ - -* Allowed to use ST-Link uploader for mbed-based projects -* Explained how to use ``lib`` directory from the PlatformIO based project in - ``readme.txt`` which will be automatically generated using - `platformio init `__ - command - (`issue #273 `_) -* Found solution for "pip/scons error: option --single-version-externally-managed not - recognized" when install PlatformIO using ``pip`` package manager - (`issue #279 `_) -* Fixed firmware uploading to Arduino Leonardo board using Mac OS - (`issue #287 `_) -* Fixed `SConsNotInstalled` error for Linux Debian-based distributives - -2.3.1 (2015-09-06) -~~~~~~~~~~~~~~~~~~ - -* Fixed critical issue when `platformio init --ide `__ command hangs PlatformIO - (`issue #283 `_) - -2.3.0 (2015-09-05) -~~~~~~~~~~~~~~~~~~ - -* Added - `native `__, - `linux_arm `__, - `linux_i686 `__, - `linux_x86_64 `__, - `windows_x86 `__ - development platforms - (`issue #263 `_) -* Added `PlatformIO Demo `_ - page to documentation -* Simplified `installation `__ - process of PlatformIO - (`issue #274 `_) -* Significantly improved `Project Generator `__ which allows to integrate with `the most popular - IDE `__ -* Added short ``-h`` help option for PlatformIO and sub-commands -* Updated `mbed `__ - framework -* Updated ``tool-teensy`` package for `Teensy `__ - platform - (`issue #268 `_) -* Added FAQ answer when `Program "platformio" not found in PATH `_ - (`issue #272 `_) -* Generate "readme.txt" for project "lib" directory - (`issue #273 `_) -* Use toolchain's includes pattern ``include*`` for Project Generator - (`issue #277 `_) -* Added support for Adafruit Gemma board to - `atmelavr `__ - platform - (`pull #256 `_) -* Fixed includes list for Windows OS when generating project for `Eclipse IDE `__ - (`issue #270 `_) -* Fixed ``AttributeError: 'module' object has no attribute 'packages'`` - (`issue #252 `_) - -2.2.2 (2015-07-30) -~~~~~~~~~~~~~~~~~~ - -* Integration with `Atom IDE `__ -* Support for off-line/unpublished/private libraries - (`issue #260 `_) -* Disable project auto-clean while building/uploading firmware using - `platformio run --disable-auto-clean `_ option - (`issue #255 `_) -* Show internal errors from "Miniterm" using `platformio serialports monitor `__ command - (`issue #257 `_) -* Fixed `platformio serialports monitor --help `__ information with HEX char for hotkeys - (`issue #253 `_) -* Handle "OSError: [Errno 13] Permission denied" for PlatformIO installer script - (`issue #254 `_) - -2.2.1 (2015-07-17) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `CLion IDE `__ - (`issue #132 `_) -* Updated ``tool-bossac`` package to 1.5 version for `atmelsam `__ platform - (`issue #251 `_) -* Updated ``sdk-esp8266`` package for `espressif `__ platform -* Fixed incorrect arguments handling for `platformio serialports monitor `_ command - (`issue #248 `_) - -2.2.0 (2015-07-01) -~~~~~~~~~~~~~~~~~~ - -* Allowed to exclude/include source files from build process using - `src_filter `__ - (`issue #240 `_) -* Launch own extra script before firmware building/uploading processes - (`issue #239 `_) -* Specify own path to the linker script (ld) using - `build_flags `__ - option - (`issue #233 `_) -* Specify library compatibility with the all platforms/frameworks - using ``*`` symbol in - `library.json `__ -* Added support for new embedded boards: *ST 32L0538DISCOVERY and Delta DFCM-NNN40* - to `Framework mbed `__ -* Updated packages for - `Framework Arduino (AVR, SAM, Espressif and Teensy cores `__, - `Framework mbed `__, - `Espressif ESP8266 SDK `__ - (`issue #246 `_) -* Fixed ``stk500v2_command(): command failed`` - (`issue #238 `_) -* Fixed IDE project generator when board is specified - (`issue #242 `_) -* Fixed relative path for includes when generating project for IDE - (`issue #243 `_) -* Fixed ESP8266 native SDK exception - (`issue #245 `_) - -2.1.2 (2015-06-21) -~~~~~~~~~~~~~~~~~~ - -* Fixed broken link to SCons installer - -2.1.1 (2015-06-09) -~~~~~~~~~~~~~~~~~~ - -* Automatically detect upload port using VID:PID board settings - (`issue #231 `_) -* Improved detection of build changes -* Avoided ``LibInstallDependencyError`` when more than 1 library is found - (`issue #229 `_) - -2.1.0 (2015-06-03) -~~~~~~~~~~~~~~~~~~ - -* Added Silicon Labs EFM32 `siliconlabsefm32 `_ - development platform - (`issue #226 `_) -* Integrate PlatformIO with `Circle CI `_ and - `Shippable CI `_ -* Described in documentation how to `create/register own board `_ for PlatformIO -* Disabled "nano.specs" for ARM-based platforms - (`issue #219 `_) -* Fixed "ConnectionError" when PlatformIO SF Storage is off-line -* Fixed resolving of C/C++ std libs by Eclipse IDE - (`issue #220 `_) -* Fixed firmware uploading using USB programmer (USBasp) for - `atmelavr `_ - platform - (`issue #221 `_) - -2.0.2 (2015-05-27) -~~~~~~~~~~~~~~~~~~ - -* Fixed libraries order for "Library Dependency Finder" under Linux OS - -2.0.1 (2015-05-27) -~~~~~~~~~~~~~~~~~~ - -* Handle new environment variable - `PLATFORMIO_BUILD_FLAGS `_ -* Pass to API requests information about Continuous Integration system. This - information will be used by PlatformIO-API. -* Use ``include`` directories from toolchain when initialising project for IDE - (`issue #210 `_) -* Added support for new WildFire boards from - `Wicked Device `_ to - `atmelavr `__ - platform -* Updated `Arduino Framework `__ to - 1.6.4 version (`issue #212 `_) -* Handle Atmel AVR Symbols when initialising project for IDE - (`issue #216 `_) -* Fixed bug with converting ``*.ino`` to ``*.cpp`` -* Fixed failing with ``platformio init --ide eclipse`` without boards - (`issue #217 `_) - -2.0.0 (2015-05-22) -~~~~~~~~~~~~~~~~~~ - -*Made in* `Paradise `_ - -* PlatformIO as `Continuous Integration `_ - (CI) tool for embedded projects - (`issue #108 `_) -* Initialise PlatformIO project for the specified IDE - (`issue #151 `_) -* PlatformIO CLI 2.0: "platform" related commands have been - moved to ``platformio platforms`` subcommand - (`issue #158 `_) -* Created `PlatformIO gitter.im `_ room - (`issue #174 `_) -* Global ``-f, --force`` option which will force to accept any - confirmation prompts - (`issue #152 `_) -* Run project with `platformio run --project-dir `_ option without changing the current working - directory - (`issue #192 `_) -* Control verbosity of `platformio run `_ command via ``-v/--verbose`` option -* Add library dependencies for build environment using - `lib_install `_ - option in ``platformio.ini`` - (`issue #134 `_) -* Specify libraries which are compatible with build environment using - `lib_use `_ - option in ``platformio.ini`` - (`issue #148 `_) -* Add more boards to PlatformIO project with - `platformio init --board `__ - command - (`issue #167 `_) -* Choose which library to update - (`issue #168 `_) -* Specify `platformio init --env-prefix `__ when initialise/update project - (`issue #182 `_) -* Added new Armstrap boards - (`issue #204 `_) -* Updated SDK for `espressif `__ - development platform to v1.1 - (`issue #179 `_) -* Disabled automatic updates by default for platforms, packages and libraries - (`issue #171 `_) -* Fixed bug with creating copies of source files - (`issue #177 `_) +See `PlatformIO Core 2.0 history `__. PlatformIO Core 1 ----------------- -1.5.0 (2015-05-15) -~~~~~~~~~~~~~~~~~~ - -* Added support of `Framework mbed `_ - for Teensy 3.1 - (`issue #183 `_) -* Added GDB as alternative uploader to `ststm32 `__ platform - (`issue #175 `_) -* Added `examples `__ - with preconfigured IDE projects - (`issue #154 `_) -* Fixed firmware uploading under Linux OS for Arduino Leonardo board - (`issue #178 `_) -* Fixed invalid "mbed" firmware for Nucleo F411RE - (`issue #185 `_) -* Fixed parsing of includes for PlatformIO Library Dependency Finder - (`issue #189 `_) -* Fixed handling symbolic links within source code directory - (`issue #190 `_) -* Fixed cancelling any previous definition of name, either built in or provided - with a ``-D`` option - (`issue #191 `_) - -1.4.0 (2015-04-11) -~~~~~~~~~~~~~~~~~~ - -* Added `espressif `_ - development platform with ESP01 board -* Integrated PlatformIO with AppVeyor Windows based Continuous Integration system - (`issue #149 `_) -* Added support for Teensy LC board to - `teensy `__ - platform -* Added support for new Arduino based boards by *SparkFun, BQ, LightUp, - LowPowerLab, Quirkbot, RedBearLab, TinyCircuits* to - `atmelavr `__ - platform -* Upgraded `Arduino Framework `__ to - 1.6.3 version (`issue #156 `_) -* Upgraded `Energia Framework `__ to - 0101E0015 version (`issue #146 `_) -* Upgraded `Arduino Framework with Teensy Core `_ - to 1.22 version - (`issue #162 `_, - `issue #170 `_) -* Fixed exceptions with PlatformIO auto-updates when Internet connection isn't - active - - -1.3.0 (2015-03-27) -~~~~~~~~~~~~~~~~~~ - -* Moved PlatformIO source code and repositories from `Ivan Kravets `_ - account to `PlatformIO Organisation `_ - (`issue #138 `_) -* Added support for new Arduino based boards by *SparkFun, RepRap, Sanguino* to - `atmelavr `__ - platform - (`issue #127 `_, - `issue #131 `_) -* Added integration instructions for `Visual Studio `_ - and `Sublime Text `_ IDEs -* Improved handling of multi-file ``*.ino/pde`` sketches - (`issue #130 `_) -* Fixed wrong insertion of function prototypes converting ``*.ino/pde`` - (`issue #137 `_, - `issue #140 `_) - - - -1.2.0 (2015-03-20) -~~~~~~~~~~~~~~~~~~ - -* Added full support of `mbed `__ - framework including libraries: *RTOS, Ethernet, DSP, FAT, USB*. -* Added `freescalekinetis `_ - development platform with Freescale Kinetis Freedom boards -* Added `nordicnrf51 `_ - development platform with supported boards from *JKSoft, Nordic, RedBearLab, - Switch Science* -* Added `nxplpc `_ - development platform with supported boards from *CQ Publishing, Embedded - Artists, NGX Technologies, NXP, Outrageous Circuits, SeeedStudio, - Solder Splash Labs, Switch Science, u-blox* -* Added support for *ST Nucleo* boards to - `ststm32 `__ - development platform -* Created new `Frameworks `__ - page in documentation and added to `PlatformIO Web Site `_ - (`issue #115 `_) -* Introduced online `Embedded Boards Explorer `_ -* Automatically append define ``-DPLATFORMIO=%version%`` to - builder (`issue #105 `_) -* Renamed ``stm32`` development platform to - `ststm32 `__ -* Renamed ``opencm3`` framework to - `libopencm3 `__ -* Fixed uploading for `atmelsam `__ - development platform -* Fixed re-arranging the ``*.ino/pde`` files when converting to ``*.cpp`` - (`issue #100 `_) - -1.1.0 (2015-03-05) -~~~~~~~~~~~~~~~~~~ - -* Implemented ``PLATFORMIO_*`` environment variables - (`issue #102 `_) -* Added support for *SainSmart* boards to - `atmelsam `__ - development platform -* Added - `Project Configuration `__ - option named `envs_dir `__ -* Disabled "prompts" automatically for *Continuous Integration* systems - (`issue #103 `_) -* Fixed firmware uploading for - `atmelavr `__ - boards which work within ``usbtiny`` protocol -* Fixed uploading for *Digispark* board (`issue #106 `_) - -1.0.1 (2015-02-27) -~~~~~~~~~~~~~~~~~~ - -**PlatformIO 1.0 - recommended for production** - -* Changed development status from ``beta`` to ``Production/Stable`` -* Added support for *ARM*-based credit-card sized computers: - `Raspberry Pi `_, - `BeagleBone `_ and `CubieBoard `_ -* Added `atmelsam `__ - development platform with supported boards: *Arduino Due and Digistump DigiX* - (`issue #71 `_) -* Added `ststm32 `__ - development platform with supported boards: *Discovery kit for STM32L151/152, - STM32F303xx, STM32F407/417 lines* and `libOpenCM3 Framework `_ - (`issue #73 `_) -* Added `teensy `_ - development platform with supported boards: *Teensy 2.x & 3.x* - (`issue #72 `_) -* Added new *Arduino* boards to - `atmelavr `__ - platform: *Arduino NG, Arduino BT, Arduino Esplora, Arduino Ethernet, - Arduino Robot Control, Arduino Robot Motor and Arduino Yun* -* Added support for *Adafruit* boards to - `atmelavr `__ - platform: *Adafruit Flora and Adafruit Trinkets* - (`issue #65 `_) -* Added support for *Digispark* boards to - `atmelavr `__ - platform: *Digispark USB Development Board and Digispark Pro* - (`issue #47 `_) -* Covered code with tests (`issue #2 `_) -* Refactored *Library Dependency Finder* (issues - `#48 `_, - `#50 `_, - `#55 `_) -* Added `src_dir `__ - option to ``[platformio]`` section of - `platformio.ini `__ - which allows to redefine location to project's source directory - (`issue #83 `_) -* Added ``--json-output`` option to - `platformio boards `__ - and `platformio search `__ - commands which allows to return the output in `JSON `_ format - (`issue #42 `_) -* Allowed to ignore some libs from *Library Dependency Finder* via - `lib_ignore `_ option -* Improved `platformio run `__ - command: asynchronous output for build process, timing and detailed - information about environment configuration - (`issue #74 `_) -* Output compiled size and static memory usage with - `platformio run `__ - command (`issue #59 `_) -* Updated `framework-arduino` AVR & SAM to 1.6 stable version -* Fixed an issue with the libraries that are git repositories - (`issue #49 `_) -* Fixed handling of assembly files - (`issue #58 `_) -* Fixed compiling error if space is in user's folder - (`issue #56 `_) -* Fixed `AttributeError: 'module' object has no attribute 'disable_warnings'` - when a version of `requests` package is less then 2.4.0 -* Fixed bug with invalid process's "return code" when PlatformIO has internal - error (`issue #81 `_) -* Several bug fixes, increased stability and performance improvements +See `PlatformIO Core 1.0 history `__. PlatformIO Core Preview ----------------------- -0.10.2 (2015-01-06) -~~~~~~~~~~~~~~~~~~~ - -* Fixed an issue with ``--json-output`` - (`issue #42 `_) -* Fixed an exception during - `platformio upgrade `__ - under Windows OS (`issue #45 `_) - -0.10.1 (2015-01-02) -~~~~~~~~~~~~~~~~~~~ - -* Added ``--json-output`` option to - `platformio list `__, - `platformio serialports list `__ and - `platformio lib list `__ - commands which allows to return the output in `JSON `_ format - (`issue #42 `_) -* Fixed missing auto-uploading by default after `platformio init `__ - command - -0.10.0 (2015-01-01) -~~~~~~~~~~~~~~~~~~~ - -**Happy New Year!** - -* Implemented `platformio boards `_ - command (`issue #11 `_) -* Added support of *Engduino* boards for - `atmelavr `__ - platform (`issue #38 `_) -* Added ``--board`` option to `platformio init `__ - command which allows to initialise project with the specified embedded boards - (`issue #21 `_) -* Added `example with uploading firmware `_ - via USB programmer (USBasp) for - `atmelavr `_ - *MCUs* (`issue #35 `_) -* Automatic detection of port on `platformio serialports monitor `_ - (`issue #37 `_) -* Allowed auto-installation of platforms when prompts are disabled (`issue #43 `_) -* Fixed urllib3's *SSL* warning under Python <= 2.7.2 (`issue #39 `_) -* Fixed bug with *Arduino USB* boards (`issue #40 `_) - -0.9.2 (2014-12-10) -~~~~~~~~~~~~~~~~~~ - -* Replaced "dark blue" by "cyan" colour for the texts (`issue #33 `_) -* Added new setting ``enable_prompts`` and allowed to disable all *PlatformIO* prompts (useful for cloud compilers) - (`issue #34 `_) -* Fixed compilation bug on *Windows* with installed *MSVC* (`issue #18 `_) - -0.9.1 (2014-12-05) -~~~~~~~~~~~~~~~~~~ - -* Ask user to install platform (when it hasn't been installed yet) within - `platformio run `__ - and `platformio show `_ commands -* Improved main `documentation `_ -* Fixed "*OSError: [Errno 2] No such file or directory*" within - `platformio run `__ - command when PlatformIO isn't installed properly -* Fixed example for Eclipse IDE with Tiva board - (`issue #32 `_) -* Upgraded Eclipse Project Examples - to latest *Luna* and *PlatformIO* releases - -0.9.0 (2014-12-01) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio settings `_ command -* Improved `platformio init `_ command. - Added new option ``--project-dir`` where you can specify another path to - directory where new project will be initialized (`issue #31 `_) -* Added *Migration Manager* which simplifies process with upgrading to a - major release -* Added *Telemetry Service* which should help us make *PlatformIO* better -* Implemented *PlatformIO AppState Manager* which allow to have multiple - ``.platformio`` states. -* Refactored *Package Manager* -* Download Manager: fixed SHA1 verification within *Cygwin Environment* - (`issue #26 `_) -* Fixed bug with code builder and built-in Arduino libraries - (`issue #28 `_) - -0.8.0 (2014-10-19) -~~~~~~~~~~~~~~~~~~ - -* Avoided trademark issues in `library.json `_ - with the new fields: `frameworks `_, - `platforms `_ - and `dependencies `_ - (`issue #17 `_) -* Switched logic from "Library Name" to "Library Registry ID" for all - `platformio lib `_ - commands (install, uninstall, update and etc.) -* Renamed ``author`` field to `authors `_ - and allowed to setup multiple authors per library in `library.json `_ -* Added option to specify "maintainer" status in `authors `_ field -* New filters/options for `platformio lib search `_ - command: ``--framework`` and ``--platform`` - -0.7.1 (2014-10-06) -~~~~~~~~~~~~~~~~~~ - -* Fixed bug with order for includes in conversation from INO/PDE to CPP -* Automatic detection of port on upload (`issue #15 `_) -* Fixed lib update crashing when no libs are installed (`issue #19 `_) - - -0.7.0 (2014-09-24) -~~~~~~~~~~~~~~~~~~ - -* Implemented new `[platformio] `_ - section for Configuration File with `home_dir `_ - option (`issue #14 `_) -* Implemented *Library Manager* (`issue #6 `_) - -0.6.0 (2014-08-09) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio serialports monitor `_ (`issue #10 `_) -* Fixed an issue ``ImportError: No module named platformio.util`` (`issue #9 `_) -* Fixed bug with auto-conversation from Arduino \*.ino to \*.cpp - -0.5.0 (2014-08-04) -~~~~~~~~~~~~~~~~~~ - -* Improved nested lookups for libraries -* Disabled default warning flag "-Wall" -* Added auto-conversation from \*.ino to valid \*.cpp for Arduino/Energia - frameworks (`issue #7 `_) -* Added `Arduino example `_ - with external library (*Adafruit CC3000*) -* Implemented `platformio upgrade `_ - command and "auto-check" for the latest - version (`issue #8 `_) -* Fixed an issue with "auto-reset" for *Raspduino* board -* Fixed a bug with nested libs building - -0.4.0 (2014-07-31) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio serialports `_ command -* Allowed to put special build flags only for ``src`` files via - `src_build_flags `_ - environment option -* Allowed to override some of settings via system environment variables - such as: ``PLATFORMIO_SRC_BUILD_FLAGS`` and ``PLATFORMIO_ENVS_DIR`` -* Added ``--upload-port`` option for `platformio run `__ command -* Implemented (especially for `SmartAnthill `_) - `platformio run -t uploadlazy `_ - target (no dependencies to framework libs, ELF and etc.) -* Allowed to skip default packages via `platformio install --skip-default-package `_ - option -* Added tools for *Raspberry Pi* platform -* Added support for *Microduino* and *Raspduino* boards in - `atmelavr `_ platform - -0.3.1 (2014-06-21) -~~~~~~~~~~~~~~~~~~ - -* Fixed auto-installer for Windows OS (bug with %PATH% custom installation) - - -0.3.0 (2014-06-21) -~~~~~~~~~~~~~~~~~~ - -* Allowed to pass multiple "SomePlatform" to install/uninstall commands -* Added "IDE Integration" section to README with Eclipse project examples -* Created auto installer script for *PlatformIO* (`issue #3 `_) -* Added "Super-Quick" way to Installation section (README) -* Implemented "build_flags" option for environments (`issue #4 `_) - - -0.2.0 (2014-06-15) -~~~~~~~~~~~~~~~~~~ - -* Resolved `issue #1 "Build referred libraries" `_ -* Renamed project's "libs" directory to "lib" -* Added `arduino-internal-library `_ example -* Changed to beta status - - -0.1.0 (2014-06-13) -~~~~~~~~~~~~~~~~~~ - -* Birth! First alpha release +See `PlatformIO Core Preview history `__. diff --git a/docs b/docs index c536bff8..6162e5d1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c536bff8352fc0a26366d1d9ca73b8e60af8d205 +Subproject commit 6162e5d14a9b64bbc61390e641e38e336822fc10 diff --git a/examples b/examples index f0f4e097..84855946 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f0f4e0971b0f9f7b4d77a47cb436f910bf3b4add +Subproject commit 84855946ea09b5e41ddbbae455f00e897060346d From 1aaa9b6707e3fc76254c6e9c26f1c412a39b3147 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 26 Aug 2020 17:44:01 +0300 Subject: [PATCH 208/223] Update changelog with static analysis section --- HISTORY.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 53a933d3..37845a52 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -70,6 +70,16 @@ PlatformIO Core 5 - Fixed an issue when running multiple test environments (`issue #3523 `_) - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) +* **Static Code Analysis** + + - Updated analysis tools: + + * ``Cppcheck v2.1`` with a new "soundy" analysis option and improved code parser + * ``PVS-Studio v7.08`` with a new file list analysis mode and extended list of diagnostic rules + + - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) + - Fixed an issue with PIO Check when a defect with multiline error message is not reported in verbose mode (`issue #3631 `_) + * **Miscellaneous** - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) From 80c1774a192221bd66cc764dc930e05ed1faf330 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 14:08:26 +0300 Subject: [PATCH 209/223] Docs: PlatformIO Core 5.0: new commands, migration guide, other improvements --- HISTORY.rst | 47 +++++++++++++---------- docs | 2 +- platformio/app.py | 2 +- platformio/clients/account.py | 2 +- platformio/clients/http.py | 2 +- platformio/commands/access.py | 6 +-- platformio/commands/account.py | 10 ++--- platformio/commands/boards.py | 2 +- platformio/commands/check/command.py | 2 +- platformio/commands/ci.py | 2 +- platformio/commands/debug/command.py | 2 +- platformio/commands/device/command.py | 2 +- platformio/commands/home/command.py | 2 +- platformio/commands/lib/command.py | 2 +- platformio/commands/org.py | 33 ++++++++-------- platformio/commands/package.py | 2 +- platformio/commands/platform.py | 2 +- platformio/commands/project.py | 6 +-- platformio/commands/remote/client/base.py | 2 +- platformio/commands/remote/command.py | 7 ++-- platformio/commands/run/command.py | 2 +- platformio/commands/settings.py | 2 +- platformio/commands/team.py | 10 +++-- platformio/commands/test/command.py | 2 +- platformio/package/manager/core.py | 2 +- platformio/project/options.py | 6 +-- 26 files changed, 86 insertions(+), 75 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 37845a52..2d37cb6b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,45 +8,51 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +- `Migration guide from 4.x to 5.0 `__ + 5.0.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ - * Integration with the new **PlatformIO Trusted Registry** - Enterprise-grade package storage with high availability (multi replicas) - Secure, fast, and reliable global content delivery network (CDN) - - Universal support for all embedded packages: + - Universal support for all packages: * Libraries * Development platforms * Toolchains - - Built-in fine-grained access control (role based, teams, organizations) - - Command Line Interface: + - Built-in fine-grained access control (role-based, teams, organizations) + - New CLI commands: - * `pio package publish `__ – publish a personal or organization package - * `pio package unpublish `__ – remove a pushed package from the registry - * Grant package access to the team members or maintainers + * `pio package `__ – manage packages in the registry + * `pio access `__ – manage package access for users, teams, and maintainers -* Integration with the new `Account Management System `__ +* Integration with the new **Account Management System** - - Manage own organizations - - Manage organization teams - - Manage resource access + - `Manage organizations and owners of an organization `__ + - `Manage teams and team memberships `__ * New **Package Management System** - - Integrated PlatformIO Core with the new PlatformIO Trusted Registry + - Integrated PlatformIO Core with the new PlatformIO Registry - Strict dependency declaration using owner name (resolves name conflicts) (`issue #1824 `_) - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) + - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry) -* **PlatformIO Build System** +* **Build System** + + - Upgraded build engine to the `SCons 4.0 - a next-generation software construction tool `__ + + * `Configuration files are Python scripts `__ – use the power of a real programming language to solve build problems + * Built-in reliable and automatic dependency analysis + * Improved support for parallel builds + * Ability to `share built files in a cache `__ to speed up multiple builds - - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ - New `Custom Targets `__ - * Pre/Post processing based on a dependent sources (other target, source file, etc.) + * Pre/Post processing based on dependent sources (another target, source file, etc.) * Command launcher with own arguments * Launch command with custom options declared in `"platformio.ini" `__ * Python callback as a target (use the power of Python interpreter and PlatformIO Build API) @@ -55,12 +61,12 @@ PlatformIO Core 5 - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) - - Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) + - Fixed an issue with the ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) * **Project Management** - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required - - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment + - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using the existing environment - Dump build system data intended for IDE extensions/plugins using a new `pio project data `__ command - Do not generate ".travis.yml" for a new project, let the user have a choice @@ -75,17 +81,16 @@ PlatformIO Core 5 - Updated analysis tools: * ``Cppcheck v2.1`` with a new "soundy" analysis option and improved code parser - * ``PVS-Studio v7.08`` with a new file list analysis mode and extended list of diagnostic rules + * ``PVS-Studio v7.08`` with a new file list analysis mode and an extended list of diagnostic rules - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) - - Fixed an issue with PIO Check when a defect with multiline error message is not reported in verbose mode (`issue #3631 `_) + - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 `_) * **Miscellaneous** - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) - - Do not escape compiler arguments in VSCode template on Windows - + - Do not escape compiler arguments in VSCode template on Windows. .. _release_notes_4: diff --git a/docs b/docs index 6162e5d1..7def3c50 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6162e5d14a9b64bbc61390e641e38e336822fc10 +Subproject commit 7def3c5008c99aae8f24984556e2a057195224e9 diff --git a/platformio/app.py b/platformio/app.py index 59900500..0196fac4 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -68,7 +68,7 @@ DEFAULT_SETTINGS = { "value": False, }, "projects_dir": { - "description": "Default location for PlatformIO projects (PIO Home)", + "description": "Default location for PlatformIO projects (PlatformIO Home)", "value": get_default_projects_dir(), "validator": projects_dir_validate, }, diff --git a/platformio/clients/account.py b/platformio/clients/account.py index e2abde17..1c4b6755 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -27,7 +27,7 @@ class AccountError(PlatformioException): class AccountNotAuthorized(AccountError): - MESSAGE = "You are not authorized! Please log in to PIO Account." + MESSAGE = "You are not authorized! Please log in to PlatformIO Account." class AccountAlreadyAuthorized(AccountError): diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 8e732958..4d59bcaa 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -45,7 +45,7 @@ class InternetIsOffline(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" "PlatformIO needs the Internet connection to" - " download dependent packages or to work with PIO Account." + " download dependent packages or to work with PlatformIO Account." ) diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 6a59be7a..8b65ba34 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -33,7 +33,7 @@ def validate_client(value): return value -@click.group("access", short_help="Manage Resource Access") +@click.group("access", short_help="Manage resource access") def cli(): pass @@ -75,7 +75,7 @@ def access_private(urn, urn_type): @click.argument("level", type=click.Choice(["admin", "maintainer", "guest"])) @click.argument( "client", - metavar="[ORGNAME:TEAMNAME|USERNAME]", + metavar="[|]", callback=lambda _, __, value: validate_client(value), ) @click.argument( @@ -108,7 +108,7 @@ def access_revoke(client, urn, urn_type): ) -@cli.command("list", short_help="List resources") +@cli.command("list", short_help="List published resources") @click.argument("owner", required=False) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") @click.option("--json-output", is_flag=True) diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 3a1492ec..88aab68b 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -24,7 +24,7 @@ from tabulate import tabulate from platformio.clients.account import AccountClient, AccountNotAuthorized -@click.group("account", short_help="Manage PIO Account") +@click.group("account", short_help="Manage PlatformIO account") def cli(): pass @@ -60,7 +60,7 @@ def validate_password(value): return value -@cli.command("register", short_help="Create new PIO Account") +@cli.command("register", short_help="Create new PlatformIO Account") @click.option( "-u", "--username", @@ -90,7 +90,7 @@ def account_register(username, email, password, firstname, lastname): ) -@cli.command("login", short_help="Log in to PIO Account") +@cli.command("login", short_help="Log in to PlatformIO Account") @click.option("-u", "--username", prompt="Username or email") @click.option("-p", "--password", prompt=True, hide_input=True) def account_login(username, password): @@ -99,7 +99,7 @@ def account_login(username, password): return click.secho("Successfully logged in!", fg="green") -@cli.command("logout", short_help="Log out of PIO Account") +@cli.command("logout", short_help="Log out of PlatformIO Account") def account_logout(): client = AccountClient() client.logout() @@ -195,7 +195,7 @@ def account_destroy(): return click.secho("User account has been destroyed.", fg="green",) -@cli.command("show", short_help="PIO Account information") +@cli.command("show", short_help="PlatformIO Account information") @click.option("--offline", is_flag=True) @click.option("--json-output", is_flag=True) def account_show(offline, json_output): diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 962ab504..4170b32f 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -22,7 +22,7 @@ from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager -@click.command("boards", short_help="Embedded Board Explorer") +@click.command("boards", short_help="Embedded board explorer") @click.argument("query", required=False) @click.option("--installed", is_flag=True) @click.option("--json-output", is_flag=True) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index a5c4e1e7..8f9a6dca 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -31,7 +31,7 @@ from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, get_project_dir -@click.command("check", short_help="Run a static analysis tool on code") +@click.command("check", short_help="Static code analysis") @click.option("-e", "--environment", multiple=True) @click.option( "-d", diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index f68b2bb7..e72ddf76 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -44,7 +44,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument raise click.BadParameter("Found invalid path: %s" % invalid_path) -@click.command("ci", short_help="Continuous Integration") +@click.command("ci", short_help="Continuous integration") @click.argument("src", nargs=-1, callback=validate_path) @click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY") @click.option("--exclude", multiple=True) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 98115cbf..fc83405c 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -33,7 +33,7 @@ from platformio.project.helpers import is_platformio_project, load_project_ide_d @click.command( "debug", context_settings=dict(ignore_unknown_options=True), - short_help="PIO Unified Debugger", + short_help="Unified debugger", ) @click.option( "-d", diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index 463116f9..a66cb996 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -26,7 +26,7 @@ from platformio.platform.factory import PlatformFactory from platformio.project.exception import NotPlatformIOProjectError -@click.group(short_help="Monitor device or list existing") +@click.group(short_help="Device manager & serial/socket monitor") def cli(): pass diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index dd733bb6..6cb26ed9 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -25,7 +25,7 @@ from platformio.compat import WINDOWS from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite -@click.command("home", short_help="PIO Home") +@click.command("home", short_help="UI to manage PlatformIO") @click.option("--port", type=int, default=8008, help="HTTP port, default=8008") @click.option( "--host", diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 96d39814..d871c917 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -50,7 +50,7 @@ def get_project_global_lib_dir(): return ProjectConfig.get_instance().get_optional_dir("globallib") -@click.group(short_help="Library Manager") +@click.group(short_help="Library manager") @click.option( "-d", "--storage-dir", diff --git a/platformio/commands/org.py b/platformio/commands/org.py index a7e0f1e9..ac13d13f 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -23,7 +23,7 @@ from platformio.clients.account import AccountClient from platformio.commands.account import validate_email, validate_username -@click.group("org", short_help="Manage Organizations") +@click.group("org", short_help="Manage organizations") def cli(): pass @@ -44,11 +44,11 @@ def org_create(orgname, email, displayname): client = AccountClient() client.create_org(orgname, email, displayname) return click.secho( - "The organization %s has been successfully created." % orgname, fg="green", + "The organization `%s` has been successfully created." % orgname, fg="green", ) -@cli.command("list", short_help="List organizations") +@cli.command("list", short_help="List organizations and their members") @click.option("--json-output", is_flag=True) def org_list(json_output): client = AccountClient() @@ -56,7 +56,7 @@ def org_list(json_output): if json_output: return click.echo(json.dumps(orgs)) if not orgs: - return click.echo("You do not have any organizations") + return click.echo("You do not have any organization") for org in orgs: click.echo() click.secho(org.get("orgname"), fg="cyan") @@ -77,15 +77,17 @@ def org_list(json_output): @cli.command("update", short_help="Update organization") -@click.argument("orgname") +@click.argument("cur_orgname") @click.option( - "--new-orgname", callback=lambda _, __, value: validate_orgname(value), + "--orgname", + callback=lambda _, __, value: validate_orgname(value), + help="A new orgname", ) @click.option("--email") -@click.option("--displayname",) -def org_update(orgname, **kwargs): +@click.option("--displayname") +def org_update(cur_orgname, **kwargs): client = AccountClient() - org = client.get_org(orgname) + org = client.get_org(cur_orgname) del org["owners"] new_org = org.copy() if not any(kwargs.values()): @@ -101,9 +103,10 @@ def org_update(orgname, **kwargs): new_org.update( {key.replace("new_", ""): value for key, value in kwargs.items() if value} ) - client.update_org(orgname, new_org) + client.update_org(cur_orgname, new_org) return click.secho( - "The organization %s has been successfully updated." % orgname, fg="green", + "The organization `%s` has been successfully updated." % cur_orgname, + fg="green", ) @@ -112,13 +115,13 @@ def org_update(orgname, **kwargs): def account_destroy(orgname): client = AccountClient() click.confirm( - "Are you sure you want to delete the %s organization account?\n" + "Are you sure you want to delete the `%s` organization account?\n" "Warning! All linked data will be permanently removed and can not be restored." % orgname, abort=True, ) client.destroy_org(orgname) - return click.secho("Organization %s has been destroyed." % orgname, fg="green",) + return click.secho("Organization `%s` has been destroyed." % orgname, fg="green",) @cli.command("add", short_help="Add a new owner to organization") @@ -128,7 +131,7 @@ def org_add_owner(orgname, username): client = AccountClient() client.add_org_owner(orgname, username) return click.secho( - "The new owner %s has been successfully added to the %s organization." + "The new owner `%s` has been successfully added to the `%s` organization." % (username, orgname), fg="green", ) @@ -141,7 +144,7 @@ def org_remove_owner(orgname, username): client = AccountClient() client.remove_org_owner(orgname, username) return click.secho( - "The %s owner has been successfully removed from the %s organization." + "The `%s` owner has been successfully removed from the `%s` organization." % (username, orgname), fg="green", ) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 6ec78d38..88f6c0d3 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -32,7 +32,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument return value -@click.group("package", short_help="Package Manager") +@click.group("package", short_help="Package manager") def cli(): pass diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 7725be39..588e7ccc 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -26,7 +26,7 @@ from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory -@click.group(short_help="Platform Manager") +@click.group(short_help="Platform manager") def cli(): pass diff --git a/platformio/commands/project.py b/platformio/commands/project.py index bd37175a..70660fa2 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -30,7 +30,7 @@ from platformio.project.exception import NotPlatformIOProjectError from platformio.project.helpers import is_platformio_project, load_project_ide_data -@click.group(short_help="Project Manager") +@click.group(short_help="Project manager") def cli(): pass @@ -333,7 +333,7 @@ def init_test_readme(test_dir): with open(os.path.join(test_dir, "README"), "w") as fp: fp.write( """ -This directory is intended for PIO Unit Testing and project tests. +This directory is intended for PlatformIO Unit Testing and project tests. Unit Testing is a software testing method by which individual units of source code, sets of one or more MCU program modules together with associated @@ -341,7 +341,7 @@ control data, usage procedures, and operating procedures, are tested to determine whether they are fit for use. Unit testing finds problems early in the development cycle. -More information about PIO Unit Testing: +More information about PlatformIO Unit Testing: - https://docs.platformio.org/page/plus/unit-testing.html """, ) diff --git a/platformio/commands/remote/client/base.py b/platformio/commands/remote/client/base.py index 806d7bda..7ca7be3b 100644 --- a/platformio/commands/remote/client/base.py +++ b/platformio/commands/remote/client/base.py @@ -72,7 +72,7 @@ class RemoteClientBase( # pylint: disable=too-many-instance-attributes def connect(self): self.log.info("Name: {name}", name=self.name) - self.log.info("Connecting to PIO Remote Cloud") + self.log.info("Connecting to PlatformIO Remote Development Cloud") # pylint: disable=protected-access proto, options = endpoints._parse(__pioremote_endpoint__) diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index 66c10690..cafbbd1f 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -33,14 +33,15 @@ from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError -@click.group("remote", short_help="PIO Remote") +@click.group("remote", short_help="Remote development") @click.option("-a", "--agent", multiple=True) @click.pass_context def cli(ctx, agent): if PY2: raise exception.UserSideException( - "PIO Remote requires Python 3.5 or above. \nPlease install the latest " - "Python 3 and reinstall PlatformIO Core using installation script:\n" + "PlatformIO Remote Development requires Python 3.5 or above. \n" + "Please install the latest Python 3 and reinstall PlatformIO Core using " + "installation script:\n" "https://docs.platformio.org/page/core/installation.html" ) ctx.obj = agent diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index c2142723..00e129af 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -36,7 +36,7 @@ except NotImplementedError: DEFAULT_JOB_NUMS = 1 -@click.command("run", short_help="Process project environments") +@click.command("run", short_help="Run project targets (build, upload, clean, etc.)") @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index 7f03f81b..695d9020 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -27,7 +27,7 @@ def format_value(raw): return str(raw) -@click.group(short_help="Manage PlatformIO settings") +@click.group(short_help="Manage system settings") def cli(): pass diff --git a/platformio/commands/team.py b/platformio/commands/team.py index 5461cabd..7c1e8638 100644 --- a/platformio/commands/team.py +++ b/platformio/commands/team.py @@ -50,7 +50,7 @@ def validate_teamname(value): return value -@click.group("team", short_help="Manage Teams") +@click.group("team", short_help="Manage organization teams") def cli(): pass @@ -119,7 +119,9 @@ def team_list(orgname, json_output): callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.option( - "--name", callback=lambda _, __, value: validate_teamname(value), + "--name", + callback=lambda _, __, value: validate_teamname(value), + help="A new team name", ) @click.option("--description",) def team_update(orgname_teamname, **kwargs): @@ -189,8 +191,8 @@ def team_add_member(orgname_teamname, username): metavar="ORGNAME:TEAMNAME", callback=lambda _, __, value: validate_orgname_teamname(value), ) -@click.argument("username",) -def org_remove_owner(orgname_teamname, username): +@click.argument("username") +def team_remove_owner(orgname_teamname, username): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() client.remove_team_member(orgname, teamname, username) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index b57b1d59..13104bb2 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -28,7 +28,7 @@ from platformio.commands.test.native import NativeTestProcessor from platformio.project.config import ProjectConfig -@click.command("test", short_help="Unit Testing") +@click.command("test", short_help="Unit testing") @click.option("--environment", "-e", multiple=True, metavar="") @click.option( "--filter", diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 2d01b155..a11217e9 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -26,7 +26,7 @@ from platformio.proc import get_pythonexe_path def get_core_package_dir(name): if name not in __core_packages__: - raise exception.PlatformioException("Please upgrade PIO Core") + raise exception.PlatformioException("Please upgrade PlatformIO Core") pm = ToolPackageManager() spec = PackageSpec( owner="platformio", name=name, requirements=__core_packages__[name] diff --git a/platformio/project/options.py b/platformio/project/options.py index 3f0cf76c..b5eaf337 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -245,7 +245,7 @@ ProjectOptions = OrderedDict( group="directory", name="test_dir", description=( - "A location where PIO Unit Testing engine looks for " + "A location where PlatformIO Unit Testing engine looks for " "test source files" ), sysenvvar="PLATFORMIO_TEST_DIR", @@ -262,8 +262,8 @@ ProjectOptions = OrderedDict( group="directory", name="shared_dir", description=( - "A location which PIO Remote uses to synchronize extra files " - "between remote machines" + "A location which PlatformIO Remote Development service uses to " + "synchronize extra files between remote machines" ), sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("$PROJECT_DIR", "shared"), From cdbb8379489675f2bb103972b8ecca9021ec060b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 18:45:52 +0300 Subject: [PATCH 210/223] Minor fixes --- HISTORY.rst | 2 +- docs | 2 +- platformio/telemetry.py | 5 ++++- platformio/util.py | 1 - 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2d37cb6b..78a67f26 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,7 +31,7 @@ PlatformIO Core 5 * Integration with the new **Account Management System** - - `Manage organizations and owners of an organization `__ + - `Manage organizations `__ - `Manage teams and team memberships `__ * New **Package Management System** diff --git a/docs b/docs index 7def3c50..1a4a4bf1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7def3c5008c99aae8f24984556e2a057195224e9 +Subproject commit 1a4a4bf127ec0651f7ca299e031379ea96734139 diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 4c5a6706..68392f98 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -145,13 +145,16 @@ class MeasurementProtocol(TelemetryBase): cmd_path = args[:1] if args[0] in ( + "access", "account", "device", - "platform", + "org", "package", + "platform", "project", "settings", "system", + "team" ): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: diff --git a/platformio/util.py b/platformio/util.py index f950a364..6da4708b 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -29,7 +29,6 @@ import click from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS from platformio.fs import cd, load_json # pylint: disable=unused-import -from platformio.package.version import pepver_to_semver # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import From 9f2c134e44903ffc8297f13ce49d7076bd9ae7af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 21:24:48 +0300 Subject: [PATCH 211/223] Do not detach a new package even if it comes from external source --- platformio/package/manager/_install.py | 27 +++++++++++++------------- platformio/telemetry.py | 2 +- tests/commands/test_lib_complex.py | 17 ++++++++-------- tests/package/test_manager.py | 6 ++++++ 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 1bafcf8c..1a83d65b 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -180,20 +180,19 @@ class PackageManagerInstallMixin(object): dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) ) - elif dst_pkg.metadata and dst_pkg.metadata.spec.external: - if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: - action = "detach-existing" - elif tmp_pkg.metadata.spec.external: - action = "detach-new" - elif dst_pkg.metadata and ( - dst_pkg.metadata.version != tmp_pkg.metadata.version - or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner - ): - action = ( - "detach-existing" - if tmp_pkg.metadata.version > dst_pkg.metadata.version - else "detach-new" - ) + elif dst_pkg.metadata: + if dst_pkg.metadata.spec.external: + if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: + action = "detach-existing" + elif ( + dst_pkg.metadata.version != tmp_pkg.metadata.version + or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner + ): + action = ( + "detach-existing" + if tmp_pkg.metadata.version > dst_pkg.metadata.version + else "detach-new" + ) def _cleanup_dir(path): if os.path.isdir(path): diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 68392f98..3fbcb74f 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -154,7 +154,7 @@ class MeasurementProtocol(TelemetryBase): "project", "settings", "system", - "team" + "team", ): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index dfb853f4..442edee3 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -98,9 +98,10 @@ def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "ArduinoJson", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "SomeLib", "OneWire", - "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8", + "ESP32WebServer", ] assert set(items1) >= set(items2) @@ -122,11 +123,11 @@ def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_c validate_cliresult(result) items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ - "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f", + "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377", - "PubSubClient@src-98ec699a461a31615982e5adaaefadda", + "platformio-libmirror", + "PubSubClient", ] assert set(items1) >= set(items2) @@ -293,13 +294,13 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "AsyncMqttClient", "AsyncTCP", - "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8", + "ESP32WebServer", "ESPAsyncTCP", "NeoPixelBus", - "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f", + "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", - "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377", - "PubSubClient@src-98ec699a461a31615982e5adaaefadda", + "platformio-libmirror", + "PubSubClient", "SomeLib", ] assert set(items1) == set(items2) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index f5939f15..299b0020 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -186,6 +186,12 @@ version = 5.2.7 pkg = lm.install_from_url("file://%s" % src_dir, spec) assert str(pkg.metadata.version) == "5.2.7" + # check package folder names + lm.memcache_reset() + assert ["local-lib-dir", "manifest-lib-name", "wifilib"] == [ + os.path.basename(pkg.path) for pkg in lm.get_installed() + ] + def test_install_from_registry(isolated_pio_core, tmpdir_factory): # Libraries From 5dee0a31e6f605e0ad1362256c9cf1f8d4ba5c5f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 21:40:17 +0300 Subject: [PATCH 212/223] Do not test for package owner if resource is external --- platformio/package/manager/base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index cf359d13..6cb609a3 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -244,10 +244,6 @@ class BasePackageManager( # pylint: disable=too-many-public-methods if spec.id and spec.id != pkg.metadata.spec.id: return False - # "owner" mismatch - if spec.owner and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner): - return False - # external "URL" mismatch if spec.external: # local folder mismatch @@ -259,6 +255,12 @@ class BasePackageManager( # pylint: disable=too-many-public-methods if spec.url != pkg.metadata.spec.url: return False + # "owner" mismatch + elif spec.owner and not ci_strings_are_equal( + spec.owner, pkg.metadata.spec.owner + ): + return False + # "name" mismatch elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): return False From be487019f561b545cd36f6e2a5c3c5e6c130d6a3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 21:54:47 +0300 Subject: [PATCH 213/223] Fix a broken handling multi-configuration project // Resolve #3615 --- platformio/commands/project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 70660fa2..f861b5b1 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -180,7 +180,10 @@ def project_init( ) if ide: - config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + with fs.cd(project_dir): + config = ProjectConfig.get_instance( + os.path.join(project_dir, "platformio.ini") + ) config.validate() pg = ProjectGenerator( config, environment or get_best_envname(config, board), ide From 7a49a741355e84b1e2685b204b2959c01c27d86a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 21:55:55 +0300 Subject: [PATCH 214/223] Bump version to 5.0.0b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 82133bd4..f5d915ff 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "0b2") +VERSION = (5, 0, "0b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 2edd7ae649c6f234db7ecf8306cb187e5b6188eb Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 31 Aug 2020 15:40:25 +0300 Subject: [PATCH 215/223] Update PVS-Studio to the latest v7.09 --- HISTORY.rst | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 78a67f26..6148e40f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -81,7 +81,7 @@ PlatformIO Core 5 - Updated analysis tools: * ``Cppcheck v2.1`` with a new "soundy" analysis option and improved code parser - * ``PVS-Studio v7.08`` with a new file list analysis mode and an extended list of diagnostic rules + * ``PVS-Studio v7.09`` with a new file list analysis mode and an extended list of analysis diagnostics - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 `_) diff --git a/platformio/__init__.py b/platformio/__init__.py index f5d915ff..e940a364 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -53,7 +53,7 @@ __core_packages__ = { "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", "tool-cppcheck": "~1.210.0", "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.8.0", + "tool-pvs-studio": "~7.9.0", } __check_internet_hosts__ = [ From 5cc21511ad2a9a8de560f0bb8e32bf45b3f2fd8c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 16:07:16 +0300 Subject: [PATCH 216/223] Show owner name for packages --- platformio/commands/lib/command.py | 7 ++--- platformio/commands/lib/helpers.py | 5 +++- platformio/commands/platform.py | 8 +++--- platformio/package/manager/_legacy.py | 2 ++ platformio/project/config.py | 6 +++++ tests/commands/test_lib.py | 37 ++++++++++++++++++++++----- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index d871c917..23a5b46e 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -450,12 +450,13 @@ def lib_show(library, json_output): if json_output: return click.echo(dump_json_to_unicode(lib)) - click.secho(lib["name"], fg="cyan") - click.echo("=" * len(lib["name"])) - click.secho("#ID: %d" % lib["id"], bold=True) + title = "{ownername}/{name}".format(**lib) + click.secho(title, fg="cyan") + click.echo("=" * len(title)) click.echo(lib["description"]) click.echo() + click.secho("ID: %d" % lib["id"]) click.echo( "Version: %s, released %s" % ( diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index a5b0e260..7a156e0f 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -87,5 +87,8 @@ def save_project_libdeps(project_dir, specs, environments=None, action="add"): pass if action == "add": lib_deps.extend(spec.as_dependency() for spec in specs) - config.set("env:" + env, "lib_deps", lib_deps) + if lib_deps: + config.set("env:" + env, "lib_deps", lib_deps) + elif config.has_option("env:" + env, "lib_deps"): + config.remove_option("env:" + env, "lib_deps") config.save() diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 588e7ccc..054a7a12 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -140,6 +140,7 @@ def _get_registry_platform_data( # pylint: disable=unused-argument return None data = dict( + ownername=_data.get("ownername"), name=_data["name"], title=_data["title"], description=_data["description"], @@ -242,12 +243,11 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if json_output: return click.echo(dump_json_to_unicode(data)) + dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"] click.echo( - "{name} ~ {title}".format( - name=click.style(data["name"], fg="cyan"), title=data["title"] - ) + "{dep} ~ {title}".format(dep=click.style(dep, fg="cyan"), title=data["title"]) ) - click.echo("=" * (3 + len(data["name"] + data["title"]))) + click.echo("=" * (3 + len(dep + data["title"]))) click.echo(data["description"]) click.echo() if "version" in data: diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py index 95f628d0..5c35ebeb 100644 --- a/platformio/package/manager/_legacy.py +++ b/platformio/package/manager/_legacy.py @@ -53,6 +53,8 @@ class PackageManagerLegacyMixin(object): if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external: manifest["__src_url"] = pkg.metadata.spec.url manifest["version"] = str(pkg.metadata.version) + if pkg.metadata and pkg.metadata.spec.owner: + manifest["ownername"] = pkg.metadata.spec.owner return manifest def legacy_get_installed(self): diff --git a/platformio/project/config.py b/platformio/project/config.py index 786f080a..2d841b39 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -358,6 +358,12 @@ class ProjectConfigBase(object): click.secho("Warning! %s" % warning, fg="yellow") return True + def remove_option(self, section, option): + return self._parser.remove_option(section, option) + + def remove_section(self, section): + return self._parser.remove_section(section) + class ProjectConfigDirsMixin(object): def _get_core_dir(self, exists=False): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 332161e1..b077f63b 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -44,16 +44,21 @@ lib_deps = ArduinoJson @ 5.10.1 """ ) - result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "install", "64"]) + result = clirunner.invoke( + cmd_lib, + ["-d", str(project_dir), "install", "64", "knolleary/PubSubClient@~2.7"], + ) validate_cliresult(result) aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert config.get("env:one", "lib_deps") == [ - "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"] + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", ] assert config.get("env:two", "lib_deps") == [ "CustomLib", "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", ] # ensure "build" version without NPM spec @@ -68,6 +73,7 @@ lib_deps = config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert config.get("env:one", "lib_deps") == [ "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], ] @@ -85,23 +91,41 @@ lib_deps = ) validate_cliresult(result) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) - assert len(config.get("env:one", "lib_deps")) == 3 - assert config.get("env:one", "lib_deps")[2] == ( + assert len(config.get("env:one", "lib_deps")) == 4 + assert config.get("env:one", "lib_deps")[3] == ( "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3" ) # test uninstalling + # from all envs result = clirunner.invoke( cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"] ) validate_cliresult(result) + # from "one" env + result = clirunner.invoke( + cmd_lib, + [ + "-d", + str(project_dir), + "-e", + "one", + "uninstall", + "knolleary/PubSubClient@~2.7", + ], + ) + validate_cliresult(result) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert len(config.get("env:one", "lib_deps")) == 2 - assert len(config.get("env:two", "lib_deps")) == 1 + assert len(config.get("env:two", "lib_deps")) == 2 assert config.get("env:one", "lib_deps") == [ "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ] + assert config.get("env:two", "lib_deps") == [ + "CustomLib", + "knolleary/PubSubClient@~2.7", + ] # test list result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) @@ -122,7 +146,8 @@ lib_deps = item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome" ) ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"]) - assert data["two"] == [] + assert len(data["two"]) == 1 + assert data["two"][0]["name"] == "PubSubClient" assert "__pkg_dir" in data["one"][0] assert ( ame_lib["__src_url"] From 44c2b65372804866fa1946a4b6fb784d86c32cb6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 17:31:32 +0300 Subject: [PATCH 217/223] Show ignored project environments only in the verbose mode // Resolve #3641 --- HISTORY.rst | 1 + platformio/commands/run/command.py | 6 ++++-- tests/commands/test_lib.py | 34 ++++++++++++++++++------------ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6148e40f..286890f9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -90,6 +90,7 @@ PlatformIO Core 5 - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) + - Show ignored project environments only in the verbose mode (`issue #3641 `_) - Do not escape compiler arguments in VSCode template on Windows. .. _release_notes_4: diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 00e129af..db4b4121 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -147,7 +147,7 @@ def cli( command_failed = any(r.get("succeeded") is False for r in results) if not is_test_running and (command_failed or not silent) and len(results) > 1: - print_processing_summary(results) + print_processing_summary(results, verbose) if command_failed: raise exception.ReturnErrorCode(1) @@ -220,7 +220,7 @@ def print_processing_footer(result): ) -def print_processing_summary(results): +def print_processing_summary(results, verbose=False): tabular_data = [] succeeded_nums = 0 failed_nums = 0 @@ -232,6 +232,8 @@ def print_processing_summary(results): failed_nums += 1 status_str = click.style("FAILED", fg="red") elif result.get("succeeded") is None: + if not verbose: + continue status_str = "IGNORED" else: succeeded_nums += 1 diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index b077f63b..b2541841 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -51,15 +51,19 @@ lib_deps = validate_cliresult(result) aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) - assert config.get("env:one", "lib_deps") == [ - "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], - "knolleary/PubSubClient@~2.7", - ] - assert config.get("env:two", "lib_deps") == [ - "CustomLib", - "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], - "knolleary/PubSubClient@~2.7", - ] + assert sorted(config.get("env:one", "lib_deps")) == sorted( + [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + ] + ) + assert sorted(config.get("env:two", "lib_deps")) == sorted( + [ + "CustomLib", + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + ] + ) # ensure "build" version without NPM spec result = clirunner.invoke( @@ -71,11 +75,13 @@ lib_deps = PackageType.LIBRARY, "mbed-sam-grove", "LinkedList" ) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) - assert config.get("env:one", "lib_deps") == [ - "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], - "knolleary/PubSubClient@~2.7", - "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], - ] + assert sorted(config.get("env:one", "lib_deps")) == sorted( + [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + ] + ) # check external package via Git repo result = clirunner.invoke( From 6e5198f3739f536935916f700bd09a9f96ad0278 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 18:49:00 +0300 Subject: [PATCH 218/223] Minor improvements --- platformio/builder/main.py | 2 +- platformio/package/exception.py | 2 +- platformio/package/manager/_install.py | 2 +- platformio/package/manager/_uninstall.py | 8 +++++--- tests/commands/test_platform.py | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e73f6869..1547cf99 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -160,7 +160,7 @@ env.LoadPioPlatform() env.SConscriptChdir(0) env.SConsignFile( - join("$BUILD_DIR", ".sconsign%d%d.db" % (sys.version_info[0], sys.version_info[1])) + join("$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])) ) for item in env.GetExtraScripts("pre"): diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 0f34592f..5d63649e 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -55,7 +55,7 @@ class MissingPackageManifestError(ManifestException): class UnknownPackageError(UserSideException): MESSAGE = ( - "Could not find a package with '{0}' requirements for your system '%s'" + "Could not find the package with '{0}' requirements for your system '%s'" % util.get_systype() ) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 1a83d65b..9d82d6fe 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -106,7 +106,7 @@ class PackageManagerInstallMixin(object): if not silent: self.print_message( - "{name} @ {version} has been successfully installed!".format( + "{name} @ {version} has been installed!".format( **pkg.metadata.as_dict() ), fg="green", diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 322eced6..68f7a300 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -37,9 +37,8 @@ class PackageManagerUninstallMixin(object): if not silent: self.print_message( - "Removing %s @ %s: \t" + "Removing %s @ %s" % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), - nl=False, ) # firstly, remove dependencies @@ -68,7 +67,10 @@ class PackageManagerUninstallMixin(object): self.memcache_reset() if not silent: - click.echo("[%s]" % click.style("OK", fg="green")) + self.print_message( + "{name} @ {version} has been removed!".format(**pkg.metadata.as_dict()), + fg="green", + ) return pkg diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 39afbeb5..cfb7fe31 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -119,7 +119,7 @@ def test_update_check(clirunner, validate_cliresult, isolated_pio_core): def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cli_platform.platform_update) validate_cliresult(result) - assert "Removing atmelavr @ 2.0.0:" in result.output + assert "Removing atmelavr @ 2.0.0" in result.output assert "Platform Manager: Installing platformio/atmelavr @" in result.output assert len(isolated_pio_core.join("packages").listdir()) == 2 From c8ea64edabcc0408ac7a5978f08eaf73fa54afe1 Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Wed, 2 Sep 2020 18:13:20 +0200 Subject: [PATCH 219/223] Fix link to FAQ sections (#3642) * Fix link to FAQ sections Use consistently the same host and url and fix one unmatched anchor. * Update HISTORY.rst Co-authored-by: Ivan Kravets --- platformio/exception.py | 8 ++++---- platformio/maintenance.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/platformio/exception.py b/platformio/exception.py index 8ae549bc..ef1d3bab 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -55,8 +55,8 @@ class InvalidUdevRules(PlatformioException): class MissedUdevRules(InvalidUdevRules): MESSAGE = ( - "Warning! Please install `99-platformio-udev.rules`. \nMode details: " - "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules" + "Warning! Please install `99-platformio-udev.rules`. \nMore details: " + "https://docs.platformio.org/page/faq.html#platformio-udev-rules" ) @@ -64,8 +64,8 @@ class OutdatedUdevRules(InvalidUdevRules): MESSAGE = ( "Warning! Your `{0}` are outdated. Please update or reinstall them." - "\n Mode details: https://docs.platformio.org" - "/en/latest/faq.html#platformio-udev-rules" + "\nMore details: " + "https://docs.platformio.org/page/faq.html#platformio-udev-rules" ) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 1900db49..e038bcc0 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -148,7 +148,7 @@ def after_upgrade(ctx): click.secho("Please remove multiple PIO Cores from a system:", fg="yellow") click.secho( "https://docs.platformio.org/page/faq.html" - "#multiple-pio-cores-in-a-system", + "#multiple-platformio-cores-in-a-system", fg="cyan", ) click.secho("*" * terminal_width, fg="yellow") From fe4112a2a3c4b07c0cd0506d43c8c7dc0ff7d538 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 20:36:56 +0300 Subject: [PATCH 220/223] Follow SemVer complaint version constraints when checking library updates // Resolve #1281 --- HISTORY.rst | 3 ++- platformio/commands/lib/command.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 286890f9..fe9e137f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,8 +37,9 @@ PlatformIO Core 5 * New **Package Management System** - Integrated PlatformIO Core with the new PlatformIO Registry - - Strict dependency declaration using owner name (resolves name conflicts) (`issue #1824 `_) + - Support for owner-based dependency declaration (resolves name conflicts) (`issue #1824 `_) - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) + - Follow SemVer complaint version constraints when checking library updates `issue #1281 `_) - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry) * **Build System** diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 23a5b46e..543e439c 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -254,8 +254,9 @@ def lib_update( # pylint: disable=too-many-arguments for storage_dir in storage_dirs: if not json_output: print_storage_header(storage_dirs, storage_dir) + lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, []) lm = LibraryPackageManager(storage_dir) - _libraries = libraries or lm.get_installed() + _libraries = libraries or lib_deps or lm.get_installed() if only_check and json_output: result = [] @@ -286,9 +287,13 @@ def lib_update( # pylint: disable=too-many-arguments to_spec = ( None if isinstance(library, PackageItem) else PackageSpec(library) ) - lm.update( - library, to_spec=to_spec, only_check=only_check, silent=silent - ) + try: + lm.update( + library, to_spec=to_spec, only_check=only_check, silent=silent + ) + except UnknownPackageError as e: + if library not in lib_deps: + raise e if json_output: return click.echo( From 083edc4c76cc0e54689258743927e8e6a3d8d4cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 20:52:11 +0300 Subject: [PATCH 221/223] Refactor to os.path --- platformio/builder/tools/piolib.py | 115 +++++++++++++++-------------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 35d1462e..f6b9824b 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -23,7 +23,6 @@ import io import os import re import sys -from os.path import basename, commonprefix, isdir, isfile, join, realpath, sep import click import SCons.Scanner # pylint: disable=import-error @@ -49,7 +48,7 @@ class LibBuilderFactory(object): @staticmethod def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" - if isfile(join(path, "library.json")): + if os.path.isfile(os.path.join(path, "library.json")): clsname = "PlatformIOLibBuilder" else: used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) @@ -66,12 +65,12 @@ class LibBuilderFactory(object): @staticmethod def get_used_frameworks(env, path): if any( - isfile(join(path, fname)) + os.path.isfile(os.path.join(path, fname)) for fname in ("library.properties", "keywords.txt") ): return ["arduino"] - if isfile(join(path, "module.json")): + if os.path.isfile(os.path.join(path, "module.json")): return ["mbed"] include_re = re.compile( @@ -87,7 +86,7 @@ class LibBuilderFactory(object): fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT ): continue - with io.open(join(root, fname), errors="ignore") as fp: + with io.open(os.path.join(root, fname), errors="ignore") as fp: content = fp.read() if not content: continue @@ -114,7 +113,7 @@ class LibBuilderBase(object): def __init__(self, env, path, manifest=None, verbose=False): self.env = env.Clone() self.envorigin = env.Clone() - self.path = realpath(env.subst(path)) + self.path = os.path.realpath(env.subst(path)) self.verbose = verbose try: @@ -148,11 +147,11 @@ class LibBuilderBase(object): p2 = p2.lower() if p1 == p2: return True - return commonprefix((p1 + sep, p2)) == p1 + sep + return os.path.commonprefix((p1 + os.path.sep, p2)) == p1 + os.path.sep @property def name(self): - return self._manifest.get("name", basename(self.path)) + return self._manifest.get("name", os.path.basename(self.path)) @property def version(self): @@ -173,13 +172,19 @@ class LibBuilderBase(object): @property def include_dir(self): - if not all(isdir(join(self.path, d)) for d in ("include", "src")): + if not all( + os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src") + ): return None - return join(self.path, "include") + return os.path.join(self.path, "include") @property def src_dir(self): - return join(self.path, "src") if isdir(join(self.path, "src")) else self.path + return ( + os.path.join(self.path, "src") + if os.path.isdir(os.path.join(self.path, "src")) + else self.path + ) def get_include_dirs(self): items = [] @@ -192,7 +197,9 @@ class LibBuilderBase(object): @property def build_dir(self): lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] - return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) + return os.path.join( + "$BUILD_DIR", "lib%s" % lib_hash, os.path.basename(self.path) + ) @property def build_flags(self): @@ -271,7 +278,7 @@ class LibBuilderBase(object): if self.extra_script: self.env.SConscriptChdir(1) self.env.SConscript( - realpath(self.extra_script), + os.path.realpath(self.extra_script), exports={"env": self.env, "pio_lib_builder": self}, ) self.env.ProcessUnFlags(self.build_unflags) @@ -297,14 +304,14 @@ class LibBuilderBase(object): def get_search_files(self): items = [ - join(self.src_dir, item) + os.path.join(self.src_dir, item) for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter) ] include_dir = self.include_dir if include_dir: items.extend( [ - join(include_dir, item) + os.path.join(include_dir, item) for item in self.env.MatchSourceFiles(include_dir) ] ) @@ -373,7 +380,7 @@ class LibBuilderBase(object): continue _f_part = _h_path[: _h_path.rindex(".")] for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT: - if not isfile("%s.%s" % (_f_part, ext)): + if not os.path.isfile("%s.%s" % (_f_part, ext)): continue _c_path = self.env.File("%s.%s" % (_f_part, ext)) if _c_path not in result: @@ -467,23 +474,23 @@ class UnknownLibBuilder(LibBuilderBase): class ArduinoLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "library.properties") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "library.properties") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) - if isdir(join(self.path, "src")): + if os.path.isdir(os.path.join(self.path, "src")): return include_dirs - if isdir(join(self.path, "utility")): - include_dirs.append(join(self.path, "utility")) + if os.path.isdir(os.path.join(self.path, "utility")): + include_dirs.append(os.path.join(self.path, "utility")) return include_dirs @property def src_filter(self): - src_dir = join(self.path, "src") - if isdir(src_dir): + src_dir = os.path.join(self.path, "src") + if os.path.isdir(src_dir): # pylint: disable=no-member src_filter = LibBuilderBase.src_filter.fget(self) for root, _, files in os.walk(src_dir, followlinks=True): @@ -495,20 +502,20 @@ class ArduinoLibBuilder(LibBuilderBase): if not found: continue rel_path = root.replace(src_dir, "") - if rel_path.startswith(sep): - rel_path = rel_path[1:] + sep + if rel_path.startswith(os.path.sep): + rel_path = rel_path[1:] + os.path.sep src_filter.append("-<%s*.[aA][sS][mM]>" % rel_path) return src_filter src_filter = [] - is_utility = isdir(join(self.path, "utility")) + is_utility = os.path.isdir(os.path.join(self.path, "utility")) for ext in piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT: # arduino ide ignores files with .asm or .ASM extensions if ext.lower() == "asm": continue src_filter.append("+<*.%s>" % ext) if is_utility: - src_filter.append("+" % (sep, ext)) + src_filter.append("+" % (os.path.sep, ext)) return src_filter @property @@ -541,21 +548,21 @@ class ArduinoLibBuilder(LibBuilderBase): class MbedLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "module.json") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "module.json") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() @property def include_dir(self): - if isdir(join(self.path, "include")): - return join(self.path, "include") + if os.path.isdir(os.path.join(self.path, "include")): + return os.path.join(self.path, "include") return None @property def src_dir(self): - if isdir(join(self.path, "source")): - return join(self.path, "source") + if os.path.isdir(os.path.join(self.path, "source")): + return os.path.join(self.path, "source") return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member def get_include_dirs(self): @@ -565,13 +572,13 @@ class MbedLibBuilder(LibBuilderBase): # library with module.json for p in self._manifest.get("extraIncludes", []): - include_dirs.append(join(self.path, p)) + include_dirs.append(os.path.join(self.path, p)) # old mbed library without manifest, add to CPPPATH all folders if not self._manifest: for root, _, __ in os.walk(self.path): part = root.replace(self.path, "").lower() - if any(s in part for s in ("%s." % sep, "test", "example")): + if any(s in part for s in ("%s." % os.path.sep, "test", "example")): continue if root not in include_dirs: include_dirs.append(root) @@ -587,7 +594,7 @@ class MbedLibBuilder(LibBuilderBase): def _process_mbed_lib_confs(self): mbed_lib_paths = [ - join(root, "mbed_lib.json") + os.path.join(root, "mbed_lib.json") for root, _, files in os.walk(self.path) if "mbed_lib.json" in files ] @@ -596,8 +603,8 @@ class MbedLibBuilder(LibBuilderBase): mbed_config_path = None for p in self.env.get("CPPPATH"): - mbed_config_path = join(self.env.subst(p), "mbed_config.h") - if isfile(mbed_config_path): + mbed_config_path = os.path.join(self.env.subst(p), "mbed_config.h") + if os.path.isfile(mbed_config_path): break mbed_config_path = None if not mbed_config_path: @@ -689,26 +696,26 @@ class MbedLibBuilder(LibBuilderBase): class PlatformIOLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "library.json") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "library.json") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() def _has_arduino_manifest(self): - return isfile(join(self.path, "library.properties")) + return os.path.isfile(os.path.join(self.path, "library.properties")) @property def include_dir(self): if "includeDir" in self._manifest.get("build", {}): with fs.cd(self.path): - return realpath(self._manifest.get("build").get("includeDir")) + return os.path.realpath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): with fs.cd(self.path): - return realpath(self._manifest.get("build").get("srcDir")) + return os.path.realpath(self._manifest.get("build").get("srcDir")) return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member @property @@ -786,10 +793,10 @@ class PlatformIOLibBuilder(LibBuilderBase): if ( "build" not in self._manifest and self._has_arduino_manifest() - and not isdir(join(self.path, "src")) - and isdir(join(self.path, "utility")) + and not os.path.isdir(os.path.join(self.path, "src")) + and os.path.isdir(os.path.join(self.path, "utility")) ): - include_dirs.append(join(self.path, "utility")) + include_dirs.append(os.path.join(self.path, "utility")) for path in self.env.get("CPPPATH", []): if path not in self.envorigin.get("CPPPATH", []): @@ -808,7 +815,7 @@ class ProjectAsLibBuilder(LibBuilderBase): @property def include_dir(self): include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - return include_dir if isdir(include_dir) else None + return include_dir if os.path.isdir(include_dir) else None @property def src_dir(self): @@ -817,7 +824,7 @@ class ProjectAsLibBuilder(LibBuilderBase): def get_include_dirs(self): include_dirs = [] project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - if isdir(project_include_dir): + if os.path.isdir(project_include_dir): include_dirs.append(project_include_dir) for include_dir in LibBuilderBase.get_include_dirs(self): if include_dir not in include_dirs: @@ -831,7 +838,7 @@ class ProjectAsLibBuilder(LibBuilderBase): if "__test" in COMMAND_LINE_TARGETS: items.extend( [ - join("$PROJECT_TEST_DIR", item) + os.path.join("$PROJECT_TEST_DIR", item) for item in self.env.MatchSourceFiles( "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) @@ -884,7 +891,7 @@ class ProjectAsLibBuilder(LibBuilderBase): did_install = False lm = LibraryPackageManager( - self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) + self.env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) ) for spec in not_found_specs: try: @@ -975,12 +982,12 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches found_incompat = False for storage_dir in env.GetLibSourceDirs(): - storage_dir = realpath(storage_dir) - if not isdir(storage_dir): + storage_dir = os.path.realpath(storage_dir) + if not os.path.isdir(storage_dir): continue for item in sorted(os.listdir(storage_dir)): - lib_dir = join(storage_dir, item) - if item == "__cores__" or not isdir(lib_dir): + lib_dir = os.path.join(storage_dir, item) + if item == "__cores__" or not os.path.isdir(lib_dir): continue try: lb = LibBuilderFactory.new(env, lib_dir) From fec4569ada6c753ffe55c7a8b4449042cd0ffc3b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Sep 2020 14:37:24 +0300 Subject: [PATCH 222/223] Docs: Update docs with new owner-based dependency form --- HISTORY.rst | 4 ++-- docs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fe9e137f..d2622833 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -81,8 +81,8 @@ PlatformIO Core 5 - Updated analysis tools: - * ``Cppcheck v2.1`` with a new "soundy" analysis option and improved code parser - * ``PVS-Studio v7.09`` with a new file list analysis mode and an extended list of analysis diagnostics + * `Cppcheck `__ v2.1 with a new "soundy" analysis option and improved code parser + * `PVS-Studio `__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 `_) diff --git a/docs b/docs index 1a4a4bf1..31718e53 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1a4a4bf127ec0651f7ca299e031379ea96734139 +Subproject commit 31718e5365e11839db35ff920f71067c9c1e092a From cf4b835b0c757a8c6d7d68b3606162857829e8b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Sep 2020 14:42:59 +0300 Subject: [PATCH 223/223] Bump version to 5.0.0 --- HISTORY.rst | 2 +- README.rst | 8 ++++---- docs | 2 +- platformio/__init__.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d2622833..c28e2f39 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,7 +10,7 @@ PlatformIO Core 5 - `Migration guide from 4.x to 5.0 `__ -5.0.0 (2020-??-??) +5.0.0 (2020-09-03) ~~~~~~~~~~~~~~~~~~ * Integration with the new **PlatformIO Trusted Registry** diff --git a/README.rst b/README.rst index c4ab3d5f..bdd2858b 100644 --- a/README.rst +++ b/README.rst @@ -66,10 +66,10 @@ Instruments Professional ------------ -* `PIO Check `_ -* `PIO Remote `_ -* `PIO Unified Debugger `_ -* `PIO Unit Testing `_ +* `Debugging `_ +* `Unit Testing `_ +* `Static Code Analysis `_ +* `Remote Development `_ Registry -------- diff --git a/docs b/docs index 31718e53..03a83c99 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 31718e5365e11839db35ff920f71067c9c1e092a +Subproject commit 03a83c996f0c209ce0faaa2bcc285447a7780500 diff --git a/platformio/__init__.py b/platformio/__init__.py index e940a364..74707d9c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "0b3") +VERSION = (5, 0, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -47,7 +47,7 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.2.3", + "contrib-piohome": "~3.3.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0",