Merge branch 'release/v5.0.1'

HISTORY.rst

@@ -8,11 +8,23 @@ PlatformIO Core 5
 
 **A professional collaborative platform for embedded development**
 
-- `Migration guide from 4.x to 5.0 <https://docs.platformio.org/page/core/migration.html>`__
+5.0.1 (2020-09-10)
+~~~~~~~~~~~~~~~~~~
+
+- Added support for "owner" requirement when declaring ``dependencies`` using `library.json <https://docs.platformio.org/page/librarymanager/config.html#dependencies>`__
+- Fixed an issue when using a custom git/ssh package with `platform_packages <https://docs.platformio.org/page/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3624 <https://github.com/platformio/platformio-core/issues/3624>`_)
+- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development <https://docs.platformio.org/page/plus/pio-remote.html>`__ on RaspberryPi device (`issue #3652 <https://github.com/platformio/platformio-core/issues/3652>`_)
+- Fixed an issue when `pio package unpublish <https://docs.platformio.org/page/core/userguide/package/cmd_unpublish.html>`__ command crashes (`issue #3660 <https://github.com/platformio/platformio-core/issues/3660>`_)
+- Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
+- Fixed an issue with incorrect value for C++ language standard in IDE projects when an in-progress language standard is used (`issue #3653 <https://github.com/platformio/platformio-core/issues/3653>`_)
+- Fixed an issue with "Invalid simple block (semantic_version)" from library dependency that refs to an external source (repository, ZIP/Tar archives) (`issue #3658 <https://github.com/platformio/platformio-core/issues/3658>`_)
+- Fixed an issue when can not remove update or remove external dev-platform using PlatformIO Home (`issue #3663 <https://github.com/platformio/platformio-core/issues/3663>`_)
 
 5.0.0 (2020-09-03)
 ~~~~~~~~~~~~~~~~~~
 
+Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/page/core/migration.html>`__.
+
 * Integration with the new **PlatformIO Trusted Registry**
 
   - Enterprise-grade package storage with high availability (multi replicas)

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (5, 0, 0)
+VERSION = (5, 0, 1)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/clients/account.py

@@ -80,7 +80,9 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
             )
 
         data = self.fetch_json_data(
-            "post", "/v1/login", data={"username": username, "password": password},
+            "post",
+            "/v1/login",
+            data={"username": username, "password": password},
         )
         app.set_state_item("account", data)
         return data
@@ -108,7 +110,9 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         self.delete_local_session()
         try:
             self.fetch_json_data(
-                "post", "/v1/logout", data={"refresh_token": refresh_token},
+                "post",
+                "/v1/logout",
+                data={"refresh_token": refresh_token},
             )
         except AccountError:
             pass
@@ -153,15 +157,26 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         ).get("auth_token")
 
     def forgot_password(self, username):
-        return self.fetch_json_data("post", "/v1/forgot", data={"username": username},)
+        return self.fetch_json_data(
+            "post",
+            "/v1/forgot",
+            data={"username": username},
+        )
 
     def get_profile(self):
-        return self.send_auth_request("get", "/v1/profile",)
+        return self.send_auth_request(
+            "get",
+            "/v1/profile",
+        )
 
     def update_profile(self, profile, current_password):
         profile["current_password"] = current_password
         self.delete_local_state("summary")
-        response = self.send_auth_request("put", "/v1/profile", data=profile,)
+        response = self.send_auth_request(
+            "put",
+            "/v1/profile",
+            data=profile,
+        )
         return response
 
     def get_account_info(self, offline=False):
@@ -178,7 +193,10 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
                     "username": account.get("username"),
                 }
             }
-        result = self.send_auth_request("get", "/v1/summary",)
+        result = self.send_auth_request(
+            "get",
+            "/v1/summary",
+        )
         account["summary"] = dict(
             profile=result.get("profile"),
             packages=result.get("packages"),
@@ -203,7 +221,10 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return self.send_auth_request("get", "/v1/orgs/%s" % orgname)
 
     def list_orgs(self):
-        return self.send_auth_request("get", "/v1/orgs",)
+        return self.send_auth_request(
+            "get",
+            "/v1/orgs",
+        )
 
     def update_org(self, orgname, data):
         return self.send_auth_request(
@@ -211,19 +232,29 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         )
 
     def destroy_org(self, orgname):
-        return self.send_auth_request("delete", "/v1/orgs/%s" % orgname,)
+        return self.send_auth_request(
+            "delete",
+            "/v1/orgs/%s" % orgname,
+        )
 
     def add_org_owner(self, orgname, username):
         return self.send_auth_request(
-            "post", "/v1/orgs/%s/owners" % orgname, data={"username": username},
+            "post",
+            "/v1/orgs/%s/owners" % orgname,
+            data={"username": username},
         )
 
     def list_org_owners(self, orgname):
-        return self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,)
+        return self.send_auth_request(
+            "get",
+            "/v1/orgs/%s/owners" % orgname,
+        )
 
     def remove_org_owner(self, orgname, username):
         return self.send_auth_request(
-            "delete", "/v1/orgs/%s/owners" % orgname, data={"username": username},
+            "delete",
+            "/v1/orgs/%s/owners" % orgname,
+            data={"username": username},
         )
 
     def create_team(self, orgname, teamname, description):
@@ -235,16 +266,21 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
 
     def destroy_team(self, orgname, teamname):
         return self.send_auth_request(
-            "delete", "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            "delete",
+            "/v1/orgs/%s/teams/%s" % (orgname, teamname),
         )
 
     def get_team(self, orgname, teamname):
         return self.send_auth_request(
-            "get", "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            "get",
+            "/v1/orgs/%s/teams/%s" % (orgname, teamname),
         )
 
     def list_teams(self, orgname):
-        return self.send_auth_request("get", "/v1/orgs/%s/teams" % orgname,)
+        return self.send_auth_request(
+            "get",
+            "/v1/orgs/%s/teams" % orgname,
+        )
 
     def update_team(self, orgname, teamname, data):
         return self.send_auth_request(

platformio/clients/registry.py

@@ -70,12 +70,16 @@ class RegistryClient(HTTPClient):
         if version:
             path += "/" + version
         return self.send_auth_request(
-            "delete", path, params={"undo": 1 if undo else 0},
+            "delete",
+            path,
+            params={"undo": 1 if undo else 0},
         )
 
     def update_resource(self, urn, private):
         return self.send_auth_request(
-            "put", "/v3/resources/%s" % urn, data={"private": int(private)},
+            "put",
+            "/v3/resources/%s" % urn,
+            data={"private": int(private)},
         )
 
     def grant_access_for_resource(self, urn, client, level):
@@ -87,7 +91,9 @@ class RegistryClient(HTTPClient):
 
     def revoke_access_from_resource(self, urn, client):
         return self.send_auth_request(
-            "delete", "/v3/resources/%s/access" % urn, data={"client": client},
+            "delete",
+            "/v3/resources/%s/access" % urn,
+            data={"client": client},
         )
 
     def list_resources(self, owner):

platformio/commands/access.py

@@ -47,27 +47,31 @@ def validate_urn(value):
 
 @cli.command("public", short_help="Make resource public")
 @click.argument(
-    "urn", callback=lambda _, __, value: validate_urn(value),
+    "urn",
+    callback=lambda _, __, value: validate_urn(value),
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_public(urn, urn_type):
     client = RegistryClient()
     client.update_resource(urn=urn, private=0)
     return click.secho(
-        "The resource %s has been successfully updated." % urn, fg="green",
+        "The resource %s has been successfully updated." % urn,
+        fg="green",
     )
 
 
 @cli.command("private", short_help="Make resource private")
 @click.argument(
-    "urn", callback=lambda _, __, value: validate_urn(value),
+    "urn",
+    callback=lambda _, __, value: validate_urn(value),
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_private(urn, urn_type):
     client = RegistryClient()
     client.update_resource(urn=urn, private=1)
     return click.secho(
-        "The resource %s has been successfully updated." % urn, fg="green",
+        "The resource %s has been successfully updated." % urn,
+        fg="green",
     )
 
 
@@ -79,14 +83,16 @@ def access_private(urn, urn_type):
     callback=lambda _, __, value: validate_client(value),
 )
 @click.argument(
-    "urn", callback=lambda _, __, value: validate_urn(value),
+    "urn",
+    callback=lambda _, __, value: validate_urn(value),
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_grant(level, client, urn, urn_type):
     reg_client = RegistryClient()
     reg_client.grant_access_for_resource(urn=urn, client=client, level=level)
     return click.secho(
-        "Access for resource %s has been granted for %s" % (urn, client), fg="green",
+        "Access for resource %s has been granted for %s" % (urn, client),
+        fg="green",
     )
 
 
@@ -97,14 +103,16 @@ def access_grant(level, client, urn, urn_type):
     callback=lambda _, __, value: validate_client(value),
 )
 @click.argument(
-    "urn", callback=lambda _, __, value: validate_urn(value),
+    "urn",
+    callback=lambda _, __, value: validate_urn(value),
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_revoke(client, urn, urn_type):
     reg_client = RegistryClient()
     reg_client.revoke_access_from_resource(urn=urn, client=client)
     return click.secho(
-        "Access for resource %s has been revoked for %s" % (urn, client), fg="green",
+        "Access for resource %s has been revoked for %s" % (urn, client),
+        fg="green",
     )
 
 

platformio/commands/account.py

@@ -192,7 +192,10 @@ def account_destroy():
         client.logout()
     except AccountNotAuthorized:
         pass
-    return click.secho("User account has been destroyed.", fg="green",)
+    return click.secho(
+        "User account has been destroyed.",
+        fg="green",
+    )
 
 
 @cli.command("show", short_help="PlatformIO Account information")

platformio/commands/device/command.py

@@ -203,7 +203,9 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
             kwargs["port"] = ports[0]["port"]
         elif "platform" in project_options and "board" in project_options:
             board_hwids = device_helpers.get_board_hwids(
-                kwargs["project_dir"], platform, project_options["board"],
+                kwargs["project_dir"],
+                platform,
+                project_options["board"],
             )
             for item in ports:
                 for hwid in board_hwids:

platformio/commands/lib/command.py

@@ -22,11 +22,7 @@ from tabulate import tabulate
 
 from platformio import exception, fs, util
 from platformio.commands import PlatformioCLI
-from platformio.commands.lib.helpers import (
+from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps
-    get_builtin_libs,
-    is_builtin_lib,
-    save_project_libdeps,
-)
 from platformio.compat import dump_json_to_unicode
 from platformio.package.exception import NotGlobalLibDir, UnknownPackageError
 from platformio.package.manager.library import LibraryPackageManager
@@ -164,15 +160,8 @@ def lib_install(  # pylint: disable=too-many-arguments,unused-argument
             }
 
         elif storage_dir in storage_libdeps:
-            builtin_lib_storages = None
             for library in storage_libdeps[storage_dir]:
-                try:
+                lm.install(library, silent=silent, force=force)
-                    lm.install(library, silent=silent, force=force)
-                except UnknownPackageError as e:
-                    if builtin_lib_storages is None:
-                        builtin_lib_storages = get_builtin_libs()
-                    if not silent or not is_builtin_lib(builtin_lib_storages, library):
-                        click.secho("Warning! %s" % e, fg="yellow")
 
     if save and installed_pkgs:
         _save_deps(ctx, installed_pkgs)

platformio/commands/lib/helpers.py

@@ -45,10 +45,11 @@ def get_builtin_libs(storage_names=None):
     return items
 
 
-def is_builtin_lib(storages, name):
+def is_builtin_lib(name, storages=None):
-    for storage in storages or []:
+    for storage in storages or get_builtin_libs():
-        if any(lib.get("name") == name for lib in storage["items"]):
+        for lib in storage["items"]:
-            return True
+            if lib.get("name") == name:
+                return True
     return False
 
 

platformio/commands/org.py

@@ -34,17 +34,21 @@ def validate_orgname(value):
 
 @cli.command("create", short_help="Create a new organization")
 @click.argument(
-    "orgname", callback=lambda _, __, value: validate_orgname(value),
+    "orgname",
+    callback=lambda _, __, value: validate_orgname(value),
 )
 @click.option(
     "--email", callback=lambda _, __, value: validate_email(value) if value else value
 )
-@click.option("--displayname",)
+@click.option(
+    "--displayname",
+)
 def org_create(orgname, email, displayname):
     client = AccountClient()
     client.create_org(orgname, email, displayname)
     return click.secho(
-        "The organization `%s` has been successfully created." % orgname, fg="green",
+        "The organization `%s` has been successfully created." % orgname,
+        fg="green",
     )
 
 
@@ -121,12 +125,19 @@ def account_destroy(orgname):
         abort=True,
     )
     client.destroy_org(orgname)
-    return click.secho("Organization `%s` has been destroyed." % orgname, fg="green",)
+    return click.secho(
+        "Organization `%s` has been destroyed." % orgname,
+        fg="green",
+    )
 
 
 @cli.command("add", short_help="Add a new owner to organization")
-@click.argument("orgname",)
+@click.argument(
-@click.argument("username",)
+    "orgname",
+)
+@click.argument(
+    "username",
+)
 def org_add_owner(orgname, username):
     client = AccountClient()
     client.add_org_owner(orgname, username)
@@ -138,8 +149,12 @@ def org_add_owner(orgname, username):
 
 
 @cli.command("remove", short_help="Remove an owner from organization")
-@click.argument("orgname",)
+@click.argument(
-@click.argument("username",)
+    "orgname",
+)
+@click.argument(
+    "username",
+)
 def org_remove_owner(orgname, username):
     client = AccountClient()
     client.remove_org_owner(orgname, username)

platformio/commands/package.py

@@ -107,7 +107,7 @@ def package_unpublish(package, type, undo):  # pylint: disable=redefined-builtin
         type=type,
         name=spec.name,
         owner=spec.owner,
-        version=spec.requirements,
+        version=str(spec.requirements),
         undo=undo,
     )
     click.secho(response.get("message"), fg="green")

platformio/commands/remote/factory/client.py

@@ -45,7 +45,10 @@ class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory
             return d
 
         d = self.login(
-            credentials.UsernamePassword(auth_token.encode(), get_host_id().encode(),),
+            credentials.UsernamePassword(
+                auth_token.encode(),
+                get_host_id().encode(),
+            ),
             client=self.remote_client,
         )
         d.addCallback(self.remote_client.cb_client_authorization_made)

platformio/commands/team.py

@@ -63,13 +63,16 @@ def cli():
         value, teamname_validate=True
     ),
 )
-@click.option("--description",)
+@click.option(
+    "--description",
+)
 def team_create(orgname_teamname, description):
     orgname, teamname = orgname_teamname.split(":", 1)
     client = AccountClient()
     client.create_team(orgname, teamname, description)
     return click.secho(
-        "The team %s has been successfully created." % teamname, fg="green",
+        "The team %s has been successfully created." % teamname,
+        fg="green",
     )
 
 
@@ -123,7 +126,9 @@ def team_list(orgname, json_output):
     callback=lambda _, __, value: validate_teamname(value),
     help="A new team name",
 )
-@click.option("--description",)
+@click.option(
+    "--description",
+)
 def team_update(orgname_teamname, **kwargs):
     orgname, teamname = orgname_teamname.split(":", 1)
     client = AccountClient()
@@ -142,7 +147,8 @@ def team_update(orgname_teamname, **kwargs):
         new_team.update({key: value for key, value in kwargs.items() if value})
     client.update_team(orgname, teamname, new_team)
     return click.secho(
-        "The team %s has been successfully updated." % teamname, fg="green",
+        "The team %s has been successfully updated." % teamname,
+        fg="green",
     )
 
 
@@ -163,7 +169,8 @@ def team_destroy(orgname_teamname):
     client = AccountClient()
     client.destroy_team(orgname, teamname)
     return click.secho(
-        "The team %s has been successfully destroyed." % teamname, fg="green",
+        "The team %s has been successfully destroyed." % teamname,
+        fg="green",
     )
 
 
@@ -173,7 +180,9 @@ def team_destroy(orgname_teamname):
     metavar="ORGNAME:TEAMNAME",
     callback=lambda _, __, value: validate_orgname_teamname(value),
 )
-@click.argument("username",)
+@click.argument(
+    "username",
+)
 def team_add_member(orgname_teamname, username):
     orgname, teamname = orgname_teamname.split(":", 1)
     client = AccountClient()

platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -63,7 +63,7 @@ SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}")
 SET(CMAKE_CXX_FLAGS "{{ _normalize_path(to_unix_path(cxx_flags)) }}")
 SET(CMAKE_C_FLAGS "{{ _normalize_path(to_unix_path(cc_flags)) }}")
 
-% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)")
 % cc_stds = STD_RE.findall(cc_flags)
 % cxx_stds = STD_RE.findall(cxx_flags)
 % if cc_stds:

platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl

@@ -1,5 +1,5 @@
 % import re
-% STD_RE = re.compile(r"(\-std=[a-z\+]+\d+)")
+% STD_RE = re.compile(r"(\-std=[a-z\+]+\w+)")
 % cxx_stds = STD_RE.findall(cxx_flags)
 % cxx_std = cxx_stds[-1] if cxx_stds else ""
 %

platformio/ide/tpls/emacs/.ccls.tpl

@@ -1,5 +1,5 @@
 % import re
-% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)")
 % cc_stds = STD_RE.findall(cc_flags)
 % cxx_stds = STD_RE.findall(cxx_flags)
 %

platformio/ide/tpls/vim/.ccls.tpl

@@ -1,5 +1,5 @@
 % import re
-% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)")
 % cc_stds = STD_RE.findall(cc_flags)
 % cxx_stds = STD_RE.findall(cxx_flags)
 %

platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl

@@ -6,6 +6,14 @@
 %
 % systype = platform.system().lower()
 %
+% cpp_standards_remap = {
+%   "0x": "11",
+%   "1y": "14",
+%   "1z": "17",
+%   "2a": "20",
+%   "2b": "23"
+% }
+%
 % def _escape(text):
 %   return to_unix_path(text).replace('"', '\\"')
 % end
@@ -68,7 +76,7 @@
 %
 % cleaned_includes = filter_includes(includes, ["toolchain"])
 %
-% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)")
 % cc_stds = STD_RE.findall(cc_flags)
 % cxx_stds = STD_RE.findall(cxx_flags)
 % cc_m_flags = split_args(cc_flags)
@@ -115,7 +123,7 @@
             "cStandard": "c{{ cc_stds[-1] }}",
 % end
 % if cxx_stds:
-            "cppStandard": "c++{{ cxx_stds[-1] }}",
+            "cppStandard": "c++{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}",
 % end
 % if forced_includes:
             "forcedInclude": [

platformio/maintenance.py

@@ -124,7 +124,9 @@ class Upgrader(object):
                 continue
             result = result[0]
             pkg.metadata.spec = PackageSpec(
-                id=result["id"], owner=result["owner"]["username"], name=result["name"],
+                id=result["id"],
+                owner=result["owner"]["username"],
+                name=result["name"],
             )
             pkg.dump_meta()
         return True

platformio/package/manager/base.py

@@ -91,7 +91,10 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
     @staticmethod
     def ensure_dir_exists(path):
         if not os.path.isdir(path):
-            os.makedirs(path)
+            try:
+                os.makedirs(path)
+            except:  # pylint: disable=bare-except
+                pass
         assert os.path.isdir(path)
         return path
 

platformio/package/manager/core.py

@@ -16,11 +16,12 @@ import json
 import os
 import subprocess
 import sys
+from datetime import date
 
 from platformio import __core_packages__, exception, fs, util
 from platformio.compat import PY2
 from platformio.package.manager.tool import ToolPackageManager
-from platformio.package.meta import PackageSpec
+from platformio.package.meta import PackageItem, PackageSpec
 from platformio.proc import get_pythonexe_path
 
 
@@ -95,16 +96,8 @@ def build_contrib_pysite_deps(target_dir):
     if os.path.isdir(target_dir):
         fs.rmtree(target_dir)
     os.makedirs(target_dir)
-    with open(os.path.join(target_dir, "package.json"), "w") as fp:
-        json.dump(
-            dict(
-                name="contrib-pysite",
-                version="2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
-                system=util.get_systype(),
-            ),
-            fp,
-        )
 
+    # build dependencies
     pythonexe = get_pythonexe_path()
     for dep in get_contrib_pysite_deps():
         subprocess.check_call(
@@ -115,11 +108,36 @@ def build_contrib_pysite_deps(target_dir):
                 "install",
                 # "--no-cache-dir",
                 "--no-compile",
+                "--no-binary",
+                ":all:",
                 "-t",
                 target_dir,
                 dep,
             ]
         )
+
+    # build manifests
+    with open(os.path.join(target_dir, "package.json"), "w") as fp:
+        json.dump(
+            dict(
+                name="contrib-pysite",
+                version="2.%d%d.%s"
+                % (
+                    sys.version_info.major,
+                    sys.version_info.minor,
+                    date.today().strftime("%y%m%d"),
+                ),
+                system=util.get_systype(),
+            ),
+            fp,
+        )
+    pm = ToolPackageManager()
+    pkg = PackageItem(target_dir)
+    pkg.metadata = pm.build_metadata(
+        target_dir, PackageSpec(owner="platformio", name="contrib-pysite")
+    )
+    pkg.dump_meta()
+
     return True
 
 
@@ -130,7 +148,7 @@ def get_contrib_pysite_deps():
     twisted_version = "19.10.0" if PY2 else "20.3.0"
     result = [
         "twisted == %s" % twisted_version,
-        "autobahn == 20.4.3",
+        "autobahn == %s" % ("19.11.2" if PY2 else "20.4.3"),
         "json-rpc == 1.13.0",
     ]
 

platformio/package/manager/library.py

@@ -15,7 +15,10 @@
 import json
 import os
 
-from platformio.package.exception import MissingPackageManifestError
+from platformio.package.exception import (
+    MissingPackageManifestError,
+    UnknownPackageError,
+)
 from platformio.package.manager.base import BasePackageManager
 from platformio.package.meta import PackageItem, PackageSpec, PackageType
 from platformio.project.helpers import get_project_global_lib_dir
@@ -43,7 +46,10 @@ class LibraryPackageManager(BasePackageManager):  # pylint: disable=too-many-anc
         # automatically generate library manifest
         with open(os.path.join(root_dir, "library.json"), "w") as fp:
             json.dump(
-                dict(name=spec.name, version=self.generate_rand_version(),),
+                dict(
+                    name=spec.name,
+                    version=self.generate_rand_version(),
+                ),
                 fp,
                 indent=2,
             )
@@ -63,6 +69,33 @@ class LibraryPackageManager(BasePackageManager):  # pylint: disable=too-many-anc
                 return root
         return path
 
+    def _install(  # pylint: disable=too-many-arguments
+        self,
+        spec,
+        search_filters=None,
+        silent=False,
+        skip_dependencies=False,
+        force=False,
+    ):
+        try:
+            return super(LibraryPackageManager, self)._install(
+                spec,
+                search_filters=search_filters,
+                silent=silent,
+                skip_dependencies=skip_dependencies,
+                force=force,
+            )
+        except UnknownPackageError as e:
+            # pylint: disable=import-outside-toplevel
+            from platformio.commands.lib.helpers import is_builtin_lib
+
+            spec = self.ensure_spec(spec)
+            if is_builtin_lib(spec.name):
+                self.print_message("Already installed, built-in library", fg="yellow")
+                return True
+
+            raise e
+
     def install_dependencies(self, pkg, silent=False):
         assert isinstance(pkg, PackageItem)
         manifest = self.load_manifest(pkg)
@@ -79,9 +112,16 @@ class LibraryPackageManager(BasePackageManager):  # pylint: disable=too-many-anc
                 )
 
     def _install_dependency(self, dependency, silent=False):
-        spec = PackageSpec(
+        if set(["name", "version"]) <= set(dependency.keys()) and any(
-            name=dependency.get("name"), requirements=dependency.get("version")
+            c in dependency["version"] for c in (":", "/", "@")
-        )
+        ):
+            spec = PackageSpec("%s=%s" % (dependency["name"], dependency["version"]))
+        else:
+            spec = PackageSpec(
+                owner=dependency.get("owner"),
+                name=dependency.get("name"),
+                requirements=dependency.get("version"),
+            )
         search_filters = {
             key: value
             for key, value in dependency.items()

platformio/package/manifest/parser.py

@@ -388,7 +388,15 @@ class LibraryJsonManifestParser(BaseManifestParser):
             raw = [raw]
 
         if isinstance(raw, dict):
-            return [dict(name=name, version=version) for name, version in raw.items()]
+            result = []
+            for name, version in raw.items():
+                if "/" in name:
+                    owner, name = name.split("/", 1)
+                    result.append(dict(owner=owner, name=name, version=version))
+                else:
+                    result.append(dict(name=name, version=version))
+            return result
+
         if isinstance(raw, list):
             for i, dependency in enumerate(raw):
                 if isinstance(dependency, dict):

platformio/package/manifest/schema.py

@@ -106,6 +106,7 @@ class RepositorySchema(StrictSchema):
 
 
 class DependencySchema(StrictSchema):
+    owner = fields.Str(validate=validate.Length(min=1, max=100))
     name = fields.Str(required=True, validate=validate.Length(min=1, max=100))
     version = fields.Str(validate=validate.Length(min=1, max=100))
     authors = StrictListField(fields.Str(validate=validate.Length(min=1, max=50)))
@@ -242,7 +243,7 @@ class ManifestSchema(BaseSchema):
             raise ValidationError("Could not load SPDX licenses for validation")
         for item in spdx.get("licenses", []):
             if item.get("licenseId") == value:
-                return
+                return True
         raise ValidationError(
             "Invalid SPDX license identifier. See valid identifiers at "
             "https://spdx.org/licenses/"
@@ -251,9 +252,5 @@ class ManifestSchema(BaseSchema):
     @staticmethod
     @memoized(expire="1h")
     def load_spdx_licenses():
-        version = "3.10"
+        spdx_data_url = "https://dl.bintray.com/platformio/dl-misc/spdx-licenses-3.json"
-        spdx_data_url = (
-            "https://raw.githubusercontent.com/spdx/license-list-data"
-            "/v%s/json/licenses.json" % version
-        )
         return json.loads(fetch_remote_content(spdx_data_url))

platformio/package/meta.py

@@ -209,6 +209,7 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
         raw = raw.strip()
 
         parsers = (
+            self._parse_local_file,
             self._parse_requirements,
             self._parse_custom_name,
             self._parse_id,
@@ -227,10 +228,16 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
             # the leftover is a package name
             self.name = raw
 
-    def _parse_requirements(self, raw):
+    @staticmethod
-        if "@" not in raw:
+    def _parse_local_file(raw):
+        if raw.startswith("file://") or not any(c in raw for c in ("/", "\\")):
             return raw
-        if raw.startswith("file://") and os.path.exists(raw[7:]):
+        if os.path.exists(raw):
+            return "file://%s" % raw
+        return raw
+
+    def _parse_requirements(self, raw):
+        if "@" not in raw or raw.startswith("file://"):
             return raw
         tokens = raw.rsplit("@", 1)
         if any(s in tokens[1] for s in (":", "/")):

platformio/package/pack.py

@@ -51,7 +51,9 @@ class PackagePacker(object):
             r"[^\da-zA-Z\-\._\+]+",
             "",
             "{name}{system}-{version}.tar.gz".format(
-                name=name, system=("-" + system) if system else "", version=version,
+                name=name,
+                system=("-" + system) if system else "",
+                version=version,
             ),
         )
 

platformio/platform/base.py

@@ -94,7 +94,7 @@ class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-pub
             name = item
             version = "*"
             if "@" in item:
-                name, version = item.split("@", 2)
+                name, version = item.split("@", 1)
             spec = self.pm.ensure_spec(name)
             options = {"version": version.strip(), "optional": False}
             if spec.owner:

platformio/telemetry.py

@@ -301,7 +301,11 @@ def on_command():
 def on_exception(e):
     skip_conditions = [
         isinstance(e, cls)
-        for cls in (IOError, exception.ReturnErrorCode, exception.UserSideException,)
+        for cls in (
+            IOError,
+            exception.ReturnErrorCode,
+            exception.UserSideException,
+        )
     ]
     if any(skip_conditions):
         return

setup.py

@@ -52,7 +52,7 @@ setup(
         [">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"]
     ),
     install_requires=install_requires,
-    packages=find_packages() + ["scripts"],
+    packages=find_packages(exclude=["tests.*", "tests"]) + ["scripts"],
     package_data={
         "platformio": [
             "ide/tpls/*/.*.tpl",

tests/commands/test_account_org_team.py

@@ -100,14 +100,21 @@ def test_account_register(
 
 
 def test_account_login(
-    clirunner, validate_cliresult, isolated_pio_core,
+    clirunner,
+    validate_cliresult,
+    isolated_pio_core,
 ):
-    result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],)
+    result = clirunner.invoke(
+        cmd_account,
+        ["login", "-u", username, "-p", password],
+    )
     validate_cliresult(result)
 
 
 def test_account_summary(
-    clirunner, validate_cliresult, isolated_pio_core,
+    clirunner,
+    validate_cliresult,
+    isolated_pio_core,
 ):
     result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"])
     validate_cliresult(result)
@@ -160,13 +167,21 @@ def test_account_summary(
 
 
 def test_account_token(clirunner, validate_cliresult, isolated_pio_core):
-    result = clirunner.invoke(cmd_account, ["token", "--password", password,],)
+    result = clirunner.invoke(
+        cmd_account,
+        [
+            "token",
+            "--password",
+            password,
+        ],
+    )
     validate_cliresult(result)
     assert "Personal Authentication Token:" in result.output
     token = result.output.strip().split(": ")[-1]
 
     result = clirunner.invoke(
-        cmd_account, ["token", "--password", password, "--json-output"],
+        cmd_account,
+        ["token", "--password", password, "--json-output"],
     )
     validate_cliresult(result)
     json_result = json.loads(result.output.strip())
@@ -177,7 +192,14 @@ def test_account_token(clirunner, validate_cliresult, isolated_pio_core):
 
     clirunner.invoke(cmd_account, ["logout"])
 
-    result = clirunner.invoke(cmd_account, ["token", "--password", password,],)
+    result = clirunner.invoke(
+        cmd_account,
+        [
+            "token",
+            "--password",
+            password,
+        ],
+    )
     assert result.exit_code > 0
     assert result.exception
     assert "You are not authorized! Please log in to PIO Account" in str(
@@ -187,7 +209,8 @@ def test_account_token(clirunner, validate_cliresult, isolated_pio_core):
     os.environ["PLATFORMIO_AUTH_TOKEN"] = token
 
     result = clirunner.invoke(
-        cmd_account, ["token", "--password", password, "--json-output"],
+        cmd_account,
+        ["token", "--password", password, "--json-output"],
     )
     validate_cliresult(result)
     json_result = json.loads(result.output.strip())
@@ -197,7 +220,10 @@ def test_account_token(clirunner, validate_cliresult, isolated_pio_core):
 
     os.environ.pop("PLATFORMIO_AUTH_TOKEN")
 
-    result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],)
+    result = clirunner.invoke(
+        cmd_account,
+        ["login", "-u", username, "-p", password],
+    )
     validate_cliresult(result)
 
 
@@ -205,7 +231,13 @@ def test_account_change_password(clirunner, validate_cliresult, isolated_pio_cor
     new_password = "Testpassword123"
     result = clirunner.invoke(
         cmd_account,
-        ["password", "--old-password", password, "--new-password", new_password,],
+        [
+            "password",
+            "--old-password",
+            password,
+            "--new-password",
+            new_password,
+        ],
     )
     validate_cliresult(result)
     assert "Password successfully changed!" in result.output
@@ -213,13 +245,20 @@ def test_account_change_password(clirunner, validate_cliresult, isolated_pio_cor
     clirunner.invoke(cmd_account, ["logout"])
 
     result = clirunner.invoke(
-        cmd_account, ["login", "-u", username, "-p", new_password],
+        cmd_account,
+        ["login", "-u", username, "-p", new_password],
     )
     validate_cliresult(result)
 
     result = clirunner.invoke(
         cmd_account,
-        ["password", "--old-password", new_password, "--new-password", password,],
+        [
+            "password",
+            "--old-password",
+            new_password,
+            "--new-password",
+            password,
+        ],
     )
     validate_cliresult(result)
 
@@ -272,14 +311,20 @@ def test_account_update(
     link = link.replace("&", "&")
     session.get(link)
 
-    result = clirunner.invoke(cmd_account, ["show"],)
+    result = clirunner.invoke(
+        cmd_account,
+        ["show"],
+    )
     assert result.exit_code > 0
     assert result.exception
     assert "You are not authorized! Please log in to PIO Account" in str(
         result.exception
     )
 
-    result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],)
+    result = clirunner.invoke(
+        cmd_account,
+        ["login", "-u", username, "-p", password],
+    )
     validate_cliresult(result)
 
 
@@ -317,7 +362,8 @@ def test_account_update(
 
 def test_org_create(clirunner, validate_cliresult, isolated_pio_core):
     result = clirunner.invoke(
-        cmd_org, ["create", "--email", email, "--displayname", display_name, orgname],
+        cmd_org,
+        ["create", "--email", email, "--displayname", display_name, orgname],
     )
     validate_cliresult(result)
 
@@ -405,13 +451,21 @@ def test_org_update(clirunner, validate_cliresult, isolated_pio_core):
 def test_team_create(clirunner, validate_cliresult, isolated_pio_core):
     result = clirunner.invoke(
         cmd_team,
-        ["create", "%s:%s" % (orgname, teamname), "--description", team_description,],
+        [
+            "create",
+            "%s:%s" % (orgname, teamname),
+            "--description",
+            team_description,
+        ],
     )
     validate_cliresult(result)
 
 
 def test_team_list(clirunner, validate_cliresult, isolated_pio_core):
-    result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],)
+    result = clirunner.invoke(
+        cmd_team,
+        ["list", "%s" % orgname, "--json-output"],
+    )
     validate_cliresult(result)
     json_result = json.loads(result.output.strip())
     for item in json_result:
@@ -423,22 +477,30 @@ def test_team_list(clirunner, validate_cliresult, isolated_pio_core):
 
 def test_team_add_member(clirunner, validate_cliresult, isolated_pio_core):
     result = clirunner.invoke(
-        cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username],
+        cmd_team,
+        ["add", "%s:%s" % (orgname, teamname), second_username],
     )
     validate_cliresult(result)
 
-    result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],)
+    result = clirunner.invoke(
+        cmd_team,
+        ["list", "%s" % orgname, "--json-output"],
+    )
     validate_cliresult(result)
     assert second_username in result.output
 
 
 def test_team_remove(clirunner, validate_cliresult, isolated_pio_core):
     result = clirunner.invoke(
-        cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username],
+        cmd_team,
+        ["remove", "%s:%s" % (orgname, teamname), second_username],
     )
     validate_cliresult(result)
 
-    result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],)
+    result = clirunner.invoke(
+        cmd_team,
+        ["list", "%s" % orgname, "--json-output"],
+    )
     validate_cliresult(result)
     assert second_username not in result.output
 
@@ -459,7 +521,10 @@ def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_
     )
     validate_cliresult(result)
 
-    result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],)
+    result = clirunner.invoke(
+        cmd_team,
+        ["list", "%s" % orgname, "--json-output"],
+    )
     validate_cliresult(result)
     json_result = json.loads(result.output.strip())
     for item in json_result:

tests/commands/test_check.py

@@ -446,7 +446,10 @@ int main() {
             result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)])
             validate_cliresult(result)
             defects = sum(count_defects(result.output))
-            assert defects > 0, "Failed %s with %s" % (framework, tool,)
+            assert defects > 0, "Failed %s with %s" % (
+                framework,
+                tool,
+            )
 
 
 def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir):

tests/commands/test_lib_complex.py

@@ -339,14 +339,17 @@ def test_lib_stats(clirunner, validate_cliresult):
 
     result = clirunner.invoke(cmd_lib, ["stats", "--json-output"])
     validate_cliresult(result)
-    assert set(
+    assert (
-        [
+        set(
-            "dlweek",
+            [
-            "added",
+                "dlweek",
-            "updated",
+                "added",
-            "topkeywords",
+                "updated",
-            "dlmonth",
+                "topkeywords",
-            "dlday",
+                "dlmonth",
-            "lastkeywords",
+                "dlday",
-        ]
+                "lastkeywords",
-    ) == set(json.loads(result.output).keys())
+            ]
+        )
+        == set(json.loads(result.output).keys())
+    )

tests/commands/test_platform.py

@@ -60,7 +60,8 @@ def test_install_unknown_from_registry(clirunner):
 
 def test_install_core_3_dev_platform(clirunner, validate_cliresult, isolated_pio_core):
     result = clirunner.invoke(
-        cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"],
+        cli_platform.platform_install,
+        ["atmelavr@1.2.0", "--skip-default-package"],
     )
     assert result.exit_code == 0
 

tests/commands/test_test.py

@@ -77,7 +77,8 @@ void loop() {}
     )
 
     result = clirunner.invoke(
-        cmd_test, ["-d", str(project_dir), "--without-testing", "--without-uploading"],
+        cmd_test,
+        ["-d", str(project_dir), "--without-testing", "--without-uploading"],
     )
 
     validate_cliresult(result)
@@ -127,7 +128,8 @@ int main() {
     )
 
     native_result = clirunner.invoke(
-        cmd_test, ["-d", str(project_dir), "-e", "native"],
+        cmd_test,
+        ["-d", str(project_dir), "-e", "native"],
     )
 
     test_dir.join("unittest_transport.h").write(

tests/package/test_manager.py

@@ -230,6 +230,41 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory):
         tm.install("owner/unknown-package-tool", silent=True)
 
 
+def test_install_lib_depndencies(isolated_pio_core, tmpdir_factory):
+    tmp_dir = tmpdir_factory.mktemp("tmp")
+
+    src_dir = tmp_dir.join("lib-with-deps").mkdir()
+    root_dir = src_dir.mkdir("root")
+    root_dir.mkdir("src").join("main.cpp").write("#include <stdio.h>")
+    root_dir.join("library.json").write(
+        """
+{
+  "name": "lib-with-deps",
+  "version": "2.0.0",
+  "dependencies": [
+    {
+      "owner": "bblanchon",
+      "name": "ArduinoJson",
+      "version": "^6.16.1"
+    },
+    {
+      "name": "external-repo",
+      "version": "https://github.com/milesburton/Arduino-Temperature-Control-Library.git#4a0ccc1"
+    }
+  ]
+}
+"""
+    )
+
+    lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage")))
+    lm.install("file://%s" % str(src_dir), silent=True)
+    installed = lm.get_installed()
+    assert len(installed) == 4
+    assert set(["external-repo", "ArduinoJson", "lib-with-deps", "OneWire"]) == set(
+        p.metadata.name for p in installed
+    )
+
+
 def test_install_force(isolated_pio_core, tmpdir_factory):
     lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage")))
     # install #64 ArduinoJson

tests/package/test_manifest.py

@@ -44,7 +44,7 @@ def test_library_json_parser():
     "dependencies": {
         "deps1": "1.2.0",
         "deps2": "https://github.com/username/package.git",
-        "@owner/deps3": "^2.1.3"
+        "owner/deps3": "^2.1.3"
     },
     "customField": "Custom Value"
 }
@@ -65,9 +65,9 @@ def test_library_json_parser():
             "homepage": "http://old.url.format",
             "build": {"flags": ["-DHELLO"]},
             "dependencies": [
-                {"name": "@owner/deps3", "version": "^2.1.3"},
                 {"name": "deps1", "version": "1.2.0"},
                 {"name": "deps2", "version": "https://github.com/username/package.git"},
+                {"owner": "owner", "name": "deps3", "version": "^2.1.3"},
             ],
             "customField": "Custom Value",
         },
@@ -83,7 +83,7 @@ def test_library_json_parser():
     },
     "dependencies": [
         {"name": "deps1", "version": "1.0.0"},
-        {"name": "@owner/deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"},
+        {"owner": "owner", "name": "deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"},
         {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]}
     ]
 }
@@ -98,13 +98,14 @@ def test_library_json_parser():
             "export": {"exclude": ["audio_samples"]},
             "platforms": ["atmelavr"],
             "dependencies": [
+                {"name": "deps1", "version": "1.0.0"},
                 {
-                    "name": "@owner/deps2",
+                    "owner": "owner",
+                    "name": "deps2",
                     "version": "1.0.0",
                     "platforms": ["*"],
                     "frameworks": ["arduino", "espidf"],
                 },
-                {"name": "deps1", "version": "1.0.0"},
                 {
                     "name": "deps3",
                     "version": "1.0.0",
@@ -115,16 +116,16 @@ def test_library_json_parser():
     )
 
     raw_data = parser.LibraryJsonManifestParser(
-        '{"dependencies": ["dep1", "dep2", "@owner/dep3"]}'
+        '{"dependencies": ["dep1", "dep2", "owner/dep3@1.2.3"]}'
     ).as_dict()
     raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"])
     assert not jsondiff.diff(
         raw_data,
         {
             "dependencies": [
-                {"name": "@owner/dep3"},
                 {"name": "dep1"},
                 {"name": "dep2"},
+                {"name": "owner/dep3@1.2.3"},
             ],
         },
     )

tests/package/test_meta.py

@@ -90,12 +90,13 @@ def test_spec_local_urls(tmpdir_factory):
     assert PackageSpec("file:///tmp/some-lib/") == PackageSpec(
         url="file:///tmp/some-lib/", name="some-lib"
     )
-    assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec(
+    # detached package
-        url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1"
+    assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec(
+        url="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib"
     )
-    # detached folder with "@" symbol
+    # detached folder without scheme
     pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir()
-    assert PackageSpec("file://%s" % str(pkg_dir)) == PackageSpec(
+    assert PackageSpec(str(pkg_dir)) == PackageSpec(
         name="detached", url="file://%s" % pkg_dir
     )