Switch library manager to the new package manager

platformio/builder/tools/piolib.py

@@ -34,11 +34,13 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
 from platformio.compat import WINDOWS, hashlib_encode_data, string_types
-from platformio.managers.lib import LibraryManager
+from platformio.package.exception import UnknownPackageError
+from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manifest.parser import (
     ManifestParserError,
     ManifestParserFactory,
 )
+from platformio.package.meta import PackageSourceItem
 from platformio.project.options import ProjectOptions
 
 
@@ -851,34 +853,36 @@ class ProjectAsLibBuilder(LibBuilderBase):
         pass
 
     def install_dependencies(self):
-        def _is_builtin(uri):
+        def _is_builtin(spec):
             for lb in self.env.GetLibBuilders():
-                if lb.name == uri:
+                if lb.name == spec:
                     return True
             return False
 
-        not_found_uri = []
+        not_found_specs = []
-        for uri in self.dependencies:
+        for spec in self.dependencies:
             # check if built-in library
-            if _is_builtin(uri):
+            if _is_builtin(spec):
                 continue
 
             found = False
             for storage_dir in self.env.GetLibSourceDirs():
-                lm = LibraryManager(storage_dir)
+                lm = LibraryPackageManager(storage_dir)
-                if lm.get_package_dir(*lm.parse_pkg_uri(uri)):
+                if lm.get_package(spec):
                     found = True
                     break
             if not found:
-                not_found_uri.append(uri)
+                not_found_specs.append(spec)
 
         did_install = False
-        lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")))
+        lm = LibraryPackageManager(
-        for uri in not_found_uri:
+            self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))
+        )
+        for spec in not_found_specs:
             try:
-                lm.install(uri)
+                lm.install(spec)
                 did_install = True
-            except (exception.LibNotFound, exception.InternetIsOffline) as e:
+            except (UnknownPackageError, exception.InternetIsOffline) as e:
                 click.secho("Warning! %s" % e, fg="yellow")
 
         # reset cache
@@ -886,17 +890,17 @@ class ProjectAsLibBuilder(LibBuilderBase):
             DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None)
 
     def process_dependencies(self):  # pylint: disable=too-many-branches
-        for uri in self.dependencies:
+        for spec in self.dependencies:
             found = False
             for storage_dir in self.env.GetLibSourceDirs():
                 if found:
                     break
-                lm = LibraryManager(storage_dir)
+                lm = LibraryPackageManager(storage_dir)
-                lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri))
+                pkg = lm.get_package(spec)
-                if not lib_dir:
+                if not pkg:
                     continue
                 for lb in self.env.GetLibBuilders():
-                    if lib_dir != lb.path:
+                    if pkg.path != lb.path:
                         continue
                     if lb not in self.depbuilders:
                         self.depend_recursive(lb)
@@ -908,7 +912,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
             # look for built-in libraries by a name
             # which don't have package manifest
             for lb in self.env.GetLibBuilders():
-                if lb.name != uri:
+                if lb.name != spec:
                     continue
                 if lb not in self.depbuilders:
                     self.depend_recursive(lb)
@@ -1000,10 +1004,6 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
 
 
 def ConfigureProjectLibBuilder(env):
-    def _get_vcs_info(lb):
-        path = LibraryManager.get_src_manifest_path(lb.path)
-        return fs.load_json(path) if path else None
-
     def _correct_found_libs(lib_builders):
         # build full dependency graph
         found_lbs = [lb for lb in lib_builders if lb.dependent]
@@ -1019,15 +1019,13 @@ def ConfigureProjectLibBuilder(env):
         margin = "|   " * (level)
         for lb in root.depbuilders:
             title = "<%s>" % lb.name
-            vcs_info = _get_vcs_info(lb)
+            pkg = PackageSourceItem(lb.path)
-            if lb.version:
+            if pkg.metadata:
-                title += " %s" % lb.version
+                title += " %s" % pkg.metadata.version
-            if vcs_info and vcs_info.get("version"):
-                title += " #%s" % vcs_info.get("version")
             click.echo("%s|-- %s" % (margin, title), nl=False)
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                if vcs_info:
+                if pkg.metadata and pkg.metadata.spec.external:
-                    click.echo(" [%s]" % vcs_info.get("url"), nl=False)
+                    click.echo(" [%s]" % pkg.metadata.spec.url, nl=False)
                 click.echo(" (", nl=False)
                 click.echo(lb.path, nl=False)
                 click.echo(")", nl=False)

platformio/clients/http.py

@@ -30,7 +30,9 @@ class HTTPClientError(PlatformioException):
 
 
 class HTTPClient(object):
-    def __init__(self, base_url):
+    def __init__(
+        self, base_url,
+    ):
         if base_url.endswith("/"):
             base_url = base_url[:-1]
         self.base_url = base_url
@@ -51,6 +53,7 @@ class HTTPClient(object):
         self._session.close()
         self._session = None
 
+    @util.throttle(500)
     def send_request(self, method, path, **kwargs):
         # check Internet before and resolve issue with 60 seconds timeout
         # print(self, method, path, kwargs)

platformio/commands/lib/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/lib/command.py

@@ -18,16 +18,21 @@ import os
 import time
 
 import click
-import semantic_version
 from tabulate import tabulate
 
 from platformio import exception, fs, util
 from platformio.commands import PlatformioCLI
+from platformio.commands.lib.helpers import (
+    get_builtin_libs,
+    is_builtin_lib,
+    save_project_libdeps,
+)
 from platformio.compat import dump_json_to_unicode
-from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
+from platformio.package.exception import UnknownPackageError
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.meta import PackageSourceItem, PackageSpec
 from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
-from platformio.project.exception import InvalidProjectConfError
 from platformio.project.helpers import get_project_dir, is_platformio_project
 
 try:
@@ -124,89 +129,106 @@ def cli(ctx, **options):
 @cli.command("install", short_help="Install library")
 @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]")
 @click.option(
-    "--save",
+    "--save/--no-save",
     is_flag=True,
-    help="Save installed libraries into the `platformio.ini` dependency list",
+    default=True,
+    help="Save installed libraries into the `platformio.ini` dependency list"
+    " (enabled by default)",
 )
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option(
-    "--interactive", is_flag=True, help="Allow to make a choice for all prompts"
+    "--interactive",
+    is_flag=True,
+    help="Deprecated! Please use a strict dependency specification (owner/libname)",
 )
 @click.option(
     "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists"
 )
 @click.pass_context
-def lib_install(  # pylint: disable=too-many-arguments
+def lib_install(  # pylint: disable=too-many-arguments,unused-argument
     ctx, libraries, save, silent, interactive, force
 ):
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
 
-    installed_manifests = {}
+    installed_pkgs = {}
     for storage_dir in storage_dirs:
         if not silent and (libraries or storage_dir in storage_libdeps):
             print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryManager(storage_dir)
+        lm = LibraryPackageManager(storage_dir)
+
         if libraries:
-            for library in libraries:
+            installed_pkgs = {
-                pkg_dir = lm.install(
+                library: lm.install(library, silent=silent, force=force)
-                    library, silent=silent, interactive=interactive, force=force
+                for library in libraries
-                )
+            }
-                installed_manifests[library] = lm.load_manifest(pkg_dir)
+
         elif storage_dir in storage_libdeps:
             builtin_lib_storages = None
             for library in storage_libdeps[storage_dir]:
                 try:
-                    pkg_dir = lm.install(
+                    lm.install(library, silent=silent, force=force)
-                        library, silent=silent, interactive=interactive, force=force
+                except UnknownPackageError as e:
-                    )
-                    installed_manifests[library] = lm.load_manifest(pkg_dir)
-                except exception.LibNotFound as e:
                     if builtin_lib_storages is None:
                         builtin_lib_storages = get_builtin_libs()
                     if not silent or not is_builtin_lib(builtin_lib_storages, library):
                         click.secho("Warning! %s" % e, fg="yellow")
 
-    if not save or not libraries:
+    if save and installed_pkgs:
-        return
+        _save_deps(ctx, installed_pkgs)
+
+
+def _save_deps(ctx, pkgs, action="add"):
+    specs = []
+    for library, pkg in pkgs.items():
+        spec = PackageSpec(library)
+        if spec.external:
+            specs.append(spec)
+        else:
+            specs.append(
+                PackageSpec(
+                    owner=pkg.metadata.spec.owner,
+                    name=pkg.metadata.spec.name,
+                    requirements=spec.requirements
+                    or (
+                        ("^%s" % pkg.metadata.version)
+                        if not pkg.metadata.version.build
+                        else pkg.metadata.version
+                    ),
+                )
+            )
 
     input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
     project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
     for input_dir in input_dirs:
-        config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini"))
+        if not is_platformio_project(input_dir):
-        config.validate(project_environments)
-        for env in config.envs():
-            if project_environments and env not in project_environments:
             continue
-            config.expand_interpolations = False
+        save_project_libdeps(input_dir, specs, project_environments, action=action)
-            try:
-                lib_deps = config.get("env:" + env, "lib_deps")
-            except InvalidProjectConfError:
-                lib_deps = []
-            for library in libraries:
-                if library in lib_deps:
-                    continue
-                manifest = installed_manifests[library]
-                try:
-                    assert library.lower() == manifest["name"].lower()
-                    assert semantic_version.Version(manifest["version"])
-                    lib_deps.append("{name}@^{version}".format(**manifest))
-                except (AssertionError, ValueError):
-                    lib_deps.append(library)
-            config.set("env:" + env, "lib_deps", lib_deps)
-            config.save()
 
 
-@cli.command("uninstall", short_help="Uninstall libraries")
+@cli.command("uninstall", short_help="Remove libraries")
 @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]")
+@click.option(
+    "--save/--no-save",
+    is_flag=True,
+    default=True,
+    help="Remove libraries from the `platformio.ini` dependency list and save changes"
+    " (enabled by default)",
+)
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.pass_context
-def lib_uninstall(ctx, libraries):
+def lib_uninstall(ctx, libraries, save, silent):
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
+    uninstalled_pkgs = {}
     for storage_dir in storage_dirs:
         print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryManager(storage_dir)
+        lm = LibraryPackageManager(storage_dir)
-        for library in libraries:
+        uninstalled_pkgs = {
-            lm.uninstall(library)
+            library: lm.uninstall(library, silent=silent) for library in libraries
+        }
+
+    if save and uninstalled_pkgs:
+        _save_deps(ctx, uninstalled_pkgs, action="remove")
 
 
 @cli.command("update", short_help="Update installed libraries")
@@ -220,42 +242,53 @@ def lib_uninstall(ctx, libraries):
 @click.option(
     "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
 )
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option("--json-output", is_flag=True)
 @click.pass_context
-def lib_update(ctx, libraries, only_check, dry_run, json_output):
+def lib_update(  # pylint: disable=too-many-arguments
+    ctx, libraries, only_check, dry_run, silent, json_output
+):
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     only_check = dry_run or only_check
     json_result = {}
     for storage_dir in storage_dirs:
         if not json_output:
             print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryManager(storage_dir)
+        lm = LibraryPackageManager(storage_dir)
-
+        _libraries = libraries or lm.get_installed()
-        _libraries = libraries
-        if not _libraries:
-            _libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()]
 
         if only_check and json_output:
             result = []
             for library in _libraries:
-                pkg_dir = library if os.path.isdir(library) else None
+                spec = None
-                requirements = None
+                pkg = None
-                url = None
+                if isinstance(library, PackageSourceItem):
-                if not pkg_dir:
+                    pkg = library
-                    name, requirements, url = lm.parse_pkg_uri(library)
+                else:
-                    pkg_dir = lm.get_package_dir(name, requirements, url)
+                    spec = PackageSpec(library)
-                if not pkg_dir:
+                    pkg = lm.get_package(spec)
+                if not pkg:
                     continue
-                latest = lm.outdated(pkg_dir, requirements)
+                outdated = lm.outdated(pkg, spec)
-                if not latest:
+                if not outdated.is_outdated(allow_incompatible=True):
                     continue
-                manifest = lm.load_manifest(pkg_dir)
+                manifest = lm.legacy_load_manifest(pkg)
-                manifest["versionLatest"] = latest
+                manifest["versionWanted"] = (
+                    str(outdated.wanted) if outdated.wanted else None
+                )
+                manifest["versionLatest"] = (
+                    str(outdated.latest) if outdated.latest else None
+                )
                 result.append(manifest)
             json_result[storage_dir] = result
         else:
             for library in _libraries:
-                lm.update(library, only_check=only_check)
+                spec = (
+                    None
+                    if isinstance(library, PackageSourceItem)
+                    else PackageSpec(library)
+                )
+                lm.update(library, spec=spec, only_check=only_check, silent=silent)
 
     if json_output:
         return click.echo(
@@ -276,8 +309,8 @@ def lib_list(ctx, json_output):
     for storage_dir in storage_dirs:
         if not json_output:
             print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryManager(storage_dir)
+        lm = LibraryPackageManager(storage_dir)
-        items = lm.get_installed()
+        items = lm.legacy_get_installed()
         if json_output:
             json_result[storage_dir] = items
         elif items:
@@ -301,6 +334,7 @@ def lib_list(ctx, json_output):
 @click.option("--json-output", is_flag=True)
 @click.option("--page", type=click.INT, default=1)
 @click.option("--id", multiple=True)
+@click.option("-o", "--owner", multiple=True)
 @click.option("-n", "--name", multiple=True)
 @click.option("-a", "--author", multiple=True)
 @click.option("-k", "--keyword", multiple=True)
@@ -404,12 +438,8 @@ def lib_builtin(storage, json_output):
 @click.argument("library", metavar="[LIBRARY]")
 @click.option("--json-output", is_flag=True)
 def lib_show(library, json_output):
-    lm = LibraryManager()
+    lib_id = LibraryPackageManager().reveal_registry_package_id(
-    name, requirements, _ = lm.parse_pkg_uri(library)
+        library, silent=json_output
-    lib_id = lm.search_lib_id(
-        {"name": name, "requirements": requirements},
-        silent=json_output,
-        interactive=not json_output,
     )
     lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:

platformio/commands/lib/helpers.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from platformio.compat import ci_strings_are_equal
+from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.package.meta import PackageSpec
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import InvalidProjectConfError
+
+
+def get_builtin_libs(storage_names=None):
+    # pylint: disable=import-outside-toplevel
+    from platformio.package.manager.library import LibraryPackageManager
+
+    items = []
+    storage_names = storage_names or []
+    pm = PlatformManager()
+    for manifest in pm.get_installed():
+        p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
+        for storage in p.get_lib_storages():
+            if storage_names and storage["name"] not in storage_names:
+                continue
+            lm = LibraryPackageManager(storage["path"])
+            items.append(
+                {
+                    "name": storage["name"],
+                    "path": storage["path"],
+                    "items": lm.legacy_get_installed(),
+                }
+            )
+    return items
+
+
+def is_builtin_lib(storages, name):
+    for storage in storages or []:
+        if any(lib.get("name") == name for lib in storage["items"]):
+            return True
+    return False
+
+
+def ignore_deps_by_specs(deps, specs):
+    result = []
+    for dep in deps:
+        depspec = PackageSpec(dep)
+        if depspec.external:
+            result.append(dep)
+            continue
+        ignore_conditions = []
+        for spec in specs:
+            if depspec.owner:
+                ignore_conditions.append(
+                    ci_strings_are_equal(depspec.owner, spec.owner)
+                    and ci_strings_are_equal(depspec.name, spec.name)
+                )
+            else:
+                ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name))
+        if not any(ignore_conditions):
+            result.append(dep)
+    return result
+
+
+def save_project_libdeps(project_dir, specs, environments=None, action="add"):
+    config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini"))
+    config.validate(environments)
+    for env in config.envs():
+        if environments and env not in environments:
+            continue
+        config.expand_interpolations = False
+        lib_deps = []
+        try:
+            lib_deps = ignore_deps_by_specs(config.get("env:" + env, "lib_deps"), specs)
+        except InvalidProjectConfError:
+            pass
+        if action == "add":
+            lib_deps.extend(spec.as_dependency() for spec in specs)
+        config.set("env:" + env, "lib_deps", lib_deps)
+    config.save()

platformio/commands/system/command.py

@@ -26,9 +26,9 @@ from platformio.commands.system.completion import (
     install_completion_code,
     uninstall_completion_code,
 )
-from platformio.managers.lib import LibraryManager
 from platformio.managers.package import PackageManager
 from platformio.managers.platform import PlatformManager
+from platformio.package.manager.library import LibraryPackageManager
 from platformio.project.config import ProjectConfig
 
 
@@ -73,7 +73,7 @@ def system_info(json_output):
     }
     data["global_lib_nums"] = {
         "title": "Global Libraries",
-        "value": len(LibraryManager().get_installed()),
+        "value": len(LibraryPackageManager().get_installed()),
     }
     data["dev_platform_nums"] = {
         "title": "Development Platforms",

platformio/commands/update.py

@@ -15,11 +15,11 @@
 import click
 
 from platformio import app
-from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY
+from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
-from platformio.commands.lib import lib_update as cmd_lib_update
+from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.managers.core import update_core_packages
-from platformio.managers.lib import LibraryManager
+from platformio.package.manager.library import LibraryPackageManager
 
 
 @click.command(
@@ -55,5 +55,5 @@ def cli(ctx, core_packages, only_check, dry_run):
     click.echo()
     click.echo("Library Manager")
     click.echo("===============")
-    ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir]
+    ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryPackageManager().package_dir]
     ctx.invoke(cmd_lib_update, only_check=only_check)

platformio/compat.py

@@ -50,6 +50,14 @@ def get_object_members(obj, ignore_private=True):
     }
 
 
+def ci_strings_are_equal(a, b):
+    if a == b:
+        return True
+    if not a or not b:
+        return False
+    return a.strip().lower() == b.strip().lower()
+
+
 if PY2:
     import imp
 

platformio/exception.py

@@ -124,15 +124,6 @@ class PackageInstallError(PlatformIOPackageException):
 #
 
 
-class LibNotFound(PlatformioException):
-
-    MESSAGE = (
-        "Library `{0}` has not been found in PlatformIO Registry.\n"
-        "You can ignore this message, if `{0}` is a built-in library "
-        "(included in framework, SDK). E.g., SPI, Wire, etc."
-    )
-
-
 class NotGlobalLibDir(UserSideException):
 
     MESSAGE = (

platformio/maintenance.py

@@ -21,13 +21,13 @@ import semantic_version
 
 from platformio import __version__, app, exception, fs, telemetry, util
 from platformio.commands import PlatformioCLI
-from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY
+from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
-from platformio.commands.lib import lib_update as cmd_lib_update
+from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.commands.upgrade import get_latest_version
 from platformio.managers.core import update_core_packages
-from platformio.managers.lib import LibraryManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.package.manager.library import LibraryPackageManager
 from platformio.proc import is_container
 
 
@@ -240,7 +240,7 @@ def check_platformio_upgrade():
     click.echo("")
 
 
-def check_internal_updates(ctx, what):
+def check_internal_updates(ctx, what):  # pylint: disable=too-many-branches
     last_check = app.get_state_item("last_check", {})
     interval = int(app.get_setting("check_%s_interval" % what)) * 3600 * 24
     if (time() - interval) < last_check.get(what + "_update", 0):
@@ -251,8 +251,9 @@ def check_internal_updates(ctx, what):
 
     util.internet_on(raise_exception=True)
 
-    pm = PlatformManager() if what == "platforms" else LibraryManager()
     outdated_items = []
+    pm = PlatformManager() if what == "platforms" else LibraryPackageManager()
+    if isinstance(pm, PlatformManager):
         for manifest in pm.get_installed():
             if manifest["name"] in outdated_items:
                 continue
@@ -265,6 +266,12 @@ def check_internal_updates(ctx, what):
             ]
             if any(conds):
                 outdated_items.append(manifest["name"])
+    else:
+        for pkg in pm.get_installed():
+            if pkg.metadata.name in outdated_items:
+                continue
+            if pm.outdated(pkg).is_outdated():
+                outdated_items.append(pkg.metadata.name)
 
     if not outdated_items:
         return

platformio/managers/lib.py

@@ -1,374 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
-# pylint: disable=too-many-return-statements
-
-import json
-from glob import glob
-from os.path import isdir, join
-
-import click
-import semantic_version
-
-from platformio import app, exception, util
-from platformio.compat import glob_escape
-from platformio.managers.package import BasePkgManager
-from platformio.managers.platform import PlatformFactory, PlatformManager
-from platformio.package.exception import ManifestException
-from platformio.package.manifest.parser import ManifestParserFactory
-from platformio.project.config import ProjectConfig
-
-
-class LibraryManager(BasePkgManager):
-
-    FILE_CACHE_VALID = "30d"  # 1 month
-
-    def __init__(self, package_dir=None):
-        self.config = ProjectConfig.get_instance()
-        super(LibraryManager, self).__init__(
-            package_dir or self.config.get_optional_dir("globallib")
-        )
-
-    @property
-    def manifest_names(self):
-        return [".library.json", "library.json", "library.properties", "module.json"]
-
-    def get_manifest_path(self, pkg_dir):
-        path = BasePkgManager.get_manifest_path(self, pkg_dir)
-        if path:
-            return path
-
-        # if library without manifest, returns first source file
-        src_dir = join(glob_escape(pkg_dir))
-        if isdir(join(pkg_dir, "src")):
-            src_dir = join(src_dir, "src")
-        chs_files = glob(join(src_dir, "*.[chS]"))
-        if chs_files:
-            return chs_files[0]
-        cpp_files = glob(join(src_dir, "*.cpp"))
-        if cpp_files:
-            return cpp_files[0]
-
-        return None
-
-    def max_satisfying_repo_version(self, versions, requirements=None):
-        def _cmp_dates(datestr1, datestr2):
-            date1 = util.parse_date(datestr1)
-            date2 = util.parse_date(datestr2)
-            if date1 == date2:
-                return 0
-            return -1 if date1 < date2 else 1
-
-        semver_spec = None
-        try:
-            semver_spec = (
-                semantic_version.SimpleSpec(requirements) if requirements else None
-            )
-        except ValueError:
-            pass
-
-        item = {}
-
-        for v in versions:
-            semver_new = self.parse_semver_version(v["name"])
-            if semver_spec:
-                # pylint: disable=unsupported-membership-test
-                if not semver_new or semver_new not in semver_spec:
-                    continue
-                if not item or self.parse_semver_version(item["name"]) < semver_new:
-                    item = v
-            elif requirements:
-                if requirements == v["name"]:
-                    return v
-
-            else:
-                if not item or _cmp_dates(item["released"], v["released"]) == -1:
-                    item = v
-        return item
-
-    def get_latest_repo_version(self, name, requirements, silent=False):
-        item = self.max_satisfying_repo_version(
-            util.get_api_result(
-                "/lib/info/%d"
-                % self.search_lib_id(
-                    {"name": name, "requirements": requirements}, silent=silent
-                ),
-                cache_valid="1h",
-            )["versions"],
-            requirements,
-        )
-        return item["name"] if item else None
-
-    def _install_from_piorepo(self, name, requirements):
-        assert name.startswith("id="), name
-        version = self.get_latest_repo_version(name, requirements)
-        if not version:
-            raise exception.UndefinedPackageVersion(
-                requirements or "latest", util.get_systype()
-            )
-        dl_data = util.get_api_result(
-            "/lib/download/" + str(name[3:]), dict(version=version), cache_valid="30d"
-        )
-        assert dl_data
-
-        return self._install_from_url(
-            name,
-            dl_data["url"].replace("http://", "https://")
-            if app.get_setting("strict_ssl")
-            else dl_data["url"],
-            requirements,
-        )
-
-    def search_lib_id(  # pylint: disable=too-many-branches
-        self, filters, silent=False, interactive=False
-    ):
-        assert isinstance(filters, dict)
-        assert "name" in filters
-
-        # try to find ID within installed packages
-        lib_id = self._get_lib_id_from_installed(filters)
-        if lib_id:
-            return lib_id
-
-        # looking in PIO Library Registry
-        if not silent:
-            click.echo(
-                "Looking for %s library in registry"
-                % click.style(filters["name"], fg="cyan")
-            )
-        query = []
-        for key in filters:
-            if key not in ("name", "authors", "frameworks", "platforms"):
-                continue
-            values = filters[key]
-            if not isinstance(values, list):
-                values = [v.strip() for v in values.split(",") if v]
-            for value in values:
-                query.append(
-                    '%s:"%s"' % (key[:-1] if key.endswith("s") else key, value)
-                )
-
-        lib_info = None
-        result = util.get_api_result(
-            "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h"
-        )
-        if result["total"] == 1:
-            lib_info = result["items"][0]
-        elif result["total"] > 1:
-            if silent and not interactive:
-                lib_info = result["items"][0]
-            else:
-                click.secho(
-                    "Conflict: More than one library has been found "
-                    "by request %s:" % json.dumps(filters),
-                    fg="yellow",
-                    err=True,
-                )
-                # pylint: disable=import-outside-toplevel
-                from platformio.commands.lib import print_lib_item
-
-                for item in result["items"]:
-                    print_lib_item(item)
-
-                if not interactive:
-                    click.secho(
-                        "Automatically chose the first available library "
-                        "(use `--interactive` option to make a choice)",
-                        fg="yellow",
-                        err=True,
-                    )
-                    lib_info = result["items"][0]
-                else:
-                    deplib_id = click.prompt(
-                        "Please choose library ID",
-                        type=click.Choice([str(i["id"]) for i in result["items"]]),
-                    )
-                    for item in result["items"]:
-                        if item["id"] == int(deplib_id):
-                            lib_info = item
-                            break
-
-        if not lib_info:
-            if list(filters) == ["name"]:
-                raise exception.LibNotFound(filters["name"])
-            raise exception.LibNotFound(str(filters))
-        if not silent:
-            click.echo(
-                "Found: %s"
-                % click.style(
-                    "https://platformio.org/lib/show/{id}/{name}".format(**lib_info),
-                    fg="blue",
-                )
-            )
-        return int(lib_info["id"])
-
-    def _get_lib_id_from_installed(self, filters):
-        if filters["name"].startswith("id="):
-            return int(filters["name"][3:])
-        package_dir = self.get_package_dir(
-            filters["name"], filters.get("requirements", filters.get("version"))
-        )
-        if not package_dir:
-            return None
-        manifest = self.load_manifest(package_dir)
-        if "id" not in manifest:
-            return None
-
-        for key in ("frameworks", "platforms"):
-            if key not in filters:
-                continue
-            if key not in manifest:
-                return None
-            if not util.items_in_list(
-                util.items_to_list(filters[key]), util.items_to_list(manifest[key])
-            ):
-                return None
-
-        if "authors" in filters:
-            if "authors" not in manifest:
-                return None
-            manifest_authors = manifest["authors"]
-            if not isinstance(manifest_authors, list):
-                manifest_authors = [manifest_authors]
-            manifest_authors = [
-                a["name"]
-                for a in manifest_authors
-                if isinstance(a, dict) and "name" in a
-            ]
-            filter_authors = filters["authors"]
-            if not isinstance(filter_authors, list):
-                filter_authors = [filter_authors]
-            if not set(filter_authors) <= set(manifest_authors):
-                return None
-
-        return int(manifest["id"])
-
-    def install(  # pylint: disable=arguments-differ
-        self,
-        name,
-        requirements=None,
-        silent=False,
-        after_update=False,
-        interactive=False,
-        force=False,
-    ):
-        _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
-        if not _url:
-            name = "id=%d" % self.search_lib_id(
-                {"name": _name, "requirements": _requirements},
-                silent=silent,
-                interactive=interactive,
-            )
-            requirements = _requirements
-        pkg_dir = BasePkgManager.install(
-            self,
-            name,
-            requirements,
-            silent=silent,
-            after_update=after_update,
-            force=force,
-        )
-
-        if not pkg_dir:
-            return None
-
-        manifest = None
-        try:
-            manifest = ManifestParserFactory.new_from_dir(pkg_dir).as_dict()
-        except ManifestException:
-            pass
-        if not manifest or not manifest.get("dependencies"):
-            return pkg_dir
-
-        if not silent:
-            click.secho("Installing dependencies", fg="yellow")
-
-        builtin_lib_storages = None
-        for filters in manifest["dependencies"]:
-            assert "name" in filters
-
-            # avoid circle dependencies
-            if not self.INSTALL_HISTORY:
-                self.INSTALL_HISTORY = []
-            history_key = str(filters)
-            if history_key in self.INSTALL_HISTORY:
-                continue
-            self.INSTALL_HISTORY.append(history_key)
-
-            if any(s in filters.get("version", "") for s in ("\\", "/")):
-                self.install(
-                    "{name}={version}".format(**filters),
-                    silent=silent,
-                    after_update=after_update,
-                    interactive=interactive,
-                    force=force,
-                )
-            else:
-                try:
-                    lib_id = self.search_lib_id(filters, silent, interactive)
-                except exception.LibNotFound as e:
-                    if builtin_lib_storages is None:
-                        builtin_lib_storages = get_builtin_libs()
-                    if not silent or is_builtin_lib(
-                        builtin_lib_storages, filters["name"]
-                    ):
-                        click.secho("Warning! %s" % e, fg="yellow")
-                    continue
-
-                if filters.get("version"):
-                    self.install(
-                        lib_id,
-                        filters.get("version"),
-                        silent=silent,
-                        after_update=after_update,
-                        interactive=interactive,
-                        force=force,
-                    )
-                else:
-                    self.install(
-                        lib_id,
-                        silent=silent,
-                        after_update=after_update,
-                        interactive=interactive,
-                        force=force,
-                    )
-        return pkg_dir
-
-
-def get_builtin_libs(storage_names=None):
-    items = []
-    storage_names = storage_names or []
-    pm = PlatformManager()
-    for manifest in pm.get_installed():
-        p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
-        for storage in p.get_lib_storages():
-            if storage_names and storage["name"] not in storage_names:
-                continue
-            lm = LibraryManager(storage["path"])
-            items.append(
-                {
-                    "name": storage["name"],
-                    "path": storage["path"],
-                    "items": lm.get_installed(),
-                }
-            )
-    return items
-
-
-def is_builtin_lib(storages, name):
-    for storage in storages or []:
-        if any(l.get("name") == name for l in storage["items"]):
-            return True
-    return False

platformio/managers/package.py

@@ -482,7 +482,7 @@ class PkgInstallerMixin(object):
                 self.unpack(dlpath, tmp_dir)
                 os.remove(dlpath)
             else:
-                vcs = VCSClientFactory.newClient(tmp_dir, url)
+                vcs = VCSClientFactory.new(tmp_dir, url)
                 assert vcs.export()
                 src_manifest_dir = vcs.storage_dir
                 src_manifest["version"] = vcs.get_current_revision()
@@ -628,9 +628,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
         if "__src_url" in manifest:
             try:
-                vcs = VCSClientFactory.newClient(
+                vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"], silent=True)
-                    pkg_dir, manifest["__src_url"], silent=True
-                )
             except (AttributeError, exception.PlatformioException):
                 return None
             if not vcs.can_be_updated:
@@ -800,7 +798,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             return True
 
         if "__src_url" in manifest:
-            vcs = VCSClientFactory.newClient(pkg_dir, manifest["__src_url"])
+            vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"])
             assert vcs.update()
             self._update_src_manifest(
                 dict(version=vcs.get_current_revision()), vcs.storage_dir

platformio/package/manager/_install.py

@@ -20,7 +20,7 @@ import tempfile
 import click
 
 from platformio import app, compat, fs, util
-from platformio.package.exception import PackageException
+from platformio.package.exception import MissingPackageManifestError, PackageException
 from platformio.package.meta import PackageSourceItem, PackageSpec
 from platformio.package.unpack import FileUnpacker
 from platformio.package.vcsclient import VCSClientFactory
@@ -83,7 +83,7 @@ class PackageManagerInstallMixin(object):
             msg = "Installing %s" % click.style(spec.humanize(), fg="cyan")
             self.print_message(msg)
 
-        if spec.url:
+        if spec.external:
             pkg = self.install_from_url(spec.url, spec, silent=silent)
         else:
             pkg = self.install_from_registry(spec, search_filters, silent=silent)
@@ -152,7 +152,7 @@ class PackageManagerInstallMixin(object):
                 assert os.path.isfile(dl_path)
                 self.unpack(dl_path, tmp_dir)
             else:
-                vcs = VCSClientFactory.newClient(tmp_dir, url)
+                vcs = VCSClientFactory.new(tmp_dir, url)
                 assert vcs.export()
 
             root_dir = self.find_pkg_root(tmp_dir, spec)
@@ -189,12 +189,20 @@ class PackageManagerInstallMixin(object):
 
         # what to do with existing package?
         action = "overwrite"
-        if dst_pkg.metadata and dst_pkg.metadata.spec.url:
+        if tmp_pkg.metadata.spec.has_custom_name():
+            action = "overwrite"
+            dst_pkg = PackageSourceItem(
+                os.path.join(self.package_dir, tmp_pkg.metadata.spec.name)
+            )
+        elif dst_pkg.metadata and dst_pkg.metadata.spec.external:
             if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url:
                 action = "detach-existing"
-        elif tmp_pkg.metadata.spec.url:
+        elif tmp_pkg.metadata.spec.external:
             action = "detach-new"
-        elif dst_pkg.metadata and dst_pkg.metadata.version != tmp_pkg.metadata.version:
+        elif dst_pkg.metadata and (
+            dst_pkg.metadata.version != tmp_pkg.metadata.version
+            or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner
+        ):
             action = (
                 "detach-existing"
                 if tmp_pkg.metadata.version > dst_pkg.metadata.version
@@ -231,7 +239,7 @@ class PackageManagerInstallMixin(object):
                 tmp_pkg.get_safe_dirname(),
                 tmp_pkg.metadata.version,
             )
-            if tmp_pkg.metadata.spec.url:
+            if tmp_pkg.metadata.spec.external:
                 target_dirname = "%s@src-%s" % (
                     tmp_pkg.get_safe_dirname(),
                     hashlib.md5(
@@ -247,3 +255,20 @@ class PackageManagerInstallMixin(object):
         _cleanup_dir(dst_pkg.path)
         shutil.move(tmp_pkg.path, dst_pkg.path)
         return PackageSourceItem(dst_pkg.path)
+
+    def get_installed(self):
+        result = []
+        for name in os.listdir(self.package_dir):
+            pkg_dir = os.path.join(self.package_dir, name)
+            if not os.path.isdir(pkg_dir):
+                continue
+            pkg = PackageSourceItem(pkg_dir)
+            if not pkg.metadata:
+                try:
+                    spec = self.build_legacy_spec(pkg_dir)
+                    pkg.metadata = self.build_metadata(pkg_dir, spec)
+                except MissingPackageManifestError:
+                    pass
+            if pkg.metadata:
+                result.append(pkg)
+        return result

platformio/package/manager/_legacy.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from platformio import fs
+from platformio.package.meta import PackageSourceItem, PackageSpec
+
+
+class PackageManagerLegacyMixin(object):
+    def build_legacy_spec(self, pkg_dir):
+        # find src manifest
+        src_manifest_name = ".piopkgmanager.json"
+        src_manifest_path = None
+        for name in os.listdir(pkg_dir):
+            if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)):
+                continue
+            src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name)
+            break
+
+        if src_manifest_path:
+            src_manifest = fs.load_json(src_manifest_path)
+            return PackageSpec(
+                name=src_manifest.get("name"),
+                url=src_manifest.get("url"),
+                requirements=src_manifest.get("requirements"),
+            )
+
+        # fall back to a package manifest
+        manifest = self.load_manifest(pkg_dir)
+        return PackageSpec(name=manifest.get("name"))
+
+    def legacy_load_manifest(self, pkg):
+        assert isinstance(pkg, PackageSourceItem)
+        manifest = self.load_manifest(pkg)
+        manifest["__pkg_dir"] = pkg.path
+        for key in ("name", "version"):
+            if not manifest.get(key):
+                manifest[key] = str(getattr(pkg.metadata, key))
+        if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external:
+            manifest["__src_url"] = pkg.metadata.spec.url
+            manifest["version"] = str(pkg.metadata.version)
+        return manifest
+
+    def legacy_get_installed(self):
+        return [self.legacy_load_manifest(pkg) for pkg in self.get_installed()]

platformio/package/manager/_registry.py

@@ -79,10 +79,10 @@ class RegistryFileMirrorsIterator(object):
 class PackageManageRegistryMixin(object):
     def install_from_registry(self, spec, search_filters=None, silent=False):
         if spec.owner and spec.name and not search_filters:
-            package = self.fetch_registry_package(spec.owner, spec.name)
+            package = self.fetch_registry_package(spec)
             if not package:
                 raise UnknownPackageError(spec.humanize())
-            version = self._pick_best_pkg_version(package["versions"], spec)
+            version = self.pick_best_registry_version(package["versions"], spec)
         else:
             packages = self.search_registry_packages(spec, search_filters)
             if not packages:
@@ -131,10 +131,33 @@ class PackageManageRegistryMixin(object):
             "items"
         ]
 
-    def fetch_registry_package(self, owner, name):
+    def fetch_registry_package(self, spec):
-        return self.get_registry_client_instance().get_package(
+        result = None
-            self.pkg_type, owner, name
+        if spec.owner and spec.name:
+            result = self.get_registry_client_instance().get_package(
+                self.pkg_type, spec.owner, spec.name
             )
+        if not result and (spec.id or (spec.name and not spec.owner)):
+            packages = self.search_registry_packages(spec)
+            if packages:
+                result = self.get_registry_client_instance().get_package(
+                    self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"]
+                )
+        if not result:
+            raise UnknownPackageError(spec.humanize())
+        return result
+
+    def reveal_registry_package_id(self, spec, silent=False):
+        spec = self.ensure_spec(spec)
+        if spec.id:
+            return spec.id
+        packages = self.search_registry_packages(spec)
+        if not packages:
+            raise UnknownPackageError(spec.humanize())
+        if len(packages) > 1 and not silent:
+            self.print_multi_package_issue(packages, spec)
+            click.echo("")
+        return packages[0]["id"]
 
     @staticmethod
     def print_multi_package_issue(packages, spec):
@@ -160,7 +183,7 @@ class PackageManageRegistryMixin(object):
     def find_best_registry_version(self, packages, spec):
         # find compatible version within the latest package versions
         for package in packages:
-            version = self._pick_best_pkg_version([package["version"]], spec)
+            version = self.pick_best_registry_version([package["version"]], spec)
             if version:
                 return (package, version)
 
@@ -169,9 +192,13 @@ class PackageManageRegistryMixin(object):
 
         # if the custom version requirements, check ALL package versions
         for package in packages:
-            version = self._pick_best_pkg_version(
+            version = self.pick_best_registry_version(
                 self.fetch_registry_package(
-                    package["owner"]["username"], package["name"]
+                    PackageSpec(
+                        id=package["id"],
+                        owner=package["owner"]["username"],
+                        name=package["name"],
+                    )
                 ).get("versions"),
                 spec,
             )
@@ -180,11 +207,12 @@ class PackageManageRegistryMixin(object):
             time.sleep(1)
         return None
 
-    def _pick_best_pkg_version(self, versions, spec):
+    def pick_best_registry_version(self, versions, spec=None):
+        assert not spec or isinstance(spec, PackageSpec)
         best = None
         for version in versions:
             semver = PackageMetaData.to_semver(version["name"])
-            if spec.requirements and semver not in spec.requirements:
+            if spec and spec.requirements and semver not in spec.requirements:
                 continue
             if not any(
                 self.is_system_compatible(f.get("system")) for f in version["files"]

platformio/package/manager/_uninstall.py

@@ -31,10 +31,7 @@ class PackageManagerUninstallMixin(object):
             self.unlock()
 
     def _uninstall(self, pkg, silent=False, skip_dependencies=False):
-        if not isinstance(pkg, PackageSourceItem):
+        pkg = self.get_package(pkg)
-            pkg = (
-                PackageSourceItem(pkg) if os.path.isdir(pkg) else self.get_package(pkg)
-            )
         if not pkg or not pkg.metadata:
             raise UnknownPackageError(pkg)
 
@@ -73,7 +70,7 @@ class PackageManagerUninstallMixin(object):
         if not silent:
             click.echo("[%s]" % click.style("OK", fg="green"))
 
-        return True
+        return pkg
 
     def _uninstall_dependencies(self, pkg, silent=False):
         assert isinstance(pkg, PackageSourceItem)

platformio/package/manager/_update.py

@@ -0,0 +1,166 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import click
+
+from platformio import util
+from platformio.package.exception import UnknownPackageError
+from platformio.package.meta import (
+    PackageOutdatedResult,
+    PackageSourceItem,
+    PackageSpec,
+)
+from platformio.package.vcsclient import VCSBaseException, VCSClientFactory
+
+
+class PackageManagerUpdateMixin(object):
+    def outdated(self, pkg, spec=None):
+        assert isinstance(pkg, PackageSourceItem)
+        assert not spec or isinstance(spec, PackageSpec)
+        assert os.path.isdir(pkg.path) and pkg.metadata
+
+        # skip detached package to a specific version
+        detached_conditions = [
+            "@" in pkg.path,
+            pkg.metadata.spec and not pkg.metadata.spec.external,
+            not spec,
+        ]
+        if all(detached_conditions):
+            return PackageOutdatedResult(current=pkg.metadata.version, detached=True)
+
+        latest = None
+        wanted = None
+        if pkg.metadata.spec.external:
+            latest = self._fetch_vcs_latest_version(pkg)
+        else:
+            try:
+                reg_pkg = self.fetch_registry_package(pkg.metadata.spec)
+                latest = (
+                    self.pick_best_registry_version(reg_pkg["versions"]) or {}
+                ).get("name")
+                if spec:
+                    wanted = (
+                        self.pick_best_registry_version(reg_pkg["versions"], spec) or {}
+                    ).get("name")
+                    if not wanted:  # wrong library
+                        latest = None
+            except UnknownPackageError:
+                pass
+
+        return PackageOutdatedResult(
+            current=pkg.metadata.version, latest=latest, wanted=wanted
+        )
+
+    def _fetch_vcs_latest_version(self, pkg):
+        vcs = None
+        try:
+            vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url, silent=True)
+        except VCSBaseException:
+            return None
+        if not vcs.can_be_updated:
+            return None
+        return str(
+            self.build_metadata(
+                pkg.path, pkg.metadata.spec, vcs_revision=vcs.get_latest_revision()
+            ).version
+        )
+
+    def update(self, pkg, spec=None, only_check=False, silent=False):
+        pkg = self.get_package(pkg)
+        if not pkg or not pkg.metadata:
+            raise UnknownPackageError(pkg)
+
+        if not silent:
+            click.echo(
+                "{} {:<45} {:<30}".format(
+                    "Checking" if only_check else "Updating",
+                    click.style(pkg.metadata.spec.humanize(), fg="cyan"),
+                    "%s (%s)" % (pkg.metadata.version, spec.requirements)
+                    if spec and spec.requirements
+                    else str(pkg.metadata.version),
+                ),
+                nl=False,
+            )
+        if not util.internet_on():
+            if not silent:
+                click.echo("[%s]" % (click.style("Off-line", fg="yellow")))
+            return pkg
+
+        outdated = self.outdated(pkg, spec)
+        if not silent:
+            self.print_outdated_state(outdated)
+
+        up_to_date = any(
+            [
+                outdated.detached,
+                not outdated.latest,
+                outdated.latest and outdated.current == outdated.latest,
+                outdated.wanted and outdated.current == outdated.wanted,
+            ]
+        )
+        if only_check or up_to_date:
+            return pkg
+
+        try:
+            self.lock()
+            return self._update(pkg, outdated, silent=silent)
+        finally:
+            self.unlock()
+
+    @staticmethod
+    def print_outdated_state(outdated):
+        if outdated.detached:
+            return click.echo("[%s]" % (click.style("Detached", fg="yellow")))
+        if not outdated.latest or outdated.current == outdated.latest:
+            return click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
+        if outdated.wanted and outdated.current == outdated.wanted:
+            return click.echo(
+                "[%s]"
+                % (click.style("Incompatible (%s)" % outdated.latest, fg="yellow"))
+            )
+        return click.echo(
+            "[%s]" % (click.style(str(outdated.wanted or outdated.latest), fg="red"))
+        )
+
+    def _update(self, pkg, outdated, silent=False):
+        if pkg.metadata.spec.external:
+            vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url)
+            assert vcs.update()
+            pkg.metadata.version = self._fetch_vcs_latest_version(pkg)
+            pkg.dump_meta()
+            return pkg
+
+        new_pkg = self.install(
+            PackageSpec(
+                id=pkg.metadata.spec.id,
+                owner=pkg.metadata.spec.owner,
+                name=pkg.metadata.spec.name,
+                requirements=outdated.wanted or outdated.latest,
+            ),
+            silent=silent,
+        )
+        if new_pkg:
+            old_pkg = self.get_package(
+                PackageSpec(
+                    id=pkg.metadata.spec.id,
+                    owner=pkg.metadata.spec.owner,
+                    name=pkg.metadata.name,
+                    requirements=pkg.metadata.version,
+                )
+            )
+            if old_pkg:
+                self.uninstall(old_pkg, silent=silent, skip_dependencies=True)
+        return new_pkg

platformio/package/manager/base.py

@@ -18,14 +18,17 @@ from datetime import datetime
 import click
 import semantic_version
 
-from platformio import fs, util
+from platformio import util
 from platformio.commands import PlatformioCLI
+from platformio.compat import ci_strings_are_equal
 from platformio.package.exception import ManifestException, MissingPackageManifestError
 from platformio.package.lockfile import LockFile
 from platformio.package.manager._download import PackageManagerDownloadMixin
 from platformio.package.manager._install import PackageManagerInstallMixin
+from platformio.package.manager._legacy import PackageManagerLegacyMixin
 from platformio.package.manager._registry import PackageManageRegistryMixin
 from platformio.package.manager._uninstall import PackageManagerUninstallMixin
+from platformio.package.manager._update import PackageManagerUpdateMixin
 from platformio.package.manifest.parser import ManifestParserFactory
 from platformio.package.meta import (
     PackageMetaData,
@@ -41,6 +44,8 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
     PackageManageRegistryMixin,
     PackageManagerInstallMixin,
     PackageManagerUninstallMixin,
+    PackageManagerUpdateMixin,
+    PackageManagerLegacyMixin,
 ):
     _MEMORY_CACHE = {}
 
@@ -83,10 +88,6 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
             return True
         return util.items_in_list(value, util.get_systype())
 
-    @staticmethod
-    def generate_rand_version():
-        return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S")
-
     @staticmethod
     def ensure_dir_exists(path):
         if not os.path.isdir(path):
@@ -162,27 +163,9 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
                     click.secho(str(e), fg="yellow")
         raise MissingPackageManifestError(", ".join(self.manifest_names))
 
-    def build_legacy_spec(self, pkg_dir):
+    @staticmethod
-        # find src manifest
+    def generate_rand_version():
-        src_manifest_name = ".piopkgmanager.json"
+        return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S")
-        src_manifest_path = None
-        for name in os.listdir(pkg_dir):
-            if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)):
-                continue
-            src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name)
-            break
-
-        if src_manifest_path:
-            src_manifest = fs.load_json(src_manifest_path)
-            return PackageSpec(
-                name=src_manifest.get("name"),
-                url=src_manifest.get("url"),
-                requirements=src_manifest.get("requirements"),
-            )
-
-        # fall back to a package manifest
-        manifest = self.load_manifest(pkg_dir)
-        return PackageSpec(name=manifest.get("name"))
 
     def build_metadata(self, pkg_dir, spec, vcs_revision=None):
         manifest = self.load_manifest(pkg_dir)
@@ -192,7 +175,7 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
             version=manifest.get("version"),
             spec=spec,
         )
-        if not metadata.name or spec.is_custom_name():
+        if not metadata.name or spec.has_custom_name():
             metadata.name = spec.name
         if vcs_revision:
             metadata.version = "%s+sha.%s" % (
@@ -203,42 +186,27 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
             metadata.version = self.generate_rand_version()
         return metadata
 
-    def get_installed(self):
-        result = []
-        for name in os.listdir(self.package_dir):
-            pkg_dir = os.path.join(self.package_dir, name)
-            if not os.path.isdir(pkg_dir):
-                continue
-            pkg = PackageSourceItem(pkg_dir)
-            if not pkg.metadata:
-                try:
-                    spec = self.build_legacy_spec(pkg_dir)
-                    pkg.metadata = self.build_metadata(pkg_dir, spec)
-                except MissingPackageManifestError:
-                    pass
-            if pkg.metadata:
-                result.append(pkg)
-        return result
-
     def get_package(self, spec):
-        def _ci_strings_are_equal(a, b):
+        if isinstance(spec, PackageSourceItem):
-            if a == b:
+            return spec
-                return True
+
-            if not a or not b:
+        if not isinstance(spec, PackageSpec) and os.path.isdir(spec):
-                return False
+            for pkg in self.get_installed():
-            return a.strip().lower() == b.strip().lower()
+                if spec == pkg.path:
+                    return pkg
+            return None
 
         spec = self.ensure_spec(spec)
         best = None
         for pkg in self.get_installed():
             skip_conditions = [
                 spec.owner
-                and not _ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner),
+                and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner),
-                spec.url and spec.url != pkg.metadata.spec.url,
+                spec.external and spec.url != pkg.metadata.spec.url,
                 spec.id and spec.id != pkg.metadata.spec.id,
                 not spec.id
-                and not spec.url
+                and not spec.external
-                and not _ci_strings_are_equal(spec.name, pkg.metadata.name),
+                and not ci_strings_are_equal(spec.name, pkg.metadata.name),
             ]
             if any(skip_conditions):
                 continue

platformio/package/manager/library.py

@@ -21,7 +21,7 @@ from platformio.package.meta import PackageSpec, PackageType
 from platformio.project.helpers import get_project_global_lib_dir
 
 
-class LibraryPackageManager(BasePackageManager):
+class LibraryPackageManager(BasePackageManager):  # pylint: disable=too-many-ancestors
     def __init__(self, package_dir=None):
         super(LibraryPackageManager, self).__init__(
             PackageType.LIBRARY, package_dir or get_project_global_lib_dir()

platformio/package/manager/platform.py

@@ -17,7 +17,7 @@ from platformio.package.meta import PackageType
 from platformio.project.config import ProjectConfig
 
 
-class PlatformPackageManager(BasePackageManager):
+class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-ancestors
     def __init__(self, package_dir=None):
         self.config = ProjectConfig.get_instance()
         super(PlatformPackageManager, self).__init__(

platformio/package/manager/tool.py

@@ -17,7 +17,7 @@ from platformio.package.meta import PackageType
 from platformio.project.config import ProjectConfig
 
 
-class ToolPackageManager(BasePackageManager):
+class ToolPackageManager(BasePackageManager):  # pylint: disable=too-many-ancestors
     def __init__(self, package_dir=None):
         self.config = ProjectConfig.get_instance()
         super(ToolPackageManager, self).__init__(

platformio/package/manifest/schema.py

@@ -250,7 +250,7 @@ class ManifestSchema(BaseSchema):
     def load_spdx_licenses():
         r = requests.get(
             "https://raw.githubusercontent.com/spdx/license-list-data"
-            "/v3.9/json/licenses.json"
+            "/v3.10/json/licenses.json"
         )
         r.raise_for_status()
         return r.json()

platformio/package/meta.py

@@ -65,7 +65,44 @@ class PackageType(object):
         return None
 
 
-class PackageSpec(object):
+class PackageOutdatedResult(object):
+    def __init__(self, current, latest=None, wanted=None, detached=False):
+        self.current = current
+        self.latest = latest
+        self.wanted = wanted
+        self.detached = detached
+
+    def __repr__(self):
+        return (
+            "PackageOutdatedResult <current={current} latest={latest} wanted={wanted} "
+            "detached={detached}>".format(
+                current=self.current,
+                latest=self.latest,
+                wanted=self.wanted,
+                detached=self.detached,
+            )
+        )
+
+    def __setattr__(self, name, value):
+        if (
+            value
+            and name in ("current", "latest", "wanted")
+            and not isinstance(value, semantic_version.Version)
+        ):
+            value = semantic_version.Version(str(value))
+        return super(PackageOutdatedResult, self).__setattr__(name, value)
+
+    def is_outdated(self, allow_incompatible=False):
+        if self.detached or not self.latest or self.current == self.latest:
+            return False
+        if allow_incompatible:
+            return self.current != self.latest
+        if self.wanted:
+            return self.current != self.wanted
+        return True
+
+
+class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
     def __init__(  # pylint: disable=redefined-builtin,too-many-arguments
         self, raw=None, owner=None, id=None, name=None, requirements=None, url=None
     ):
@@ -74,6 +111,7 @@ class PackageSpec(object):
         self.name = name
         self._requirements = None
         self.url = url
+        self.raw = raw
         if requirements:
             self.requirements = requirements
         self._name_is_custom = False
@@ -104,6 +142,10 @@ class PackageSpec(object):
             "requirements={requirements} url={url}>".format(**self.as_dict())
         )
 
+    @property
+    def external(self):
+        return bool(self.url)
+
     @property
     def requirements(self):
         return self._requirements
@@ -116,24 +158,24 @@ class PackageSpec(object):
         self._requirements = (
             value
             if isinstance(value, semantic_version.SimpleSpec)
-            else semantic_version.SimpleSpec(value)
+            else semantic_version.SimpleSpec(str(value))
         )
 
     def humanize(self):
+        result = ""
         if self.url:
             result = self.url
-        elif self.id:
+        elif self.name:
-            result = "id:%d" % self.id
-        else:
-            result = ""
             if self.owner:
                 result = self.owner + "/"
             result += self.name
+        elif self.id:
+            result = "id:%d" % self.id
         if self.requirements:
             result += " @ " + str(self.requirements)
         return result
 
-    def is_custom_name(self):
+    def has_custom_name(self):
         return self._name_is_custom
 
     def as_dict(self):
@@ -145,6 +187,19 @@ class PackageSpec(object):
             url=self.url,
         )
 
+    def as_dependency(self):
+        if self.url:
+            return self.raw or self.url
+        result = ""
+        if self.name:
+            result = "%s/%s" % (self.owner, self.name) if self.owner else self.name
+        elif self.id:
+            result = str(self.id)
+        assert result
+        if self.requirements:
+            result = "%s@%s" % (result, self.requirements)
+        return result
+
     def _parse(self, raw):
         if raw is None:
             return

platformio/package/vcsclient.py

@@ -17,7 +17,11 @@ from os.path import join
 from subprocess import CalledProcessError, check_call
 from sys import modules
 
-from platformio.exception import PlatformioException, UserSideException
+from platformio.package.exception import (
+    PackageException,
+    PlatformioException,
+    UserSideException,
+)
 from platformio.proc import exec_command
 
 try:
@@ -26,9 +30,13 @@ except ImportError:
     from urlparse import urlparse
 
 
+class VCSBaseException(PackageException):
+    pass
+
+
 class VCSClientFactory(object):
     @staticmethod
-    def newClient(src_dir, remote_url, silent=False):
+    def new(src_dir, remote_url, silent=False):
         result = urlparse(remote_url)
         type_ = result.scheme
         tag = None
@@ -41,12 +49,15 @@ class VCSClientFactory(object):
         if "#" in remote_url:
             remote_url, tag = remote_url.rsplit("#", 1)
         if not type_:
-            raise PlatformioException("VCS: Unknown repository type %s" % remote_url)
+            raise VCSBaseException("VCS: Unknown repository type %s" % remote_url)
+        try:
             obj = getattr(modules[__name__], "%sClient" % type_.title())(
                 src_dir, remote_url, tag, silent
             )
             assert isinstance(obj, VCSClientBase)
             return obj
+        except (AttributeError, AssertionError):
+            raise VCSBaseException("VCS: Unknown repository type %s" % remote_url)
 
 
 class VCSClientBase(object):
@@ -101,7 +112,7 @@ class VCSClientBase(object):
             check_call(args, **kwargs)
             return True
         except CalledProcessError as e:
-            raise PlatformioException("VCS: Could not process command %s" % e.cmd)
+            raise VCSBaseException("VCS: Could not process command %s" % e.cmd)
 
     def get_cmd_output(self, args, **kwargs):
         args = [self.command] + args
@@ -110,7 +121,7 @@ class VCSClientBase(object):
         result = exec_command(args, **kwargs)
         if result["returncode"] == 0:
             return result["out"].strip()
-        raise PlatformioException(
+        raise VCSBaseException(
             "VCS: Could not receive an output from `%s` command (%s)" % (args, result)
         )
 
@@ -227,7 +238,6 @@ class SvnClient(VCSClientBase):
         return self.run_cmd(args)
 
     def update(self):
-
         args = ["update"]
         return self.run_cmd(args)
 
@@ -239,4 +249,4 @@ class SvnClient(VCSClientBase):
             line = line.strip()
             if line.startswith("Revision:"):
                 return line.split(":", 1)[1].strip()
-        raise PlatformioException("Could not detect current SVN revision")
+        raise VCSBaseException("Could not detect current SVN revision")

tests/commands/test_ci.py

@@ -15,7 +15,7 @@
 from os.path import isfile, join
 
 from platformio.commands.ci import cli as cmd_ci
-from platformio.commands.lib import cli as cmd_lib
+from platformio.commands.lib.command import cli as cmd_lib
 
 
 def test_ci_empty(clirunner):

tests/commands/test_lib.py

@@ -13,332 +13,184 @@
 # limitations under the License.
 
 import json
-import re
+import os
 
-from platformio import exception
+import semantic_version
-from platformio.commands import PlatformioCLI
-from platformio.commands.lib import cli as cmd_lib
 
-PlatformioCLI.leftover_args = ["--json-output"]  # hook for click
+from platformio.clients.registry import RegistryClient
+from platformio.commands.lib.command import cli as cmd_lib
+from platformio.package.meta import PackageType
+from platformio.package.vcsclient import VCSClientFactory
+from platformio.project.config import ProjectConfig
 
 
-def test_search(clirunner, validate_cliresult):
+def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory):
-    result = clirunner.invoke(cmd_lib, ["search", "DHT22"])
+    regclient = RegistryClient()
+    project_dir = tmpdir_factory.mktemp("project")
+    project_dir.join("platformio.ini").write(
+        """
+[env]
+lib_deps = ArduinoJson
+
+[env:one]
+board = devkit
+
+[env:two]
+framework = foo
+lib_deps =
+    CustomLib
+    ArduinoJson @ 5.10.1
+"""
+    )
+    result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "install", "64"])
     validate_cliresult(result)
-    match = re.search(r"Found\s+(\d+)\slibraries:", result.output)
+    aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson")
-    assert int(match.group(1)) > 2
+    config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini"))
+    assert config.get("env:one", "lib_deps") == [
+        "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"]
+    ]
+    assert config.get("env:two", "lib_deps") == [
+        "CustomLib",
+        "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"],
+    ]
 
-    result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"])
+    # ensure "build" version without NPM spec
+    result = clirunner.invoke(
+        cmd_lib,
+        ["-d", str(project_dir), "-e", "one", "install", "mbed-sam-grove/LinkedList"],
+    )
     validate_cliresult(result)
-    match = re.search(r"Found\s+(\d+)\slibraries:", result.output)
+    ll_pkg_data = regclient.get_package(
-    assert int(match.group(1)) > 1
+        PackageType.LIBRARY, "mbed-sam-grove", "LinkedList"
+    )
+    config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini"))
+    assert config.get("env:one", "lib_deps") == [
+        "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"],
+        "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"],
+    ]
 
-
+    # check external package via Git repo
-def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core):
     result = clirunner.invoke(
         cmd_lib,
         [
-            "-g",
+            "-d",
+            str(project_dir),
+            "-e",
+            "one",
             "install",
-            "64",
+            "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3",
-            "ArduinoJson@~5.10.0",
-            "547@2.2.4",
-            "AsyncMqttClient@<=0.8.2",
-            "Adafruit PN532@1.2.0",
         ],
     )
     validate_cliresult(result)
+    config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini"))
+    assert len(config.get("env:one", "lib_deps")) == 3
+    assert config.get("env:one", "lib_deps")[2] == (
+        "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3"
+    )
 
-    # install unknown library
+    # test uninstalling
-    result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"])
+    result = clirunner.invoke(
-    assert result.exit_code != 0
+        cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"]
-    assert isinstance(result.exception, exception.LibNotFound)
+    )
-
+    validate_cliresult(result)
-    items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
+    config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini"))
-    items2 = [
+    assert len(config.get("env:one", "lib_deps")) == 2
-        "ArduinoJson",
+    assert len(config.get("env:two", "lib_deps")) == 1
-        "ArduinoJson@5.10.1",
+    assert config.get("env:one", "lib_deps") == [
-        "NeoPixelBus",
+        "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"],
-        "AsyncMqttClient",
+        "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3",
-        "ESPAsyncTCP",
-        "AsyncTCP",
-        "Adafruit PN532",
-        "Adafruit BusIO",
     ]
-    assert set(items1) == set(items2)
+
+    # test list
+    result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"])
+    validate_cliresult(result)
+    assert "Version: 0.8.3+sha." in result.stdout
+    assert (
+        "Source: git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3"
+        in result.stdout
+    )
+    result = clirunner.invoke(
+        cmd_lib, ["-d", str(project_dir), "list", "--json-output"]
+    )
+    validate_cliresult(result)
+    data = {}
+    for key, value in json.loads(result.stdout).items():
+        data[os.path.basename(key)] = value
+    ame_lib = next(
+        item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome"
+    )
+    ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"])
+    assert data["two"] == []
+    assert "__pkg_dir" in data["one"][0]
+    assert (
+        ame_lib["__src_url"]
+        == "git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3"
+    )
+    assert ame_lib["version"] == ("0.8.3+sha.%s" % ame_vcs.get_current_revision())
 
 
-def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core):
+def test_update(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory):
+    storage_dir = tmpdir_factory.mktemp("test-updates")
+    result = clirunner.invoke(
+        cmd_lib,
+        ["-d", str(storage_dir), "install", "ArduinoJson @ 5.10.1", "Blynk @ ~0.5.0"],
+    )
+    validate_cliresult(result)
+    result = clirunner.invoke(
+        cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "--json-output"]
+    )
+    validate_cliresult(result)
+    outdated = json.loads(result.stdout)
+    assert len(outdated) == 2
+    # ArduinoJson
+    assert outdated[0]["version"] == "5.10.1"
+    assert outdated[0]["versionWanted"] is None
+    assert semantic_version.Version(
+        outdated[0]["versionLatest"]
+    ) > semantic_version.Version("6.16.0")
+    # Blynk
+    assert outdated[1]["version"] == "0.5.4"
+    assert outdated[1]["versionWanted"] is None
+    assert semantic_version.Version(
+        outdated[1]["versionLatest"]
+    ) > semantic_version.Version("0.6.0")
+
+    # check with spec
     result = clirunner.invoke(
         cmd_lib,
         [
-            "-g",
+            "-d",
-            "install",
+            str(storage_dir),
-            "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip",
+            "update",
-            "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2",
+            "--dry-run",
-            "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz",
+            "--json-output",
-            "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
+            "ArduinoJson @ ^5",
         ],
     )
     validate_cliresult(result)
-
+    outdated = json.loads(result.stdout)
-    # incorrect requirements
+    assert outdated[0]["version"] == "5.10.1"
+    assert outdated[0]["versionWanted"] == "5.13.4"
+    assert semantic_version.Version(
+        outdated[0]["versionLatest"]
+    ) > semantic_version.Version("6.16.0")
+    # update with spec
     result = clirunner.invoke(
-        cmd_lib,
+        cmd_lib, ["-d", str(storage_dir), "update", "--silent", "ArduinoJson @ ^5.10.1"]
-        [
-            "-g",
-            "install",
-            "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3",
-        ],
     )
-    assert result.exit_code != 0
+    validate_cliresult(result)
-
-    items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
-    items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire", "ESP32WebServer"]
-    assert set(items1) >= set(items2)
-
-
-def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core):
     result = clirunner.invoke(
-        cmd_lib,
+        cmd_lib, ["-d", str(storage_dir), "list", "--json-output"]
-        [
-            "-g",
-            "install",
-            "https://github.com/gioblu/PJON.git#3.0",
-            "https://github.com/gioblu/PJON.git#6.2",
-            "https://github.com/bblanchon/ArduinoJson.git",
-            "https://gitlab.com/ivankravets/rs485-nodeproto.git",
-            "https://github.com/platformio/platformio-libmirror.git",
-            # "https://developer.mbed.org/users/simon/code/TextLCD/",
-            "knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163",
-        ],
     )
     validate_cliresult(result)
-    items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
+    items = json.loads(result.stdout)
-    items2 = [
+    assert len(items) == 2
-        "PJON",
+    assert items[0]["version"] == "5.13.4"
-        "PJON@src-79de467ebe19de18287becff0a1fb42d",
+    assert items[1]["version"] == "0.5.4"
-        "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
-        "rs485-nodeproto",
-        "platformio-libmirror",
-        "PubSubClient",
-    ]
-    assert set(items1) >= set(items2)
 
-
+    # Check incompatible
-def test_install_duplicates(clirunner, validate_cliresult, without_internet):
-    # registry
     result = clirunner.invoke(
-        cmd_lib,
+        cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "ArduinoJson @ ^5"]
-        ["-g", "install", "http://dl.platformio.org/libraries/archives/0/9540.tar.gz"],
     )
     validate_cliresult(result)
-    assert "is already installed" in result.output
+    assert "Incompatible" in result.stdout
-
-    # archive
-    result = clirunner.invoke(
-        cmd_lib,
-        [
-            "-g",
-            "install",
-            "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
-        ],
-    )
-    validate_cliresult(result)
-    assert "is already installed" in result.output
-
-    # repository
-    result = clirunner.invoke(
-        cmd_lib,
-        ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"],
-    )
-    validate_cliresult(result)
-    assert "is already installed" in result.output
-
-
-def test_global_lib_list(clirunner, validate_cliresult):
-    result = clirunner.invoke(cmd_lib, ["-g", "list"])
-    validate_cliresult(result)
-    assert all(
-        [
-            n in result.output
-            for n in (
-                "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
-                "Version: 5.10.1",
-                "Source: git+https://github.com/gioblu/PJON.git#3.0",
-                "Version: 1fb26fd",
-            )
-        ]
-    )
-
-    result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
-    assert all(
-        [
-            n in result.output
-            for n in (
-                "__pkg_dir",
-                '"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"',
-                '"version": "5.10.1"',
-            )
-        ]
-    )
-    items1 = [i["name"] for i in json.loads(result.output)]
-    items2 = [
-        "ESP32WebServer",
-        "ArduinoJson",
-        "ArduinoJson",
-        "ArduinoJson",
-        "ArduinoJson",
-        "AsyncMqttClient",
-        "AsyncTCP",
-        "SomeLib",
-        "ESPAsyncTCP",
-        "NeoPixelBus",
-        "OneWire",
-        "PJON",
-        "PJON",
-        "PubSubClient",
-        "Adafruit PN532",
-        "Adafruit BusIO",
-        "platformio-libmirror",
-        "rs485-nodeproto",
-    ]
-    assert sorted(items1) == sorted(items2)
-
-    versions1 = [
-        "{name}@{version}".format(**item) for item in json.loads(result.output)
-    ]
-    versions2 = [
-        "ArduinoJson@5.8.2",
-        "ArduinoJson@5.10.1",
-        "AsyncMqttClient@0.8.2",
-        "NeoPixelBus@2.2.4",
-        "PJON@07fe9aa",
-        "PJON@1fb26fd",
-        "PubSubClient@bef5814",
-        "Adafruit PN532@1.2.0",
-    ]
-    assert set(versions1) >= set(versions2)
-
-
-def test_global_lib_update_check(clirunner, validate_cliresult):
-    result = clirunner.invoke(
-        cmd_lib, ["-g", "update", "--only-check", "--json-output"]
-    )
-    validate_cliresult(result)
-    output = json.loads(result.output)
-    assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([l["name"] for l in output])
-
-
-def test_global_lib_update(clirunner, validate_cliresult):
-    # update library using package directory
-    result = clirunner.invoke(
-        cmd_lib, ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"]
-    )
-    validate_cliresult(result)
-    oudated = json.loads(result.output)
-    assert len(oudated) == 1
-    assert "__pkg_dir" in oudated[0]
-    result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]])
-    validate_cliresult(result)
-    assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output
-
-    # update rest libraries
-    result = clirunner.invoke(cmd_lib, ["-g", "update"])
-    validate_cliresult(result)
-    assert result.output.count("[Detached]") == 5
-    assert result.output.count("[Up-to-date]") == 12
-
-    # update unknown library
-    result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"])
-    assert result.exit_code != 0
-    assert isinstance(result.exception, exception.UnknownPackage)
-
-
-def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core):
-    # uninstall using package directory
-    result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
-    validate_cliresult(result)
-    items = json.loads(result.output)
-    items = sorted(items, key=lambda item: item["__pkg_dir"])
-    result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]])
-    validate_cliresult(result)
-    assert ("Uninstalling %s" % items[0]["name"]) in result.output
-
-    # uninstall the rest libraries
-    result = clirunner.invoke(
-        cmd_lib,
-        [
-            "-g",
-            "uninstall",
-            "OneWire",
-            "https://github.com/bblanchon/ArduinoJson.git",
-            "ArduinoJson@!=5.6.7",
-            "Adafruit PN532",
-        ],
-    )
-    validate_cliresult(result)
-
-    items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
-    items2 = [
-        "rs485-nodeproto",
-        "platformio-libmirror",
-        "PubSubClient",
-        "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
-        "ESPAsyncTCP",
-        "ESP32WebServer",
-        "NeoPixelBus",
-        "PJON",
-        "AsyncMqttClient",
-        "ArduinoJson",
-        "SomeLib_ID54",
-        "PJON@src-79de467ebe19de18287becff0a1fb42d",
-        "AsyncTCP",
-    ]
-    assert set(items1) == set(items2)
-
-    # uninstall unknown library
-    result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"])
-    assert result.exit_code != 0
-    assert isinstance(result.exception, exception.UnknownPackage)
-
-
-def test_lib_show(clirunner, validate_cliresult):
-    result = clirunner.invoke(cmd_lib, ["show", "64"])
-    validate_cliresult(result)
-    assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")])
-    result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"])
-    validate_cliresult(result)
-    assert "OneWire" in result.output
-
-
-def test_lib_builtin(clirunner, validate_cliresult):
-    result = clirunner.invoke(cmd_lib, ["builtin"])
-    validate_cliresult(result)
-    result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"])
-    validate_cliresult(result)
-
-
-def test_lib_stats(clirunner, validate_cliresult):
-    result = clirunner.invoke(cmd_lib, ["stats"])
-    validate_cliresult(result)
-    assert all(
-        [
-            s in result.output
-            for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show")
-        ]
-    )
-
-    result = clirunner.invoke(cmd_lib, ["stats", "--json-output"])
-    validate_cliresult(result)
-    assert set(
-        [
-            "dlweek",
-            "added",
-            "updated",
-            "topkeywords",
-            "dlmonth",
-            "dlday",
-            "lastkeywords",
-        ]
-    ) == set(json.loads(result.output).keys())

tests/commands/test_lib_complex.py

@@ -0,0 +1,348 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import re
+
+from platformio import exception
+from platformio.commands import PlatformioCLI
+from platformio.commands.lib.command import cli as cmd_lib
+from platformio.package.exception import UnknownPackageError
+
+PlatformioCLI.leftover_args = ["--json-output"]  # hook for click
+
+
+def test_search(clirunner, validate_cliresult):
+    result = clirunner.invoke(cmd_lib, ["search", "DHT22"])
+    validate_cliresult(result)
+    match = re.search(r"Found\s+(\d+)\slibraries:", result.output)
+    assert int(match.group(1)) > 2
+
+    result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"])
+    validate_cliresult(result)
+    match = re.search(r"Found\s+(\d+)\slibraries:", result.output)
+    assert int(match.group(1)) > 1
+
+
+def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core):
+    result = clirunner.invoke(
+        cmd_lib,
+        [
+            "-g",
+            "install",
+            "64",
+            "ArduinoJson@~5.10.0",
+            "547@2.2.4",
+            "AsyncMqttClient@<=0.8.2",
+            "Adafruit PN532@1.2.0",
+        ],
+    )
+    validate_cliresult(result)
+
+    # install unknown library
+    result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"])
+    assert result.exit_code != 0
+    assert isinstance(result.exception, UnknownPackageError)
+
+    items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
+    items2 = [
+        "ArduinoJson",
+        "ArduinoJson@5.10.1",
+        "NeoPixelBus",
+        "AsyncMqttClient",
+        "ESPAsyncTCP",
+        "AsyncTCP",
+        "Adafruit PN532",
+        "Adafruit BusIO",
+    ]
+    assert set(items1) == set(items2)
+
+
+def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core):
+    result = clirunner.invoke(
+        cmd_lib,
+        [
+            "-g",
+            "install",
+            "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip",
+            "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2",
+            "SomeLib=https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz",
+            "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
+        ],
+    )
+    validate_cliresult(result)
+
+    # incorrect requirements
+    result = clirunner.invoke(
+        cmd_lib,
+        [
+            "-g",
+            "install",
+            "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3",
+        ],
+    )
+    assert result.exit_code != 0
+
+    items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
+    items2 = [
+        "ArduinoJson",
+        "SomeLib",
+        "OneWire",
+        "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8",
+    ]
+    assert set(items1) >= set(items2)
+
+
+def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core):
+    result = clirunner.invoke(
+        cmd_lib,
+        [
+            "-g",
+            "install",
+            "https://github.com/gioblu/PJON.git#3.0",
+            "https://github.com/gioblu/PJON.git#6.2",
+            "https://github.com/bblanchon/ArduinoJson.git",
+            "https://github.com/platformio/platformio-libmirror.git",
+            # "https://developer.mbed.org/users/simon/code/TextLCD/",
+            "https://github.com/knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163",
+        ],
+    )
+    validate_cliresult(result)
+    items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
+    items2 = [
+        "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f",
+        "PJON@src-79de467ebe19de18287becff0a1fb42d",
+        "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
+        "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377",
+        "PubSubClient@src-98ec699a461a31615982e5adaaefadda",
+    ]
+    assert set(items1) >= set(items2)
+
+
+def test_install_duplicates(clirunner, validate_cliresult, without_internet):
+    # registry
+    result = clirunner.invoke(
+        cmd_lib,
+        [
+            "-g",
+            "install",
+            "https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz",
+        ],
+    )
+    validate_cliresult(result)
+    assert "is already installed" in result.output
+
+    # archive
+    result = clirunner.invoke(
+        cmd_lib,
+        [
+            "-g",
+            "install",
+            "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
+        ],
+    )
+    validate_cliresult(result)
+    assert "is already installed" in result.output
+
+    # repository
+    result = clirunner.invoke(
+        cmd_lib,
+        ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"],
+    )
+    validate_cliresult(result)
+    assert "is already installed" in result.output
+
+
+def test_global_lib_list(clirunner, validate_cliresult):
+    result = clirunner.invoke(cmd_lib, ["-g", "list"])
+    validate_cliresult(result)
+    assert all(
+        [
+            n in result.output
+            for n in (
+                "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
+                "Version: 5.10.1",
+                "Source: git+https://github.com/gioblu/PJON.git#3.0",
+                "Version: 3.0.0+sha.1fb26fd",
+            )
+        ]
+    )
+
+    result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
+    assert all(
+        [
+            n in result.output
+            for n in (
+                "__pkg_dir",
+                '"__src_url": "git+https://github.com/gioblu/PJON.git#6.2"',
+                '"version": "5.10.1"',
+            )
+        ]
+    )
+    items1 = [i["name"] for i in json.loads(result.output)]
+    items2 = [
+        "Adafruit BusIO",
+        "Adafruit PN532",
+        "ArduinoJson",
+        "ArduinoJson",
+        "ArduinoJson",
+        "ArduinoJson",
+        "AsyncMqttClient",
+        "AsyncTCP",
+        "DallasTemperature",
+        "ESP32WebServer",
+        "ESPAsyncTCP",
+        "NeoPixelBus",
+        "OneWire",
+        "PJON",
+        "PJON",
+        "platformio-libmirror",
+        "PubSubClient",
+    ]
+    assert sorted(items1) == sorted(items2)
+
+    versions1 = [
+        "{name}@{version}".format(**item) for item in json.loads(result.output)
+    ]
+    versions2 = [
+        "ArduinoJson@5.8.2",
+        "ArduinoJson@5.10.1",
+        "AsyncMqttClient@0.8.2",
+        "NeoPixelBus@2.2.4",
+        "PJON@6.2.0+sha.07fe9aa",
+        "PJON@3.0.0+sha.1fb26fd",
+        "PubSubClient@2.6.0+sha.bef5814",
+        "Adafruit PN532@1.2.0",
+    ]
+    assert set(versions1) >= set(versions2)
+
+
+def test_global_lib_update_check(clirunner, validate_cliresult):
+    result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"])
+    validate_cliresult(result)
+    output = json.loads(result.output)
+    assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([lib["name"] for lib in output])
+
+
+def test_global_lib_update(clirunner, validate_cliresult):
+    # update library using package directory
+    result = clirunner.invoke(
+        cmd_lib, ["-g", "update", "NeoPixelBus", "--dry-run", "--json-output"]
+    )
+    validate_cliresult(result)
+    oudated = json.loads(result.output)
+    assert len(oudated) == 1
+    assert "__pkg_dir" in oudated[0]
+    result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]])
+    validate_cliresult(result)
+    assert "Removing NeoPixelBus @ 2.2.4" in result.output
+
+    # update rest libraries
+    result = clirunner.invoke(cmd_lib, ["-g", "update"])
+    validate_cliresult(result)
+    assert result.output.count("[Detached]") == 1
+    assert result.output.count("[Up-to-date]") == 15
+
+    # update unknown library
+    result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"])
+    assert result.exit_code != 0
+    assert isinstance(result.exception, UnknownPackageError)
+
+
+def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core):
+    # uninstall using package directory
+    result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
+    validate_cliresult(result)
+    items = json.loads(result.output)
+    items = sorted(items, key=lambda item: item["__pkg_dir"])
+    result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]])
+    validate_cliresult(result)
+    assert ("Removing %s" % items[0]["name"]) in result.output
+
+    # uninstall the rest libraries
+    result = clirunner.invoke(
+        cmd_lib,
+        [
+            "-g",
+            "uninstall",
+            "OneWire",
+            "https://github.com/bblanchon/ArduinoJson.git",
+            "ArduinoJson@!=5.6.7",
+            "Adafruit PN532",
+        ],
+    )
+    validate_cliresult(result)
+
+    items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
+    items2 = [
+        "ArduinoJson",
+        "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
+        "AsyncMqttClient",
+        "AsyncTCP",
+        "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8",
+        "ESPAsyncTCP",
+        "NeoPixelBus",
+        "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f",
+        "PJON@src-79de467ebe19de18287becff0a1fb42d",
+        "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377",
+        "PubSubClient@src-98ec699a461a31615982e5adaaefadda",
+        "SomeLib",
+    ]
+    assert set(items1) == set(items2)
+
+    # uninstall unknown library
+    result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"])
+    assert result.exit_code != 0
+    assert isinstance(result.exception, UnknownPackageError)
+
+
+def test_lib_show(clirunner, validate_cliresult):
+    result = clirunner.invoke(cmd_lib, ["show", "64"])
+    validate_cliresult(result)
+    assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")])
+    result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"])
+    validate_cliresult(result)
+    assert "OneWire" in result.output
+
+
+def test_lib_builtin(clirunner, validate_cliresult):
+    result = clirunner.invoke(cmd_lib, ["builtin"])
+    validate_cliresult(result)
+    result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"])
+    validate_cliresult(result)
+
+
+def test_lib_stats(clirunner, validate_cliresult):
+    result = clirunner.invoke(cmd_lib, ["stats"])
+    validate_cliresult(result)
+    assert all(
+        [
+            s in result.output
+            for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show")
+        ]
+    )
+
+    result = clirunner.invoke(cmd_lib, ["stats", "--json-output"])
+    validate_cliresult(result)
+    assert set(
+        [
+            "dlweek",
+            "added",
+            "updated",
+            "topkeywords",
+            "dlmonth",
+            "dlday",
+            "lastkeywords",
+        ]
+    ) == set(json.loads(result.output).keys())

tests/package/test_manager.py

@@ -16,6 +16,7 @@ import os
 import time
 
 import pytest
+import semantic_version
 
 from platformio import fs, util
 from platformio.package.exception import (
@@ -201,6 +202,12 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory):
     assert lm.get_package("OneWire").metadata.version.major >= 2
     assert len(lm.get_installed()) == 6
 
+    # test conflicted names
+    lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage")))
+    lm.install("4@2.6.1", silent=True)
+    lm.install("5357@2.6.1", silent=True)
+    assert len(lm.get_installed()) == 2
+
     # Tools
     tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage")))
     pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True)
@@ -340,3 +347,81 @@ def test_uninstall(isolated_pio_core, tmpdir_factory):
     assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True)
     assert lm.uninstall("AsyncMqttClient-esphome", silent=True)
     assert len(lm.get_installed()) == 0
+
+
+def test_registry(isolated_pio_core):
+    lm = LibraryPackageManager()
+
+    # reveal ID
+    assert lm.reveal_registry_package_id(PackageSpec(id=13)) == 13
+    assert lm.reveal_registry_package_id(PackageSpec(name="OneWire"), silent=True) == 1
+    with pytest.raises(UnknownPackageError):
+        lm.reveal_registry_package_id(PackageSpec(name="/non-existing-package/"))
+
+    # fetch package data
+    assert lm.fetch_registry_package(PackageSpec(id=1))["name"] == "OneWire"
+    assert lm.fetch_registry_package(PackageSpec(name="ArduinoJson"))["id"] == 64
+    assert (
+        lm.fetch_registry_package(
+            PackageSpec(id=13, owner="adafruit", name="Renamed library")
+        )["name"]
+        == "Adafruit GFX Library"
+    )
+    with pytest.raises(UnknownPackageError):
+        lm.fetch_registry_package(
+            PackageSpec(owner="unknown<>owner", name="/non-existing-package/")
+        )
+    with pytest.raises(UnknownPackageError):
+        lm.fetch_registry_package(PackageSpec(name="/non-existing-package/"))
+
+
+def test_update_with_metadata(isolated_pio_core, tmpdir_factory):
+    storage_dir = tmpdir_factory.mktemp("storage")
+    lm = LibraryPackageManager(str(storage_dir))
+    pkg = lm.install("ArduinoJson @ 5.10.1", silent=True)
+
+    # tesy latest
+    outdated = lm.outdated(pkg)
+    assert str(outdated.current) == "5.10.1"
+    assert outdated.wanted is None
+    assert outdated.latest > outdated.current
+    assert outdated.latest > semantic_version.Version("5.99.99")
+
+    # test wanted
+    outdated = lm.outdated(pkg, PackageSpec("ArduinoJson@~5"))
+    assert str(outdated.current) == "5.10.1"
+    assert str(outdated.wanted) == "5.13.4"
+    assert outdated.latest > semantic_version.Version("6.16.0")
+
+    # update to the wanted 5.x
+    new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5"), silent=True)
+    assert str(new_pkg.metadata.version) == "5.13.4"
+    # check that old version is removed
+    assert len(lm.get_installed()) == 1
+
+    # update to the latest
+    lm = LibraryPackageManager(str(storage_dir))
+    pkg = lm.update("ArduinoJson", silent=True)
+    assert pkg.metadata.version == outdated.latest
+
+
+def test_update_without_metadata(isolated_pio_core, tmpdir_factory):
+    storage_dir = tmpdir_factory.mktemp("storage")
+    storage_dir.join("legacy-package").mkdir().join("library.json").write(
+        '{"name": "AsyncMqttClient-esphome", "version": "0.8.2"}'
+    )
+    storage_dir.join("legacy-dep").mkdir().join("library.json").write(
+        '{"name": "AsyncTCP-esphome", "version": "1.1.1"}'
+    )
+    lm = LibraryPackageManager(str(storage_dir))
+    pkg = lm.get_package("AsyncMqttClient-esphome")
+    outdated = lm.outdated(pkg)
+    assert len(lm.get_installed()) == 2
+    assert str(pkg.metadata.version) == "0.8.2"
+    assert outdated.latest > semantic_version.Version("0.8.2")
+
+    # update
+    lm = LibraryPackageManager(str(storage_dir))
+    new_pkg = lm.update(pkg, silent=True)
+    assert len(lm.get_installed()) == 3
+    assert new_pkg.metadata.spec.owner == "ottowinter"

tests/package/test_meta.py

@@ -17,7 +17,27 @@ import os
 import jsondiff
 import semantic_version
 
-from platformio.package.meta import PackageMetaData, PackageSpec, PackageType
+from platformio.package.meta import (
+    PackageMetaData,
+    PackageOutdatedResult,
+    PackageSpec,
+    PackageType,
+)
+
+
+def test_outdated_result():
+    result = PackageOutdatedResult(current="1.2.3", latest="2.0.0")
+    assert result.is_outdated()
+    assert result.is_outdated(allow_incompatible=True)
+    result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.5.4")
+    assert result.is_outdated()
+    assert result.is_outdated(allow_incompatible=True)
+    result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.2.3")
+    assert not result.is_outdated()
+    assert result.is_outdated(allow_incompatible=True)
+    result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", detached=True)
+    assert not result.is_outdated()
+    assert not result.is_outdated(allow_incompatible=True)
 
 
 def test_spec_owner():
@@ -45,9 +65,16 @@ def test_spec_name():
 
 def test_spec_requirements():
     assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3")
+    assert PackageSpec(
+        name="foo", requirements=semantic_version.Version("1.2.3")
+    ) == PackageSpec(name="foo", requirements="1.2.3")
     assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3")
     assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0")
+    assert PackageSpec(
+        name="hello", requirements=semantic_version.SimpleSpec("~1.2.3")
+    ) == PackageSpec(name="hello", requirements="~1.2.3")
     spec = PackageSpec("id=20 @ !=1.2.3,<2.0")
+    assert not spec.external
     assert isinstance(spec.requirements, semantic_version.SimpleSpec)
     assert semantic_version.Version("1.3.0-beta.1") in spec.requirements
     assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0")
@@ -88,7 +115,8 @@ def test_spec_external_urls():
         "Custom-Name="
         "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0"
     )
-    assert spec.is_custom_name()
+    assert spec.external
+    assert spec.has_custom_name()
     assert spec.name == "Custom-Name"
     assert spec == PackageSpec(
         url="https://github.com/platformio/platformio-core/archive/develop.tar.gz",
@@ -163,6 +191,24 @@ def test_spec_as_dict():
     )
 
 
+def test_spec_as_dependency():
+    assert PackageSpec("owner/pkgname").as_dependency() == "owner/pkgname"
+    assert PackageSpec(owner="owner", name="pkgname").as_dependency() == "owner/pkgname"
+    assert PackageSpec("bob/foo @ ^1.2.3").as_dependency() == "bob/foo@^1.2.3"
+    assert (
+        PackageSpec(
+            "https://github.com/o/r/a/develop.zip?param=value @ !=2"
+        ).as_dependency()
+        == "https://github.com/o/r/a/develop.zip?param=value @ !=2"
+    )
+    assert (
+        PackageSpec(
+            "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
+        ).as_dependency()
+        == "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
+    )
+
+
 def test_metadata_as_dict():
     metadata = PackageMetaData(PackageType.LIBRARY, "foo", "1.2.3")
     # test setter

tests/test_maintenance.py

@@ -88,7 +88,9 @@ def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_clire
     validate_cliresult(result)
     assert "There are the new updates for libraries (ArduinoJson)" in result.output
     assert "Please wait while updating libraries" in result.output
-    assert re.search(r"Updating ArduinoJson\s+@ 6.12.0\s+\[[\d\.]+\]", result.output)
+    assert re.search(
+        r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[[\d\.]+\]", result.output
+    )
 
     # check updated version
     result = clirunner.invoke(cli_pio, ["lib", "-g", "list", "--json-output"])