Use isolated library dependency storage per project build environment // Resolve #1696

HISTORY.rst

@@ -7,14 +7,20 @@ PlatformIO 4.0
 4.0.0 (2019-??-??)
 ~~~~~~~~~~~~~~~~~~
 
-* **PlatformIO-based Project**
+* **Project Management**
 
-  - Implemented unified project workspace storage (`workspace_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#workspace-dir>`__ -> ``.pio``) for PlatformIO Build System, Library Dependency Finder, and other internal services (`issue #1778 <https://github.com/platformio/platformio-core/issues/1778>`_)
+  - Unified workspace storage (`workspace_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#workspace-dir>`__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 <https://github.com/platformio/platformio-core/issues/1778>`_)
   - Share common (global) options between build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ (`issue #1643 <https://github.com/platformio/platformio-core/issues/1643>`_)
   - Include external configuration files in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ with `extra_configs <http://docs.platformio.org/page/projectconf/section_platformio.html#extra-configs>`__ option (`issue #1590 <https://github.com/platformio/platformio-core/issues/1590>`_)
   - Override default `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ with a custom using ``-c, --project-conf`` option for `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__, `platformio debug <http://docs.platformio.org/page/userguide/cmd_debug.html>`__, or `platformio test <http://docs.platformio.org/page/userguide/cmd_test.html>`__ commands (`issue #1913 <https://github.com/platformio/platformio-core/issues/1913>`_)
   - Custom project ``***_dir`` options declared in "platformio" section of `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ have higher priority than `Environment variables <http://docs.platformio.org/page/envvars.html>`__
-  - Moved ``.pioenvs`` build directory to workspace storage ``.pio/build``
+  - Use workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs``
+
+* **Library Management**
+
+  - Use isolated library dependency storage per project build environment (`issue #1696 <https://github.com/platformio/platformio-core/issues/1696>`_)
+  - Override default source and include directories for a library via `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__ manifest using ``includeDir`` and ``srcDir`` fields
+  - Use workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps``
 
 * **Infrastructure**
 
@@ -24,7 +30,6 @@ PlatformIO 4.0
 
 * **Miscellaneous**
 
-  - Override default source and include directories for a library via `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__ manifest using ``includeDir`` and ``srcDir`` fields
   - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead
 
 PlatformIO 3.0

platformio/builder/main.py

@@ -110,6 +110,7 @@ DEFAULT_ENV_OPTIONS = dict(
     PIOHOME_DIR=util.get_home_dir(),
     PROJECT_DIR=get_project_dir(),
     PROJECTWORKSPACE_DIR=get_projectworkspace_dir(),
+    PROJECTLIBDEPS_DIR=get_projectlibdeps_dir(),
     PROJECTINCLUDE_DIR=get_projectinclude_dir(),
     PROJECTSRC_DIR=get_projectsrc_dir(),
     PROJECTTEST_DIR=get_projecttest_dir(),
@@ -121,7 +122,7 @@ DEFAULT_ENV_OPTIONS = dict(
     LIBPATH=["$BUILD_DIR"],
     LIBSOURCE_DIRS=[
         get_projectlib_dir(),
-        get_projectlibdeps_dir(),
+        join("$PROJECTLIBDEPS_DIR", "$PIOENV"),
         join("$PIOHOME_DIR", "lib")
     ],
     PROGNAME="program",

platformio/commands/lib.py

@@ -21,9 +21,9 @@ from os.path import isdir, join
 import click
 
 from platformio import exception, util
-from platformio.commands import PlatformioCLI
 from platformio.managers.lib import LibraryManager, get_builtin_libs
 from platformio.proc import is_ci
+from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (
     get_project_dir, get_projectlibdeps_dir, is_platformio_project)
 
@@ -34,14 +34,10 @@ except ImportError:
 
 
 @click.group(short_help="Library Manager")
-@click.option(
-    "-g",
-    "--global",
-    is_flag=True,
-    help="Manage global PlatformIO library storage")
 @click.option(
     "-d",
     "--storage-dir",
+    multiple=True,
     default=None,
     type=click.Path(
         exists=True,
@@ -50,38 +46,56 @@ except ImportError:
         writable=True,
         resolve_path=True),
     help="Manage custom library storage")
+@click.option(
+    "-g",
+    "--global",
+    is_flag=True,
+    help="Manage global PlatformIO library storage")
+@click.option(
+    "-e",
+    "--environment",
+    multiple=True,
+    help=("Manage libraries for the specific project build environments "
+          "declared in `platformio.ini`"))
 @click.pass_context
 def cli(ctx, **options):
-    non_storage_cmds = ("search", "show", "register", "stats", "builtin")
+    storage_cmds = ("install", "uninstall", "update", "list")
     # skip commands that don't need storage folder
-    if ctx.invoked_subcommand in non_storage_cmds or \
+    if ctx.invoked_subcommand not in storage_cmds or \
             (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")):
         return
-    storage_dir = options['storage_dir']
+    storage_dirs = list(options['storage_dir'])
-    if not storage_dir:
+    if options['global']:
-        if options['global']:
+        storage_dirs.append(join(util.get_home_dir(), "lib"))
-            storage_dir = join(util.get_home_dir(), "lib")
+    if not storage_dirs:
-        elif is_platformio_project():
+        if is_platformio_project():
-            storage_dir = get_projectlibdeps_dir()
+            storage_dirs = [get_project_dir()]
         elif is_ci():
-            storage_dir = join(util.get_home_dir(), "lib")
+            storage_dirs = [join(util.get_home_dir(), "lib")]
             click.secho(
                 "Warning! Global library storage is used automatically. "
                 "Please use `platformio lib --global %s` command to remove "
                 "this warning." % ctx.invoked_subcommand,
                 fg="yellow")
-    elif is_platformio_project(storage_dir):
-        with util.cd(storage_dir):
-            storage_dir = get_projectlibdeps_dir()
 
-    if not storage_dir and not is_platformio_project():
+    if not storage_dirs:
         raise exception.NotGlobalLibDir(get_project_dir(),
                                         join(util.get_home_dir(), "lib"),
                                         ctx.invoked_subcommand)
-
+    ctx.obj = []
-    ctx.obj = LibraryManager(storage_dir)
+    for storage_dir in storage_dirs:
-    if not PlatformioCLI.in_silence():
+        if is_platformio_project(storage_dir):
-        click.echo("Library Storage: " + storage_dir)
+            with util.cd(storage_dir):
+                config = ProjectConfig.get_instance(
+                    join(storage_dir, "platformio.ini"))
+                config.validate(options['environment'])
+                libdeps_dir = get_projectlibdeps_dir()
+                for env in config.envs():
+                    if (not options['environment']
+                            or env in options['environment']):
+                        ctx.obj.append(join(libdeps_dir, env))
+        else:
+            ctx.obj.append(storage_dir)
 
 
 @cli.command("install", short_help="Install library")
@@ -103,19 +117,24 @@ def cli(ctx, **options):
     is_flag=True,
     help="Reinstall/redownload library if exists")
 @click.pass_obj
-def lib_install(lm, libraries, silent, interactive, force):
+def lib_install(storage_dirs, libraries, silent, interactive, force):
-    # @TODO: "save" option
+    for storage_dir in storage_dirs:
-    for library in libraries:
+        print_storage_header(storage_dirs, storage_dir)
-        lm.install(
+        lm = LibraryManager(storage_dir)
-            library, silent=silent, interactive=interactive, force=force)
+        for library in libraries:
+            lm.install(
+                library, silent=silent, interactive=interactive, force=force)
 
 
 @cli.command("uninstall", short_help="Uninstall libraries")
 @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]")
 @click.pass_obj
-def lib_uninstall(lm, libraries):
+def lib_uninstall(storage_dirs, libraries):
-    for library in libraries:
+    for storage_dir in storage_dirs:
-        lm.uninstall(library)
+        print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
+        for library in libraries:
+            lm.uninstall(library)
 
 
 @cli.command("update", short_help="Update installed libraries")
@@ -131,68 +150,72 @@ def lib_uninstall(lm, libraries):
     help="Do not update, only check for the new versions")
 @click.option("--json-output", is_flag=True)
 @click.pass_obj
-def lib_update(lm, libraries, only_check, dry_run, json_output):
+def lib_update(storage_dirs, libraries, only_check, dry_run, json_output):
-    if not libraries:
-        libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()]
-
     only_check = dry_run or only_check
+    json_result = {}
+    for storage_dir in storage_dirs:
+        if not json_output:
+            print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
 
-    if only_check and json_output:
+        _libraries = libraries
-        result = []
+        if not _libraries:
-        for library in libraries:
+            _libraries = [
-            pkg_dir = library if isdir(library) else None
+                manifest['__pkg_dir'] for manifest in lm.get_installed()
-            requirements = None
+            ]
-            url = None
-            if not pkg_dir:
-                name, requirements, url = lm.parse_pkg_uri(library)
-                pkg_dir = lm.get_package_dir(name, requirements, url)
-            if not pkg_dir:
-                continue
-            latest = lm.outdated(pkg_dir, requirements)
-            if not latest:
-                continue
-            manifest = lm.load_manifest(pkg_dir)
-            manifest['versionLatest'] = latest
-            result.append(manifest)
-        return click.echo(json.dumps(result))
 
-    for library in libraries:
+        if only_check and json_output:
-        lm.update(library, only_check=only_check)
+            result = []
+            for library in _libraries:
+                pkg_dir = library if isdir(library) else None
+                requirements = None
+                url = None
+                if not pkg_dir:
+                    name, requirements, url = lm.parse_pkg_uri(library)
+                    pkg_dir = lm.get_package_dir(name, requirements, url)
+                if not pkg_dir:
+                    continue
+                latest = lm.outdated(pkg_dir, requirements)
+                if not latest:
+                    continue
+                manifest = lm.load_manifest(pkg_dir)
+                manifest['versionLatest'] = latest
+                result.append(manifest)
+            json_result[storage_dir] = result
+        else:
+            for library in _libraries:
+                lm.update(library, only_check=only_check)
+
+    if json_output:
+        return click.echo(
+            json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) ==
+                       1 else json_result))
 
     return True
 
 
-def print_lib_item(item):
+@cli.command("list", short_help="List installed libraries")
-    click.secho(item['name'], fg="cyan")
+@click.option("--json-output", is_flag=True)
-    click.echo("=" * len(item['name']))
+@click.pass_obj
-    if "id" in item:
+def lib_list(storage_dirs, json_output):
-        click.secho("#ID: %d" % item['id'], bold=True)
+    json_result = {}
-    if "description" in item or "url" in item:
+    for storage_dir in storage_dirs:
-        click.echo(item.get("description", item.get("url", "")))
+        if not json_output:
-    click.echo()
+            print_storage_header(storage_dirs, storage_dir)
-
+        lm = LibraryManager(storage_dir)
-    for key in ("version", "homepage", "license", "keywords"):
+        items = lm.get_installed()
-        if key not in item or not item[key]:
+        if json_output:
-            continue
+            json_result[storage_dir] = items
-        if isinstance(item[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
         else:
-            click.echo("%s: %s" % (key.title(), item[key]))
+            for item in sorted(items, key=lambda i: i['name']):
+                print_lib_item(item)
 
-    for key in ("frameworks", "platforms"):
+    if json_output:
-        if key not in item:
+        return click.echo(
-            continue
+            json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) ==
-        click.echo("Compatible %s: %s" % (key, ", ".join(
+                       1 else json_result))
-            [i['title'] if isinstance(i, dict) else i for i in item[key]])))
 
-    if "authors" in item or "authornames" in item:
+    return True
-        click.echo("Authors: %s" % ", ".join(
-            item.get("authornames",
-                     [a.get("name", "") for a in item.get("authors", [])])))
-
-    if "__src_url" in item:
-        click.secho("Source: %s" % item['__src_url'])
-    click.echo()
 
 
 @cli.command("search", short_help="Search for a library")
@@ -275,24 +298,6 @@ def lib_search(query, json_output, page, noninteractive, **filters):
             cache_valid="1d")
 
 
-@cli.command("list", short_help="List installed libraries")
-@click.option("--json-output", is_flag=True)
-@click.pass_obj
-def lib_list(lm, json_output):
-    items = lm.get_installed()
-
-    if json_output:
-        return click.echo(json.dumps(items))
-
-    if not items:
-        return None
-
-    for item in sorted(items, key=lambda i: i['name']):
-        print_lib_item(item)
-
-    return True
-
-
 @cli.command("builtin", short_help="List built-in libraries")
 @click.option("--storage", multiple=True)
 @click.option("--json-output", is_flag=True)
@@ -484,3 +489,44 @@ def lib_stats(json_output):
         click.echo()
 
     return True
+
+
+def print_storage_header(storage_dirs, storage_dir):
+    if storage_dirs and storage_dirs[0] != storage_dir:
+        click.echo("")
+    click.echo(
+        click.style("Library Storage: ", bold=True) +
+        click.style(storage_dir, fg="blue"))
+
+
+def print_lib_item(item):
+    click.secho(item['name'], fg="cyan")
+    click.echo("=" * len(item['name']))
+    if "id" in item:
+        click.secho("#ID: %d" % item['id'], bold=True)
+    if "description" in item or "url" in item:
+        click.echo(item.get("description", item.get("url", "")))
+    click.echo()
+
+    for key in ("version", "homepage", "license", "keywords"):
+        if key not in item or not item[key]:
+            continue
+        if isinstance(item[key], list):
+            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+        else:
+            click.echo("%s: %s" % (key.title(), item[key]))
+
+    for key in ("frameworks", "platforms"):
+        if key not in item:
+            continue
+        click.echo("Compatible %s: %s" % (key, ", ".join(
+            [i['title'] if isinstance(i, dict) else i for i in item[key]])))
+
+    if "authors" in item or "authornames" in item:
+        click.echo("Authors: %s" % ", ".join(
+            item.get("authornames",
+                     [a.get("name", "") for a in item.get("authors", [])])))
+
+    if "__src_url" in item:
+        click.secho("Source: %s" % item['__src_url'])
+    click.echo()

platformio/commands/run.py

@@ -23,7 +23,7 @@ from platformio.commands.device import device_monitor as cmd_device_monitor
 from platformio.commands.lib import lib_install as cmd_lib_install
 from platformio.commands.platform import \
     platform_install as cmd_platform_install
-from platformio.managers.lib import LibraryManager, is_builtin_lib
+from platformio.managers.lib import is_builtin_lib
 from platformio.managers.platform import PlatformFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (
@@ -82,6 +82,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf,
             project_conf or join(project_dir, "platformio.ini"))
         config.validate(environment)
 
+        _handle_legacy_libdeps(project_dir, config)
+
         results = []
         start_time = time()
         default_envs = config.default_envs()
@@ -94,7 +96,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf,
                 results.append((envname, None))
                 continue
 
-            if not silent and results:
+            if not silent and any(
+                    status is not None for (_, status) in results):
                 click.echo()
 
             options = config.items(env=envname, as_dict=True)
@@ -222,14 +225,14 @@ class EnvironmentProcessor(object):
         if "nobuild" not in build_targets:
             # install dependent libraries
             if "lib_install" in self.options:
-                _autoinstall_libdeps(self.cmd_ctx, [
+                _autoinstall_libdeps(self.cmd_ctx, self.name, [
                     int(d.strip())
                     for d in self.options['lib_install'].split(",")
                     if d.strip()
                 ], self.verbose)
             if "lib_deps" in self.options:
                 _autoinstall_libdeps(
-                    self.cmd_ctx,
+                    self.cmd_ctx, self.name,
                     ProjectConfig.parse_multi_values(self.options['lib_deps']),
                     self.verbose)
 
@@ -245,13 +248,31 @@ class EnvironmentProcessor(object):
         return p.run(build_vars, build_targets, self.silent, self.verbose)
 
 
-def _autoinstall_libdeps(ctx, libraries, verbose=False):
+def _handle_legacy_libdeps(project_dir, config):
+    legacy_libdeps_dir = join(project_dir, ".piolibdeps")
+    if (not isdir(legacy_libdeps_dir)
+            or legacy_libdeps_dir == get_projectlibdeps_dir()):
+        return
+    if not config.has_section("env"):
+        config.add_section("env")
+    lib_extra_dirs = []
+    if config.has_option("env", "lib_extra_dirs"):
+        lib_extra_dirs = config.getlist("env", "lib_extra_dirs")
+    lib_extra_dirs.append(legacy_libdeps_dir)
+    config.set("env", "lib_extra_dirs", lib_extra_dirs)
+    click.secho(
+        "DEPRECATED! A legacy library storage `{0}` has been found in a "
+        "project. \nPlease declare project dependencies in `platformio.ini`"
+        " file using `lib_deps` option and remove `{0}` folder."
+        "\nMore details -> http://docs.platformio.org/page/projectconf/"
+        "section_env_library.html#lib-deps".format(legacy_libdeps_dir),
+        fg="yellow")
+
+
+def _autoinstall_libdeps(ctx, envname, libraries, verbose=False):
     if not libraries:
         return
-    storage_dir = get_projectlibdeps_dir()
+    ctx.obj = [join(get_projectlibdeps_dir(), envname)]
-    ctx.obj = LibraryManager(storage_dir)
-    if verbose:
-        click.echo("Library Storage: " + storage_dir)
     for lib in libraries:
         try:
             ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose)

platformio/commands/update.py

@@ -54,5 +54,5 @@ def cli(ctx, core_packages, only_check, dry_run):
     click.echo()
     click.echo("Library Manager")
     click.echo("===============")
-    ctx.obj = LibraryManager()
+    ctx.obj = [LibraryManager().package_dir]
     ctx.invoke(cmd_lib_update, only_check=only_check)

platformio/ide/projectgenerator.py

@@ -144,7 +144,8 @@ class ProjectGenerator(object):
                 "project_dir": self.project_dir,
                 "project_src_dir": get_projectsrc_dir(),
                 "project_lib_dir": get_projectlib_dir(),
-                "project_libdeps_dir": get_projectlibdeps_dir(),
+                "project_libdeps_dir": join(
+                    get_projectlibdeps_dir(), self.env_name),
                 "systype": util.get_systype(),
                 "platformio_path": self._fix_os_path(
                     sys.argv[0] if isfile(sys.argv[0])

platformio/maintenance.py

@@ -332,7 +332,7 @@ def check_internal_updates(ctx, what):
         if what == "platforms":
             ctx.invoke(cmd_platform_update, platforms=outdated_items)
         elif what == "libraries":
-            ctx.obj = pm
+            ctx.obj = [pm.package_dir]
             ctx.invoke(cmd_lib_update, libraries=outdated_items)
         click.echo()
 

platformio/managers/core.py

@@ -26,10 +26,10 @@ from platformio.managers.package import PackageManager
 from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
 
 CORE_PACKAGES = {
-    "contrib-piohome": "^2.0.1",
+    "contrib-piohome": "^2.1.0",
     "contrib-pysite":
     "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]),
-    "tool-pioplus": "^2.2.0",
+    "tool-pioplus": "^2.3.0",
     "tool-unity": "~1.20403.0",
     "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0"
 }

platformio/project/config.py

@@ -227,6 +227,11 @@ class ProjectConfig(object):
         return [(option, self.get(section, option))
                 for option in self.options(section)]
 
+    def set(self, section, option, value):
+        if isinstance(value, (list, tuple)):
+            value = "\n".join(value)
+        self._parser.set(section, option, value)
+
     def get(self, section, option):
         if not self.expand_interpolations:
             return self._parser.get(section, option)

platformio/project/helpers.py

@@ -93,15 +93,15 @@ def get_projectbuild_dir(force=False):
     return path
 
 
+def get_projectlibdeps_dir():
+    return get_project_optional_dir(
+        "libdeps_dir", join(get_projectworkspace_dir(), "libdeps"))
+
+
 def get_projectlib_dir():
     return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib"))
 
 
-def get_projectlibdeps_dir():
-    return get_project_optional_dir("libdeps_dir",
-                                    join(get_project_dir(), ".piolibdeps"))
-
-
 def get_projectsrc_dir():
     return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))
 

platformio/util.py

@@ -41,7 +41,8 @@ from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (
     get_project_dir, get_project_optional_dir, get_projectboards_dir,
     get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir,
-    get_projectsrc_dir, get_projecttest_dir, is_platformio_project)
+    get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir,
+    is_platformio_project)
 
 
 class cd(object):