From 21e2ac6695288ad0b1dae41b731a7b74acab47f9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 May 2019 00:23:24 +0300 Subject: [PATCH] Use isolated library dependency storage per project build environment // Resolve #1696 --- HISTORY.rst | 13 +- docs | 2 +- platformio/builder/main.py | 3 +- platformio/commands/lib.py | 248 +++++++++++++++++------------ platformio/commands/run.py | 39 +++-- platformio/commands/update.py | 2 +- platformio/ide/projectgenerator.py | 3 +- platformio/maintenance.py | 2 +- platformio/managers/core.py | 4 +- platformio/project/config.py | 5 + platformio/project/helpers.py | 10 +- platformio/util.py | 3 +- 12 files changed, 207 insertions(+), 127 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a9397d32..c757e11b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,14 +7,20 @@ PlatformIO 4.0 4.0.0 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* **PlatformIO-based Project** +* **Project Management** - - Implemented unified project workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Dependency Finder, and other internal services (`issue #1778 `_) + - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) - Share common (global) options between build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ (`issue #1643 `_) - Include external configuration files in `"platformio.ini" (Project Configuration File) `__ with `extra_configs `__ option (`issue #1590 `_) - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - Custom project ``***_dir`` options declared in "platformio" section of `"platformio.ini" (Project Configuration File) `__ have higher priority than `Environment variables `__ - - Moved ``.pioenvs`` build directory to workspace storage ``.pio/build`` + - Use workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` + +* **Library Management** + + - Use isolated library dependency storage per project build environment (`issue #1696 `_) + - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields + - Use workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` * **Infrastructure** @@ -24,7 +30,6 @@ PlatformIO 4.0 * **Miscellaneous** - - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead PlatformIO 3.0 diff --git a/docs b/docs index 103ed844..bbf0f91e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 103ed8445c5993ff305cc14b0fb9929d5fa47053 +Subproject commit bbf0f91e9fc5096edcd9e0656f051da3dbbabf05 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b9937453..4ccbd0f9 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -110,6 +110,7 @@ DEFAULT_ENV_OPTIONS = dict( PIOHOME_DIR=util.get_home_dir(), PROJECT_DIR=get_project_dir(), PROJECTWORKSPACE_DIR=get_projectworkspace_dir(), + PROJECTLIBDEPS_DIR=get_projectlibdeps_dir(), PROJECTINCLUDE_DIR=get_projectinclude_dir(), PROJECTSRC_DIR=get_projectsrc_dir(), PROJECTTEST_DIR=get_projecttest_dir(), @@ -121,7 +122,7 @@ DEFAULT_ENV_OPTIONS = dict( LIBPATH=["$BUILD_DIR"], LIBSOURCE_DIRS=[ get_projectlib_dir(), - get_projectlibdeps_dir(), + join("$PROJECTLIBDEPS_DIR", "$PIOENV"), join("$PIOHOME_DIR", "lib") ], PROGNAME="program", diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 9defcc51..bcba1935 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,9 +21,9 @@ from os.path import isdir, join import click from platformio import exception, util -from platformio.commands import PlatformioCLI from platformio.managers.lib import LibraryManager, get_builtin_libs from platformio.proc import is_ci +from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_dir, get_projectlibdeps_dir, is_platformio_project) @@ -34,14 +34,10 @@ except ImportError: @click.group(short_help="Library Manager") -@click.option( - "-g", - "--global", - is_flag=True, - help="Manage global PlatformIO library storage") @click.option( "-d", "--storage-dir", + multiple=True, default=None, type=click.Path( exists=True, @@ -50,38 +46,56 @@ except ImportError: writable=True, resolve_path=True), help="Manage custom library storage") +@click.option( + "-g", + "--global", + is_flag=True, + help="Manage global PlatformIO library storage") +@click.option( + "-e", + "--environment", + multiple=True, + help=("Manage libraries for the specific project build environments " + "declared in `platformio.ini`")) @click.pass_context def cli(ctx, **options): - non_storage_cmds = ("search", "show", "register", "stats", "builtin") + storage_cmds = ("install", "uninstall", "update", "list") # skip commands that don't need storage folder - if ctx.invoked_subcommand in non_storage_cmds or \ + if ctx.invoked_subcommand not in storage_cmds or \ (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")): return - storage_dir = options['storage_dir'] - if not storage_dir: - if options['global']: - storage_dir = join(util.get_home_dir(), "lib") - elif is_platformio_project(): - storage_dir = get_projectlibdeps_dir() + storage_dirs = list(options['storage_dir']) + if options['global']: + storage_dirs.append(join(util.get_home_dir(), "lib")) + if not storage_dirs: + if is_platformio_project(): + storage_dirs = [get_project_dir()] elif is_ci(): - storage_dir = join(util.get_home_dir(), "lib") + storage_dirs = [join(util.get_home_dir(), "lib")] click.secho( "Warning! Global library storage is used automatically. " "Please use `platformio lib --global %s` command to remove " "this warning." % ctx.invoked_subcommand, fg="yellow") - elif is_platformio_project(storage_dir): - with util.cd(storage_dir): - storage_dir = get_projectlibdeps_dir() - if not storage_dir and not is_platformio_project(): + if not storage_dirs: raise exception.NotGlobalLibDir(get_project_dir(), join(util.get_home_dir(), "lib"), ctx.invoked_subcommand) - - ctx.obj = LibraryManager(storage_dir) - if not PlatformioCLI.in_silence(): - click.echo("Library Storage: " + storage_dir) + ctx.obj = [] + for storage_dir in storage_dirs: + if is_platformio_project(storage_dir): + with util.cd(storage_dir): + config = ProjectConfig.get_instance( + join(storage_dir, "platformio.ini")) + config.validate(options['environment']) + libdeps_dir = get_projectlibdeps_dir() + for env in config.envs(): + if (not options['environment'] + or env in options['environment']): + ctx.obj.append(join(libdeps_dir, env)) + else: + ctx.obj.append(storage_dir) @cli.command("install", short_help="Install library") @@ -103,19 +117,24 @@ def cli(ctx, **options): is_flag=True, help="Reinstall/redownload library if exists") @click.pass_obj -def lib_install(lm, libraries, silent, interactive, force): - # @TODO: "save" option - for library in libraries: - lm.install( - library, silent=silent, interactive=interactive, force=force) +def lib_install(storage_dirs, libraries, silent, interactive, force): + for storage_dir in storage_dirs: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + for library in libraries: + lm.install( + library, silent=silent, interactive=interactive, force=force) @cli.command("uninstall", short_help="Uninstall libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") @click.pass_obj -def lib_uninstall(lm, libraries): - for library in libraries: - lm.uninstall(library) +def lib_uninstall(storage_dirs, libraries): + for storage_dir in storage_dirs: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + for library in libraries: + lm.uninstall(library) @cli.command("update", short_help="Update installed libraries") @@ -131,68 +150,72 @@ def lib_uninstall(lm, libraries): help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) @click.pass_obj -def lib_update(lm, libraries, only_check, dry_run, json_output): - if not libraries: - libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()] - +def lib_update(storage_dirs, libraries, only_check, dry_run, json_output): only_check = dry_run or only_check + json_result = {} + for storage_dir in storage_dirs: + if not json_output: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) - if only_check and json_output: - result = [] - for library in libraries: - pkg_dir = library if isdir(library) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = lm.parse_pkg_uri(library) - pkg_dir = lm.get_package_dir(name, requirements, url) - if not pkg_dir: - continue - latest = lm.outdated(pkg_dir, requirements) - if not latest: - continue - manifest = lm.load_manifest(pkg_dir) - manifest['versionLatest'] = latest - result.append(manifest) - return click.echo(json.dumps(result)) + _libraries = libraries + if not _libraries: + _libraries = [ + manifest['__pkg_dir'] for manifest in lm.get_installed() + ] - for library in libraries: - lm.update(library, only_check=only_check) + if only_check and json_output: + result = [] + for library in _libraries: + pkg_dir = library if isdir(library) else None + requirements = None + url = None + if not pkg_dir: + name, requirements, url = lm.parse_pkg_uri(library) + pkg_dir = lm.get_package_dir(name, requirements, url) + if not pkg_dir: + continue + latest = lm.outdated(pkg_dir, requirements) + if not latest: + continue + manifest = lm.load_manifest(pkg_dir) + manifest['versionLatest'] = latest + result.append(manifest) + json_result[storage_dir] = result + else: + for library in _libraries: + lm.update(library, only_check=only_check) + + if json_output: + return click.echo( + json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) == + 1 else json_result)) return True -def print_lib_item(item): - click.secho(item['name'], fg="cyan") - click.echo("=" * len(item['name'])) - if "id" in item: - click.secho("#ID: %d" % item['id'], bold=True) - if "description" in item or "url" in item: - click.echo(item.get("description", item.get("url", ""))) - click.echo() - - for key in ("version", "homepage", "license", "keywords"): - if key not in item or not item[key]: - continue - if isinstance(item[key], list): - click.echo("%s: %s" % (key.title(), ", ".join(item[key]))) +@cli.command("list", short_help="List installed libraries") +@click.option("--json-output", is_flag=True) +@click.pass_obj +def lib_list(storage_dirs, json_output): + json_result = {} + for storage_dir in storage_dirs: + if not json_output: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + items = lm.get_installed() + if json_output: + json_result[storage_dir] = items else: - click.echo("%s: %s" % (key.title(), item[key])) + for item in sorted(items, key=lambda i: i['name']): + print_lib_item(item) - for key in ("frameworks", "platforms"): - if key not in item: - continue - click.echo("Compatible %s: %s" % (key, ", ".join( - [i['title'] if isinstance(i, dict) else i for i in item[key]]))) + if json_output: + return click.echo( + json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) == + 1 else json_result)) - if "authors" in item or "authornames" in item: - click.echo("Authors: %s" % ", ".join( - item.get("authornames", - [a.get("name", "") for a in item.get("authors", [])]))) - - if "__src_url" in item: - click.secho("Source: %s" % item['__src_url']) - click.echo() + return True @cli.command("search", short_help="Search for a library") @@ -275,24 +298,6 @@ def lib_search(query, json_output, page, noninteractive, **filters): cache_valid="1d") -@cli.command("list", short_help="List installed libraries") -@click.option("--json-output", is_flag=True) -@click.pass_obj -def lib_list(lm, json_output): - items = lm.get_installed() - - if json_output: - return click.echo(json.dumps(items)) - - if not items: - return None - - for item in sorted(items, key=lambda i: i['name']): - print_lib_item(item) - - return True - - @cli.command("builtin", short_help="List built-in libraries") @click.option("--storage", multiple=True) @click.option("--json-output", is_flag=True) @@ -484,3 +489,44 @@ def lib_stats(json_output): click.echo() return True + + +def print_storage_header(storage_dirs, storage_dir): + if storage_dirs and storage_dirs[0] != storage_dir: + click.echo("") + click.echo( + click.style("Library Storage: ", bold=True) + + click.style(storage_dir, fg="blue")) + + +def print_lib_item(item): + click.secho(item['name'], fg="cyan") + click.echo("=" * len(item['name'])) + if "id" in item: + click.secho("#ID: %d" % item['id'], bold=True) + if "description" in item or "url" in item: + click.echo(item.get("description", item.get("url", ""))) + click.echo() + + for key in ("version", "homepage", "license", "keywords"): + if key not in item or not item[key]: + continue + if isinstance(item[key], list): + click.echo("%s: %s" % (key.title(), ", ".join(item[key]))) + else: + click.echo("%s: %s" % (key.title(), item[key])) + + for key in ("frameworks", "platforms"): + if key not in item: + continue + click.echo("Compatible %s: %s" % (key, ", ".join( + [i['title'] if isinstance(i, dict) else i for i in item[key]]))) + + if "authors" in item or "authornames" in item: + click.echo("Authors: %s" % ", ".join( + item.get("authornames", + [a.get("name", "") for a in item.get("authors", [])]))) + + if "__src_url" in item: + click.secho("Source: %s" % item['__src_url']) + click.echo() diff --git a/platformio/commands/run.py b/platformio/commands/run.py index e3085626..f643eb40 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -23,7 +23,7 @@ from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.commands.lib import lib_install as cmd_lib_install from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.managers.lib import LibraryManager, is_builtin_lib +from platformio.managers.lib import is_builtin_lib from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import ( @@ -82,6 +82,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, project_conf or join(project_dir, "platformio.ini")) config.validate(environment) + _handle_legacy_libdeps(project_dir, config) + results = [] start_time = time() default_envs = config.default_envs() @@ -94,7 +96,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, results.append((envname, None)) continue - if not silent and results: + if not silent and any( + status is not None for (_, status) in results): click.echo() options = config.items(env=envname, as_dict=True) @@ -222,14 +225,14 @@ class EnvironmentProcessor(object): if "nobuild" not in build_targets: # install dependent libraries if "lib_install" in self.options: - _autoinstall_libdeps(self.cmd_ctx, [ + _autoinstall_libdeps(self.cmd_ctx, self.name, [ int(d.strip()) for d in self.options['lib_install'].split(",") if d.strip() ], self.verbose) if "lib_deps" in self.options: _autoinstall_libdeps( - self.cmd_ctx, + self.cmd_ctx, self.name, ProjectConfig.parse_multi_values(self.options['lib_deps']), self.verbose) @@ -245,13 +248,31 @@ class EnvironmentProcessor(object): return p.run(build_vars, build_targets, self.silent, self.verbose) -def _autoinstall_libdeps(ctx, libraries, verbose=False): +def _handle_legacy_libdeps(project_dir, config): + legacy_libdeps_dir = join(project_dir, ".piolibdeps") + if (not isdir(legacy_libdeps_dir) + or legacy_libdeps_dir == get_projectlibdeps_dir()): + return + if not config.has_section("env"): + config.add_section("env") + lib_extra_dirs = [] + if config.has_option("env", "lib_extra_dirs"): + lib_extra_dirs = config.getlist("env", "lib_extra_dirs") + lib_extra_dirs.append(legacy_libdeps_dir) + config.set("env", "lib_extra_dirs", lib_extra_dirs) + click.secho( + "DEPRECATED! A legacy library storage `{0}` has been found in a " + "project. \nPlease declare project dependencies in `platformio.ini`" + " file using `lib_deps` option and remove `{0}` folder." + "\nMore details -> http://docs.platformio.org/page/projectconf/" + "section_env_library.html#lib-deps".format(legacy_libdeps_dir), + fg="yellow") + + +def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): if not libraries: return - storage_dir = get_projectlibdeps_dir() - ctx.obj = LibraryManager(storage_dir) - if verbose: - click.echo("Library Storage: " + storage_dir) + ctx.obj = [join(get_projectlibdeps_dir(), envname)] for lib in libraries: try: ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose) diff --git a/platformio/commands/update.py b/platformio/commands/update.py index d4217217..c4ee9a58 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -54,5 +54,5 @@ def cli(ctx, core_packages, only_check, dry_run): click.echo() click.echo("Library Manager") click.echo("===============") - ctx.obj = LibraryManager() + ctx.obj = [LibraryManager().package_dir] ctx.invoke(cmd_lib_update, only_check=only_check) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 37d0db53..58b87cf2 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -144,7 +144,8 @@ class ProjectGenerator(object): "project_dir": self.project_dir, "project_src_dir": get_projectsrc_dir(), "project_lib_dir": get_projectlib_dir(), - "project_libdeps_dir": get_projectlibdeps_dir(), + "project_libdeps_dir": join( + get_projectlibdeps_dir(), self.env_name), "systype": util.get_systype(), "platformio_path": self._fix_os_path( sys.argv[0] if isfile(sys.argv[0]) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index cf563289..3a963038 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -332,7 +332,7 @@ def check_internal_updates(ctx, what): if what == "platforms": ctx.invoke(cmd_platform_update, platforms=outdated_items) elif what == "libraries": - ctx.obj = pm + ctx.obj = [pm.package_dir] ctx.invoke(cmd_lib_update, libraries=outdated_items) click.echo() diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 3c66592c..700a928d 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -26,10 +26,10 @@ from platformio.managers.package import PackageManager from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path CORE_PACKAGES = { - "contrib-piohome": "^2.0.1", + "contrib-piohome": "^2.1.0", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.2.0", + "tool-pioplus": "^2.3.0", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0" } diff --git a/platformio/project/config.py b/platformio/project/config.py index 19842922..a7f9b7b8 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -227,6 +227,11 @@ class ProjectConfig(object): return [(option, self.get(section, option)) for option in self.options(section)] + def set(self, section, option, value): + if isinstance(value, (list, tuple)): + value = "\n".join(value) + self._parser.set(section, option, value) + def get(self, section, option): if not self.expand_interpolations: return self._parser.get(section, option) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index ec72b213..f215a511 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -93,15 +93,15 @@ def get_projectbuild_dir(force=False): return path +def get_projectlibdeps_dir(): + return get_project_optional_dir( + "libdeps_dir", join(get_projectworkspace_dir(), "libdeps")) + + def get_projectlib_dir(): return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) -def get_projectlibdeps_dir(): - return get_project_optional_dir("libdeps_dir", - join(get_project_dir(), ".piolibdeps")) - - def get_projectsrc_dir(): return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) diff --git a/platformio/util.py b/platformio/util.py index a043abd9..304d3a14 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -41,7 +41,8 @@ from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectboards_dir, get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, - get_projectsrc_dir, get_projecttest_dir, is_platformio_project) + get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir, + is_platformio_project) class cd(object):