feedc0de/platformio-core
1
0
Fork 0
forked from platformio/platformio-core
Code Pull Requests Activity

Use isolated library dependency storage per project build environment // Resolve #1696

Browse Source
This commit is contained in:
Ivan Kravets
2019-05-23 00:23:24 +03:00
parent e7d75d1412
commit 21e2ac6695
12 changed files with 207 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
HISTORY.rst
View File

@ -7,14 +7,20 @@ PlatformIO 4.0
4.0.0 (2019-??-??)
~~~~~~~~~~~~~~~~~~
* **PlatformIO-based Project**
* **Project Management**
- Implemented unified project workspace storage (`workspace_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#workspace-dir>`__ -> ``.pio``) for PlatformIO Build System, Library Dependency Finder, and other internal services (`issue #1778 <https://github.com/platformio/platformio-core/issues/1778>`_)
- Unified workspace storage (`workspace_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#workspace-dir>`__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 <https://github.com/platformio/platformio-core/issues/1778>`_)
- Share common (global) options between build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ (`issue #1643 <https://github.com/platformio/platformio-core/issues/1643>`_)
- Include external configuration files in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ with `extra_configs <http://docs.platformio.org/page/projectconf/section_platformio.html#extra-configs>`__ option (`issue #1590 <https://github.com/platformio/platformio-core/issues/1590>`_)
- Override default `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ with a custom using ``-c, --project-conf`` option for `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__, `platformio debug <http://docs.platformio.org/page/userguide/cmd_debug.html>`__, or `platformio test <http://docs.platformio.org/page/userguide/cmd_test.html>`__ commands (`issue #1913 <https://github.com/platformio/platformio-core/issues/1913>`_)
- Custom project ``***_dir`` options declared in "platformio" section of `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ have higher priority than `Environment variables <http://docs.platformio.org/page/envvars.html>`__
- Moved ``.pioenvs`` build directory to workspace storage ``.pio/build``
- Use workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs``
* **Library Management**
- Use isolated library dependency storage per project build environment (`issue #1696 <https://github.com/platformio/platformio-core/issues/1696>`_)
- Override default source and include directories for a library via `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__ manifest using ``includeDir`` and ``srcDir`` fields
- Use workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps``
* **Infrastructure**
@ -24,7 +30,6 @@ PlatformIO 4.0
* **Miscellaneous**
- Override default source and include directories for a library via `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__ manifest using ``includeDir`` and ``srcDir`` fields
- Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead
PlatformIO 3.0

2
docs

Submodule docs updated: 103ed8445c...bbf0f91e9f

3
platformio/builder/main.py
View File

@ -110,6 +110,7 @@ DEFAULT_ENV_OPTIONS = dict(
PIOHOME_DIR=util.get_home_dir(),
PROJECT_DIR=get_project_dir(),
PROJECTWORKSPACE_DIR=get_projectworkspace_dir(),
PROJECTLIBDEPS_DIR=get_projectlibdeps_dir(),
PROJECTINCLUDE_DIR=get_projectinclude_dir(),
PROJECTSRC_DIR=get_projectsrc_dir(),
PROJECTTEST_DIR=get_projecttest_dir(),
@ -121,7 +122,7 @@ DEFAULT_ENV_OPTIONS = dict(
LIBPATH=["$BUILD_DIR"],
LIBSOURCE_DIRS=[
get_projectlib_dir(),
get_projectlibdeps_dir(),
join("$PROJECTLIBDEPS_DIR", "$PIOENV"),
join("$PIOHOME_DIR", "lib")
],
PROGNAME="program",

248
platformio/commands/lib.py
View File

@ -21,9 +21,9 @@ from os.path import isdir, join
import click
from platformio import exception, util
from platformio.commands import PlatformioCLI
from platformio.managers.lib import LibraryManager, get_builtin_libs
from platformio.proc import is_ci
from platformio.project.config import ProjectConfig
from platformio.project.helpers import (
get_project_dir, get_projectlibdeps_dir, is_platformio_project)
@ -34,14 +34,10 @@ except ImportError:
@click.group(short_help="Library Manager")
@click.option(
"-g",
"--global",
is_flag=True,
help="Manage global PlatformIO library storage")
@click.option(
"-d",
"--storage-dir",
multiple=True,
default=None,
type=click.Path(
exists=True,
@ -50,38 +46,56 @@ except ImportError:
writable=True,
resolve_path=True),
help="Manage custom library storage")
@click.option(
"-g",
"--global",
is_flag=True,
help="Manage global PlatformIO library storage")
@click.option(
"-e",
"--environment",
multiple=True,
help=("Manage libraries for the specific project build environments "
"declared in `platformio.ini`"))
@click.pass_context
def cli(ctx, **options):
non_storage_cmds = ("search", "show", "register", "stats", "builtin")
storage_cmds = ("install", "uninstall", "update", "list")
# skip commands that don't need storage folder
if ctx.invoked_subcommand in non_storage_cmds or \
if ctx.invoked_subcommand not in storage_cmds or \
(len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")):
return
storage_dir = options['storage_dir']
if not storage_dir:
if options['global']:
storage_dir = join(util.get_home_dir(), "lib")
elif is_platformio_project():
storage_dir = get_projectlibdeps_dir()
storage_dirs = list(options['storage_dir'])
if options['global']:
storage_dirs.append(join(util.get_home_dir(), "lib"))
if not storage_dirs:
if is_platformio_project():
storage_dirs = [get_project_dir()]
elif is_ci():
storage_dir = join(util.get_home_dir(), "lib")
storage_dirs = [join(util.get_home_dir(), "lib")]
click.secho(
"Warning! Global library storage is used automatically. "
"Please use `platformio lib --global %s` command to remove "
"this warning." % ctx.invoked_subcommand,
fg="yellow")
elif is_platformio_project(storage_dir):
with util.cd(storage_dir):
storage_dir = get_projectlibdeps_dir()
if not storage_dir and not is_platformio_project():
if not storage_dirs:
raise exception.NotGlobalLibDir(get_project_dir(),
join(util.get_home_dir(), "lib"),
ctx.invoked_subcommand)
ctx.obj = LibraryManager(storage_dir)
if not PlatformioCLI.in_silence():
click.echo("Library Storage: " + storage_dir)
ctx.obj = []
for storage_dir in storage_dirs:
if is_platformio_project(storage_dir):
with util.cd(storage_dir):
config = ProjectConfig.get_instance(
join(storage_dir, "platformio.ini"))
config.validate(options['environment'])
libdeps_dir = get_projectlibdeps_dir()
for env in config.envs():
if (not options['environment']
or env in options['environment']):
ctx.obj.append(join(libdeps_dir, env))
else:
ctx.obj.append(storage_dir)
@cli.command("install", short_help="Install library")
@ -103,19 +117,24 @@ def cli(ctx, **options):
is_flag=True,
help="Reinstall/redownload library if exists")
@click.pass_obj
def lib_install(lm, libraries, silent, interactive, force):
# @TODO: "save" option
for library in libraries:
lm.install(
library, silent=silent, interactive=interactive, force=force)
def lib_install(storage_dirs, libraries, silent, interactive, force):
for storage_dir in storage_dirs:
print_storage_header(storage_dirs, storage_dir)
lm = LibraryManager(storage_dir)
for library in libraries:
lm.install(
library, silent=silent, interactive=interactive, force=force)
@cli.command("uninstall", short_help="Uninstall libraries")
@click.argument("libraries", nargs=-1, metavar="[LIBRARY...]")
@click.pass_obj
def lib_uninstall(lm, libraries):
for library in libraries:
lm.uninstall(library)
def lib_uninstall(storage_dirs, libraries):
for storage_dir in storage_dirs:
print_storage_header(storage_dirs, storage_dir)
lm = LibraryManager(storage_dir)
for library in libraries:
lm.uninstall(library)
@cli.command("update", short_help="Update installed libraries")
@ -131,68 +150,72 @@ def lib_uninstall(lm, libraries):
help="Do not update, only check for the new versions")
@click.option("--json-output", is_flag=True)
@click.pass_obj
def lib_update(lm, libraries, only_check, dry_run, json_output):
if not libraries:
libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()]
def lib_update(storage_dirs, libraries, only_check, dry_run, json_output):
only_check = dry_run or only_check
json_result = {}
for storage_dir in storage_dirs:
if not json_output:
print_storage_header(storage_dirs, storage_dir)
lm = LibraryManager(storage_dir)
if only_check and json_output:
result = []
for library in libraries:
pkg_dir = library if isdir(library) else None
requirements = None
url = None
if not pkg_dir:
name, requirements, url = lm.parse_pkg_uri(library)
pkg_dir = lm.get_package_dir(name, requirements, url)
if not pkg_dir:
continue
latest = lm.outdated(pkg_dir, requirements)
if not latest:
continue
manifest = lm.load_manifest(pkg_dir)
manifest['versionLatest'] = latest
result.append(manifest)
return click.echo(json.dumps(result))
_libraries = libraries
if not _libraries:
_libraries = [
manifest['__pkg_dir'] for manifest in lm.get_installed()
]
for library in libraries:
lm.update(library, only_check=only_check)
if only_check and json_output:
result = []
for library in _libraries:
pkg_dir = library if isdir(library) else None
requirements = None
url = None
if not pkg_dir:
name, requirements, url = lm.parse_pkg_uri(library)
pkg_dir = lm.get_package_dir(name, requirements, url)
if not pkg_dir:
continue
latest = lm.outdated(pkg_dir, requirements)
if not latest:
continue
manifest = lm.load_manifest(pkg_dir)
manifest['versionLatest'] = latest
result.append(manifest)
json_result[storage_dir] = result
else:
for library in _libraries:
lm.update(library, only_check=only_check)
if json_output:
return click.echo(
json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) ==
1 else json_result))
return True
def print_lib_item(item):
click.secho(item['name'], fg="cyan")
click.echo("=" * len(item['name']))
if "id" in item:
click.secho("#ID: %d" % item['id'], bold=True)
if "description" in item or "url" in item:
click.echo(item.get("description", item.get("url", "")))
click.echo()
for key in ("version", "homepage", "license", "keywords"):
if key not in item or not item[key]:
continue
if isinstance(item[key], list):
click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
@cli.command("list", short_help="List installed libraries")
@click.option("--json-output", is_flag=True)
@click.pass_obj
def lib_list(storage_dirs, json_output):
json_result = {}
for storage_dir in storage_dirs:
if not json_output:
print_storage_header(storage_dirs, storage_dir)
lm = LibraryManager(storage_dir)
items = lm.get_installed()
if json_output:
json_result[storage_dir] = items
else:
click.echo("%s: %s" % (key.title(), item[key]))
for item in sorted(items, key=lambda i: i['name']):
print_lib_item(item)
for key in ("frameworks", "platforms"):
if key not in item:
continue
click.echo("Compatible %s: %s" % (key, ", ".join(
[i['title'] if isinstance(i, dict) else i for i in item[key]])))
if json_output:
return click.echo(
json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) ==
1 else json_result))
if "authors" in item or "authornames" in item:
click.echo("Authors: %s" % ", ".join(
item.get("authornames",
[a.get("name", "") for a in item.get("authors", [])])))
if "__src_url" in item:
click.secho("Source: %s" % item['__src_url'])
click.echo()
return True
@cli.command("search", short_help="Search for a library")
@ -275,24 +298,6 @@ def lib_search(query, json_output, page, noninteractive, **filters):
cache_valid="1d")
@cli.command("list", short_help="List installed libraries")
@click.option("--json-output", is_flag=True)
@click.pass_obj
def lib_list(lm, json_output):
items = lm.get_installed()
if json_output:
return click.echo(json.dumps(items))
if not items:
return None
for item in sorted(items, key=lambda i: i['name']):
print_lib_item(item)
return True
@cli.command("builtin", short_help="List built-in libraries")
@click.option("--storage", multiple=True)
@click.option("--json-output", is_flag=True)
@ -484,3 +489,44 @@ def lib_stats(json_output):
click.echo()
return True
def print_storage_header(storage_dirs, storage_dir):
if storage_dirs and storage_dirs[0] != storage_dir:
click.echo("")
click.echo(
click.style("Library Storage: ", bold=True) +
click.style(storage_dir, fg="blue"))
def print_lib_item(item):
click.secho(item['name'], fg="cyan")
click.echo("=" * len(item['name']))
if "id" in item:
click.secho("#ID: %d" % item['id'], bold=True)
if "description" in item or "url" in item:
click.echo(item.get("description", item.get("url", "")))
click.echo()
for key in ("version", "homepage", "license", "keywords"):
if key not in item or not item[key]:
continue
if isinstance(item[key], list):
click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
else:
click.echo("%s: %s" % (key.title(), item[key]))
for key in ("frameworks", "platforms"):
if key not in item:
continue
click.echo("Compatible %s: %s" % (key, ", ".join(
[i['title'] if isinstance(i, dict) else i for i in item[key]])))
if "authors" in item or "authornames" in item:
click.echo("Authors: %s" % ", ".join(
item.get("authornames",
[a.get("name", "") for a in item.get("authors", [])])))
if "__src_url" in item:
click.secho("Source: %s" % item['__src_url'])
click.echo()

39
platformio/commands/run.py
View File

@ -23,7 +23,7 @@ from platformio.commands.device import device_monitor as cmd_device_monitor
from platformio.commands.lib import lib_install as cmd_lib_install
from platformio.commands.platform import \
platform_install as cmd_platform_install
from platformio.managers.lib import LibraryManager, is_builtin_lib
from platformio.managers.lib import is_builtin_lib
from platformio.managers.platform import PlatformFactory
from platformio.project.config import ProjectConfig
from platformio.project.helpers import (
@ -82,6 +82,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf,
project_conf or join(project_dir, "platformio.ini"))
config.validate(environment)
_handle_legacy_libdeps(project_dir, config)
results = []
start_time = time()
default_envs = config.default_envs()
@ -94,7 +96,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf,
results.append((envname, None))
continue
if not silent and results:
if not silent and any(
status is not None for (_, status) in results):
click.echo()
options = config.items(env=envname, as_dict=True)
@ -222,14 +225,14 @@ class EnvironmentProcessor(object):
if "nobuild" not in build_targets:
# install dependent libraries
if "lib_install" in self.options:
_autoinstall_libdeps(self.cmd_ctx, [
_autoinstall_libdeps(self.cmd_ctx, self.name, [
int(d.strip())
for d in self.options['lib_install'].split(",")
if d.strip()
], self.verbose)
if "lib_deps" in self.options:
_autoinstall_libdeps(
self.cmd_ctx,
self.cmd_ctx, self.name,
ProjectConfig.parse_multi_values(self.options['lib_deps']),
self.verbose)
@ -245,13 +248,31 @@ class EnvironmentProcessor(object):
return p.run(build_vars, build_targets, self.silent, self.verbose)
def _autoinstall_libdeps(ctx, libraries, verbose=False):
def _handle_legacy_libdeps(project_dir, config):
legacy_libdeps_dir = join(project_dir, ".piolibdeps")
if (not isdir(legacy_libdeps_dir)
or legacy_libdeps_dir == get_projectlibdeps_dir()):
return
if not config.has_section("env"):
config.add_section("env")
lib_extra_dirs = []
if config.has_option("env", "lib_extra_dirs"):
lib_extra_dirs = config.getlist("env", "lib_extra_dirs")
lib_extra_dirs.append(legacy_libdeps_dir)
config.set("env", "lib_extra_dirs", lib_extra_dirs)
click.secho(
"DEPRECATED! A legacy library storage `{0}` has been found in a "
"project. \nPlease declare project dependencies in `platformio.ini`"
" file using `lib_deps` option and remove `{0}` folder."
"\nMore details -> http://docs.platformio.org/page/projectconf/"
"section_env_library.html#lib-deps".format(legacy_libdeps_dir),
fg="yellow")
def _autoinstall_libdeps(ctx, envname, libraries, verbose=False):
if not libraries:
return
storage_dir = get_projectlibdeps_dir()
ctx.obj = LibraryManager(storage_dir)
if verbose:
click.echo("Library Storage: " + storage_dir)
ctx.obj = [join(get_projectlibdeps_dir(), envname)]
for lib in libraries:
try:
ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose)

2
platformio/commands/update.py
View File

@ -54,5 +54,5 @@ def cli(ctx, core_packages, only_check, dry_run):
click.echo()
click.echo("Library Manager")
click.echo("===============")
ctx.obj = LibraryManager()
ctx.obj = [LibraryManager().package_dir]
ctx.invoke(cmd_lib_update, only_check=only_check)

3
platformio/ide/projectgenerator.py
View File

@ -144,7 +144,8 @@ class ProjectGenerator(object):
"project_dir": self.project_dir,
"project_src_dir": get_projectsrc_dir(),
"project_lib_dir": get_projectlib_dir(),
"project_libdeps_dir": get_projectlibdeps_dir(),
"project_libdeps_dir": join(
get_projectlibdeps_dir(), self.env_name),
"systype": util.get_systype(),
"platformio_path": self._fix_os_path(
sys.argv[0] if isfile(sys.argv[0])

2
platformio/maintenance.py
View File

@ -332,7 +332,7 @@ def check_internal_updates(ctx, what):
if what == "platforms":
ctx.invoke(cmd_platform_update, platforms=outdated_items)
elif what == "libraries":
ctx.obj = pm
ctx.obj = [pm.package_dir]
ctx.invoke(cmd_lib_update, libraries=outdated_items)
click.echo()

4
platformio/managers/core.py
View File

@ -26,10 +26,10 @@ from platformio.managers.package import PackageManager
from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
CORE_PACKAGES = {
"contrib-piohome": "^2.0.1",
"contrib-piohome": "^2.1.0",
"contrib-pysite":
"~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]),
"tool-pioplus": "^2.2.0",
"tool-pioplus": "^2.3.0",
"tool-unity": "~1.20403.0",
"tool-scons": "~2.20501.7" if PY2 else "~3.30005.0"
}

5
platformio/project/config.py
View File

@ -227,6 +227,11 @@ class ProjectConfig(object):
return [(option, self.get(section, option))
for option in self.options(section)]
def set(self, section, option, value):
if isinstance(value, (list, tuple)):
value = "\n".join(value)
self._parser.set(section, option, value)
def get(self, section, option):
if not self.expand_interpolations:
return self._parser.get(section, option)

10
platformio/project/helpers.py
View File

@ -93,15 +93,15 @@ def get_projectbuild_dir(force=False):
return path
def get_projectlibdeps_dir():
return get_project_optional_dir(
"libdeps_dir", join(get_projectworkspace_dir(), "libdeps"))
def get_projectlib_dir():
return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib"))
def get_projectlibdeps_dir():
return get_project_optional_dir("libdeps_dir",
join(get_project_dir(), ".piolibdeps"))
def get_projectsrc_dir():
return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))

3
platformio/util.py
View File

@ -41,7 +41,8 @@ from platformio.project.config import ProjectConfig
from platformio.project.helpers import (
get_project_dir, get_project_optional_dir, get_projectboards_dir,
get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir,
get_projectsrc_dir, get_projecttest_dir, is_platformio_project)
get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir,
is_platformio_project)
class cd(object):