diff --git a/HISTORY.rst b/HISTORY.rst index 488c0dc0..d34f9974 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ PlatformIO 3.0 * Renamed ``envs_dir`` option to ``build_dir`` in `Project Configuration File "platformio.ini" `__ * Improved support of PIO Unified Debugger for Eclipse Oxygen +* Improved work in off-line mode * Fixed project generator for CLion IDE * Fixed PIO Unified Debugger for mbed framework * Fixed library updates when a version is declared in VCS format (not SemVer) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index a4bcd6ff..eb21d6c2 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -228,19 +228,6 @@ class LibBuilderBase(object): except (AssertionError, ValueError): return LibBuilderBase.COMPAT_MODE_DEFAULT - @staticmethod - def items_to_list(items): - if not isinstance(items, list): - items = [i.strip() for i in items.split(",")] - return [i.lower() for i in items if i] - - def items_in_list(self, items, ilist): - items = self.items_to_list(items) - ilist = self.items_to_list(ilist) - if "*" in items or "*" in ilist: - return True - return set(items) & set(ilist) - def is_platforms_compatible(self, platforms): return True @@ -273,7 +260,7 @@ class LibBuilderBase(object): if env_key not in self.env: continue if (key in item and - not self.items_in_list(self.env[env_key], item[key])): + not util.items_in_list(self.env[env_key], item[key])): if self.verbose: sys.stderr.write( "Skip %s incompatible dependency %s\n" % (key[:-1], @@ -496,7 +483,7 @@ class ArduinoLibBuilder(LibBuilderBase): return src_filter def is_frameworks_compatible(self, frameworks): - return self.items_in_list(frameworks, ["arduino", "energia"]) + return util.items_in_list(frameworks, ["arduino", "energia"]) class MbedLibBuilder(LibBuilderBase): @@ -527,7 +514,7 @@ class MbedLibBuilder(LibBuilderBase): return include_dirs def is_frameworks_compatible(self, frameworks): - return self.items_in_list(frameworks, ["mbed"]) + return util.items_in_list(frameworks, ["mbed"]) class PlatformIOLibBuilder(LibBuilderBase): @@ -541,7 +528,7 @@ class PlatformIOLibBuilder(LibBuilderBase): if "platforms" in manifest: manifest['platforms'] = [ "espressif8266" if p == "espressif" else p - for p in self.items_to_list(manifest['platforms']) + for p in util.items_to_list(manifest['platforms']) ] return manifest @@ -610,13 +597,13 @@ class PlatformIOLibBuilder(LibBuilderBase): items = self._manifest.get("platforms") if not items: return LibBuilderBase.is_platforms_compatible(self, platforms) - return self.items_in_list(platforms, items) + return util.items_in_list(platforms, items) def is_frameworks_compatible(self, frameworks): items = self._manifest.get("frameworks") if not items: return LibBuilderBase.is_frameworks_compatible(self, frameworks) - return self.items_in_list(frameworks, items) + return util.items_in_list(frameworks, items) def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 051720c6..6a1f90d0 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -23,8 +23,7 @@ import arrow import click from platformio import exception, util -from platformio.managers.lib import LibraryManager -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.lib import LibraryManager, get_builtin_libs from platformio.util import get_api_result @@ -99,7 +98,7 @@ def cli(ctx, **options): help="Reinstall/redownload library if exists") @click.pass_obj def lib_install(lm, libraries, silent, interactive, force): - # @TODO "save" option + # @TODO: "save" option for library in libraries: lm.install( library, silent=silent, interactive=interactive, force=force) @@ -280,25 +279,6 @@ def lib_list(lm, json_output): return True -@util.memoized -def get_builtin_libs(storage_names=None): - items = [] - storage_names = storage_names or [] - pm = PlatformManager() - for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest['__pkg_dir']) - for storage in p.get_lib_storages(): - if storage_names and storage['name'] not in storage_names: - continue - lm = LibraryManager(storage['path']) - items.append({ - "name": storage['name'], - "path": storage['path'], - "items": lm.get_installed() - }) - return items - - @cli.command("builtin", short_help="List built-in libraries") @click.option("--storage", multiple=True) @click.option("--json-output", is_flag=True) @@ -326,8 +306,13 @@ def lib_builtin(storage, json_output): def lib_show(library, json_output): lm = LibraryManager() name, requirements, _ = lm.parse_pkg_uri(library) - lib_id = lm.get_pkg_id_by_name( - name, requirements, silent=json_output, interactive=not json_output) + lib_id = lm.search_lib_id( + { + "name": name, + "requirements": requirements + }, + silent=json_output, + interactive=not json_output) lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: return click.echo(json.dumps(lib)) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index beba742e..f75a07c9 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -23,10 +23,9 @@ import click from platformio import __version__, exception, telemetry, util from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.commands.lib import lib_install as cmd_lib_install -from platformio.commands.lib import get_builtin_libs from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.managers.lib import LibraryManager +from platformio.managers.lib import LibraryManager, is_builtin_lib from platformio.managers.platform import PlatformFactory # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -309,15 +308,10 @@ def _autoinstall_libdeps(ctx, libraries, verbose=False): try: ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose) except exception.LibNotFound as e: - if not _is_builtin_lib(lib): + if verbose or not is_builtin_lib(lib): click.secho("Warning! %s" % e, fg="yellow") - - -def _is_builtin_lib(lib_name): - for storage in get_builtin_libs(): - if any([l.get("name") == lib_name for l in storage['items']]): - return True - return False + except exception.InternetIsOffline as e: + click.secho(str(e), fg="yellow") def _clean_build_dir(build_dir): diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index ef00787b..c4159d08 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -13,6 +13,7 @@ # limitations under the License. # pylint: disable=too-many-arguments, too-many-locals, too-many-branches +# pylint: disable=too-many-return-statements import json import re @@ -24,6 +25,7 @@ import click from platformio import app, commands, exception, util from platformio.managers.package import BasePkgManager +from platformio.managers.platform import PlatformFactory, PlatformManager class LibraryManager(BasePkgManager): @@ -186,29 +188,15 @@ class LibraryManager(BasePkgManager): def get_latest_repo_version(self, name, requirements, silent=False): item = self.max_satisfying_repo_version( util.get_api_result( - "/lib/info/%d" % self.get_pkg_id_by_name( - name, requirements, silent=silent), + "/lib/info/%d" % self.search_lib_id( + { + "name": name, + "requirements": requirements + }, + silent=silent), cache_valid="1h")['versions'], requirements) return item['name'] if item else None - def get_pkg_id_by_name(self, - name, - requirements, - silent=False, - interactive=False): - if name.startswith("id="): - return int(name[3:]) - # try to find ID from installed packages - package_dir = self.get_package_dir(name, requirements) - if package_dir: - manifest = self.load_manifest(package_dir) - if "id" in manifest: - return int(manifest['id']) - return int( - self.search_for_library({ - "name": name - }, silent, interactive)['id']) - def _install_from_piorepo(self, name, requirements): assert name.startswith("id="), name version = self.get_latest_repo_version(name, requirements) @@ -225,88 +213,20 @@ class LibraryManager(BasePkgManager): "http://", "https://") if app.get_setting("enable_ssl") else dl_data['url'], requirements) - def install( # pylint: disable=arguments-differ + def search_lib_id( # pylint: disable=too-many-branches self, - name, - requirements=None, - silent=False, - trigger_event=True, - interactive=False, - force=False): - pkg_dir = None - try: - _name, _requirements, _url = self.parse_pkg_uri(name, requirements) - if not _url: - name = "id=%d" % self.get_pkg_id_by_name( - _name, - _requirements, - silent=silent, - interactive=interactive) - requirements = _requirements - pkg_dir = BasePkgManager.install( - self, - name, - requirements, - silent=silent, - trigger_event=trigger_event, - force=force) - except exception.InternetIsOffline as e: - if not silent: - click.secho(str(e), fg="yellow") - return None - - if not pkg_dir: - return None - - manifest = self.load_manifest(pkg_dir) - if "dependencies" not in manifest: - return pkg_dir - - if not silent: - click.secho("Installing dependencies", fg="yellow") - - for filters in self.normalize_dependencies(manifest['dependencies']): - assert "name" in filters - if any([s in filters.get("version", "") for s in ("\\", "/")]): - self.install( - "{name}={version}".format(**filters), - silent=silent, - trigger_event=trigger_event, - interactive=interactive, - force=force) - else: - try: - lib_info = self.search_for_library(filters, silent, - interactive) - except exception.LibNotFound as e: - if not silent: - click.secho("Warning! %s" % e, fg="yellow") - continue - - if filters.get("version"): - self.install( - lib_info['id'], - filters.get("version"), - silent=silent, - trigger_event=trigger_event, - interactive=interactive, - force=force) - else: - self.install( - lib_info['id'], - silent=silent, - trigger_event=trigger_event, - interactive=interactive, - force=force) - return pkg_dir - - @staticmethod - def search_for_library( # pylint: disable=too-many-branches filters, silent=False, interactive=False): assert isinstance(filters, dict) assert "name" in filters + + # try to find ID within installed packages + lib_id = self._get_lib_id_from_installed(filters) + if lib_id: + return lib_id + + # looking in PIO Library Registry if not silent: click.echo("Looking for %s library in registry" % click.style( filters['name'], fg="cyan")) @@ -366,4 +286,141 @@ class LibraryManager(BasePkgManager): "http://platformio.org/lib/show/{id}/{name}".format( **lib_info), fg="blue")) - return lib_info + return int(lib_info['id']) + + def _get_lib_id_from_installed(self, filters): + if filters['name'].startswith("id="): + return int(filters['name'][3:]) + package_dir = self.get_package_dir(filters['name'], + filters.get("requirements", + filters.get("version"))) + if not package_dir: + return None + manifest = self.load_manifest(package_dir) + if "id" not in manifest: + return None + + for key in ("frameworks", "platforms"): + if key not in filters: + continue + if key not in manifest: + return None + if not util.items_in_list( + util.items_to_list(filters[key]), + util.items_to_list(manifest[key])): + return None + + if "authors" in filters: + if "authors" not in manifest: + return None + manifest_authors = manifest['authors'] + if not isinstance(manifest_authors, list): + manifest_authors = [manifest_authors] + manifest_authors = [ + a['name'] for a in manifest_authors + if isinstance(a, dict) and "name" in a + ] + filter_authors = filters['authors'] + if not isinstance(filter_authors, list): + filter_authors = [filter_authors] + if not set(filter_authors) <= set(manifest_authors): + return None + + return int(manifest['id']) + + def install( # pylint: disable=arguments-differ + self, + name, + requirements=None, + silent=False, + trigger_event=True, + interactive=False, + force=False): + _name, _requirements, _url = self.parse_pkg_uri(name, requirements) + if not _url: + name = "id=%d" % self.search_lib_id( + { + "name": _name, + "requirements": _requirements + }, + silent=silent, + interactive=interactive) + requirements = _requirements + pkg_dir = BasePkgManager.install( + self, + name, + requirements, + silent=silent, + trigger_event=trigger_event, + force=force) + + if not pkg_dir: + return None + + manifest = self.load_manifest(pkg_dir) + if "dependencies" not in manifest: + return pkg_dir + + if not silent: + click.secho("Installing dependencies", fg="yellow") + + for filters in self.normalize_dependencies(manifest['dependencies']): + assert "name" in filters + if any([s in filters.get("version", "") for s in ("\\", "/")]): + self.install( + "{name}={version}".format(**filters), + silent=silent, + trigger_event=trigger_event, + interactive=interactive, + force=force) + else: + try: + lib_id = self.search_lib_id(filters, silent, interactive) + except exception.LibNotFound as e: + if not silent or is_builtin_lib(filters['name']): + click.secho("Warning! %s" % e, fg="yellow") + continue + + if filters.get("version"): + self.install( + lib_id, + filters.get("version"), + silent=silent, + trigger_event=trigger_event, + interactive=interactive, + force=force) + else: + self.install( + lib_id, + silent=silent, + trigger_event=trigger_event, + interactive=interactive, + force=force) + return pkg_dir + + +@util.memoized +def get_builtin_libs(storage_names=None): + items = [] + storage_names = storage_names or [] + pm = PlatformManager() + for manifest in pm.get_installed(): + p = PlatformFactory.newPlatform(manifest['__pkg_dir']) + for storage in p.get_lib_storages(): + if storage_names and storage['name'] not in storage_names: + continue + lm = LibraryManager(storage['path']) + items.append({ + "name": storage['name'], + "path": storage['path'], + "items": lm.get_installed() + }) + return items + + +@util.memoized +def is_builtin_lib(name): + for storage in get_builtin_libs(): + if any([l.get("name") == name for l in storage['items']]): + return True + return False diff --git a/platformio/util.py b/platformio/util.py index 49fd1264..270952af 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -723,6 +723,20 @@ def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) +def items_to_list(items): + if not isinstance(items, list): + items = [i.strip() for i in items.split(",")] + return [i.lower() for i in items if i] + + +def items_in_list(needle, haystack): + needle = items_to_list(needle) + haystack = items_to_list(haystack) + if "*" in needle or "*" in haystack: + return True + return set(needle) & set(haystack) + + def rmtree_(path): def _onerror(_, name, __): diff --git a/tests/conftest.py b/tests/conftest.py index 2ed8b41e..5b8103c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ # limitations under the License. import os + import pytest from click.testing import CliRunner