Improve a work in off-line mode

HISTORY.rst

@@ -15,6 +15,7 @@ PlatformIO 3.0
 * Renamed ``envs_dir`` option to ``build_dir``
   in `Project Configuration File "platformio.ini" <http://docs.platformio.org/page/projectconf/section_platformio.html#build-dir>`__
 * Improved support of PIO Unified Debugger for Eclipse Oxygen
+* Improved work in off-line mode
 * Fixed project generator for CLion IDE
 * Fixed PIO Unified Debugger for mbed framework
 * Fixed library updates when a version is declared in VCS format (not SemVer)

platformio/builder/tools/piolib.py

@@ -228,19 +228,6 @@ class LibBuilderBase(object):
         except (AssertionError, ValueError):
             return LibBuilderBase.COMPAT_MODE_DEFAULT
 
-    @staticmethod
-    def items_to_list(items):
-        if not isinstance(items, list):
-            items = [i.strip() for i in items.split(",")]
-        return [i.lower() for i in items if i]
-
-    def items_in_list(self, items, ilist):
-        items = self.items_to_list(items)
-        ilist = self.items_to_list(ilist)
-        if "*" in items or "*" in ilist:
-            return True
-        return set(items) & set(ilist)
-
     def is_platforms_compatible(self, platforms):
         return True
 
@@ -273,7 +260,7 @@ class LibBuilderBase(object):
                 if env_key not in self.env:
                     continue
                 if (key in item and
-                        not self.items_in_list(self.env[env_key], item[key])):
+                        not util.items_in_list(self.env[env_key], item[key])):
                     if self.verbose:
                         sys.stderr.write(
                             "Skip %s incompatible dependency %s\n" % (key[:-1],
@@ -496,7 +483,7 @@ class ArduinoLibBuilder(LibBuilderBase):
         return src_filter
 
     def is_frameworks_compatible(self, frameworks):
-        return self.items_in_list(frameworks, ["arduino", "energia"])
+        return util.items_in_list(frameworks, ["arduino", "energia"])
 
 
 class MbedLibBuilder(LibBuilderBase):
@@ -527,7 +514,7 @@ class MbedLibBuilder(LibBuilderBase):
         return include_dirs
 
     def is_frameworks_compatible(self, frameworks):
-        return self.items_in_list(frameworks, ["mbed"])
+        return util.items_in_list(frameworks, ["mbed"])
 
 
 class PlatformIOLibBuilder(LibBuilderBase):
@@ -541,7 +528,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
         if "platforms" in manifest:
             manifest['platforms'] = [
                 "espressif8266" if p == "espressif" else p
-                for p in self.items_to_list(manifest['platforms'])
+                for p in util.items_to_list(manifest['platforms'])
             ]
 
         return manifest
@@ -610,13 +597,13 @@ class PlatformIOLibBuilder(LibBuilderBase):
         items = self._manifest.get("platforms")
         if not items:
             return LibBuilderBase.is_platforms_compatible(self, platforms)
-        return self.items_in_list(platforms, items)
+        return util.items_in_list(platforms, items)
 
     def is_frameworks_compatible(self, frameworks):
         items = self._manifest.get("frameworks")
         if not items:
             return LibBuilderBase.is_frameworks_compatible(self, frameworks)
-        return self.items_in_list(frameworks, items)
+        return util.items_in_list(frameworks, items)
 
     def get_include_dirs(self):
         include_dirs = LibBuilderBase.get_include_dirs(self)

platformio/commands/lib.py

@@ -23,8 +23,7 @@ import arrow
 import click
 
 from platformio import exception, util
-from platformio.managers.lib import LibraryManager
-from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.managers.lib import LibraryManager, get_builtin_libs
 from platformio.util import get_api_result
 
 
@@ -99,7 +98,7 @@ def cli(ctx, **options):
     help="Reinstall/redownload library if exists")
 @click.pass_obj
 def lib_install(lm, libraries, silent, interactive, force):
-    # @TODO "save" option
+    # @TODO: "save" option
     for library in libraries:
         lm.install(
             library, silent=silent, interactive=interactive, force=force)
@@ -280,25 +279,6 @@ def lib_list(lm, json_output):
     return True
 
 
-@util.memoized
-def get_builtin_libs(storage_names=None):
-    items = []
-    storage_names = storage_names or []
-    pm = PlatformManager()
-    for manifest in pm.get_installed():
-        p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
-        for storage in p.get_lib_storages():
-            if storage_names and storage['name'] not in storage_names:
-                continue
-            lm = LibraryManager(storage['path'])
-            items.append({
-                "name": storage['name'],
-                "path": storage['path'],
-                "items": lm.get_installed()
-            })
-    return items
-
-
 @cli.command("builtin", short_help="List built-in libraries")
 @click.option("--storage", multiple=True)
 @click.option("--json-output", is_flag=True)
@@ -326,8 +306,13 @@ def lib_builtin(storage, json_output):
 def lib_show(library, json_output):
     lm = LibraryManager()
     name, requirements, _ = lm.parse_pkg_uri(library)
-    lib_id = lm.get_pkg_id_by_name(
-        name, requirements, silent=json_output, interactive=not json_output)
+    lib_id = lm.search_lib_id(
+        {
+            "name": name,
+            "requirements": requirements
+        },
+        silent=json_output,
+        interactive=not json_output)
     lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:
         return click.echo(json.dumps(lib))

platformio/commands/run.py

@@ -23,10 +23,9 @@ import click
 from platformio import __version__, exception, telemetry, util
 from platformio.commands.device import device_monitor as cmd_device_monitor
 from platformio.commands.lib import lib_install as cmd_lib_install
-from platformio.commands.lib import get_builtin_libs
 from platformio.commands.platform import \
     platform_install as cmd_platform_install
-from platformio.managers.lib import LibraryManager
+from platformio.managers.lib import LibraryManager, is_builtin_lib
 from platformio.managers.platform import PlatformFactory
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
@@ -309,15 +308,10 @@ def _autoinstall_libdeps(ctx, libraries, verbose=False):
         try:
             ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose)
         except exception.LibNotFound as e:
-            if not _is_builtin_lib(lib):
+            if verbose or not is_builtin_lib(lib):
                 click.secho("Warning! %s" % e, fg="yellow")
-
-
-def _is_builtin_lib(lib_name):
-    for storage in get_builtin_libs():
-        if any([l.get("name") == lib_name for l in storage['items']]):
-            return True
-    return False
+        except exception.InternetIsOffline as e:
+            click.secho(str(e), fg="yellow")
 
 
 def _clean_build_dir(build_dir):

platformio/managers/lib.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 # pylint: disable=too-many-arguments, too-many-locals, too-many-branches
+# pylint: disable=too-many-return-statements
 
 import json
 import re
@@ -24,6 +25,7 @@ import click
 
 from platformio import app, commands, exception, util
 from platformio.managers.package import BasePkgManager
+from platformio.managers.platform import PlatformFactory, PlatformManager
 
 
 class LibraryManager(BasePkgManager):
@@ -186,29 +188,15 @@ class LibraryManager(BasePkgManager):
     def get_latest_repo_version(self, name, requirements, silent=False):
         item = self.max_satisfying_repo_version(
             util.get_api_result(
-                "/lib/info/%d" % self.get_pkg_id_by_name(
-                    name, requirements, silent=silent),
+                "/lib/info/%d" % self.search_lib_id(
+                    {
+                        "name": name,
+                        "requirements": requirements
+                    },
+                    silent=silent),
                 cache_valid="1h")['versions'], requirements)
         return item['name'] if item else None
 
-    def get_pkg_id_by_name(self,
-                           name,
-                           requirements,
-                           silent=False,
-                           interactive=False):
-        if name.startswith("id="):
-            return int(name[3:])
-        # try to find ID from installed packages
-        package_dir = self.get_package_dir(name, requirements)
-        if package_dir:
-            manifest = self.load_manifest(package_dir)
-            if "id" in manifest:
-                return int(manifest['id'])
-        return int(
-            self.search_for_library({
-                "name": name
-            }, silent, interactive)['id'])
-
     def _install_from_piorepo(self, name, requirements):
         assert name.startswith("id="), name
         version = self.get_latest_repo_version(name, requirements)
@@ -225,88 +213,20 @@ class LibraryManager(BasePkgManager):
             "http://", "https://") if app.get_setting("enable_ssl") else
                                       dl_data['url'], requirements)
 
-    def install(  # pylint: disable=arguments-differ
+    def search_lib_id(  # pylint: disable=too-many-branches
             self,
-            name,
-            requirements=None,
-            silent=False,
-            trigger_event=True,
-            interactive=False,
-            force=False):
-        pkg_dir = None
-        try:
-            _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
-            if not _url:
-                name = "id=%d" % self.get_pkg_id_by_name(
-                    _name,
-                    _requirements,
-                    silent=silent,
-                    interactive=interactive)
-                requirements = _requirements
-            pkg_dir = BasePkgManager.install(
-                self,
-                name,
-                requirements,
-                silent=silent,
-                trigger_event=trigger_event,
-                force=force)
-        except exception.InternetIsOffline as e:
-            if not silent:
-                click.secho(str(e), fg="yellow")
-            return None
-
-        if not pkg_dir:
-            return None
-
-        manifest = self.load_manifest(pkg_dir)
-        if "dependencies" not in manifest:
-            return pkg_dir
-
-        if not silent:
-            click.secho("Installing dependencies", fg="yellow")
-
-        for filters in self.normalize_dependencies(manifest['dependencies']):
-            assert "name" in filters
-            if any([s in filters.get("version", "") for s in ("\\", "/")]):
-                self.install(
-                    "{name}={version}".format(**filters),
-                    silent=silent,
-                    trigger_event=trigger_event,
-                    interactive=interactive,
-                    force=force)
-            else:
-                try:
-                    lib_info = self.search_for_library(filters, silent,
-                                                       interactive)
-                except exception.LibNotFound as e:
-                    if not silent:
-                        click.secho("Warning! %s" % e, fg="yellow")
-                    continue
-
-                if filters.get("version"):
-                    self.install(
-                        lib_info['id'],
-                        filters.get("version"),
-                        silent=silent,
-                        trigger_event=trigger_event,
-                        interactive=interactive,
-                        force=force)
-                else:
-                    self.install(
-                        lib_info['id'],
-                        silent=silent,
-                        trigger_event=trigger_event,
-                        interactive=interactive,
-                        force=force)
-        return pkg_dir
-
-    @staticmethod
-    def search_for_library(  # pylint: disable=too-many-branches
             filters,
             silent=False,
             interactive=False):
         assert isinstance(filters, dict)
         assert "name" in filters
+
+        # try to find ID within installed packages
+        lib_id = self._get_lib_id_from_installed(filters)
+        if lib_id:
+            return lib_id
+
+        # looking in PIO Library Registry
         if not silent:
             click.echo("Looking for %s library in registry" % click.style(
                 filters['name'], fg="cyan"))
@@ -366,4 +286,141 @@ class LibraryManager(BasePkgManager):
                 "http://platformio.org/lib/show/{id}/{name}".format(
                     **lib_info),
                 fg="blue"))
-        return lib_info
+        return int(lib_info['id'])
+
+    def _get_lib_id_from_installed(self, filters):
+        if filters['name'].startswith("id="):
+            return int(filters['name'][3:])
+        package_dir = self.get_package_dir(filters['name'],
+                                           filters.get("requirements",
+                                                       filters.get("version")))
+        if not package_dir:
+            return None
+        manifest = self.load_manifest(package_dir)
+        if "id" not in manifest:
+            return None
+
+        for key in ("frameworks", "platforms"):
+            if key not in filters:
+                continue
+            if key not in manifest:
+                return None
+            if not util.items_in_list(
+                    util.items_to_list(filters[key]),
+                    util.items_to_list(manifest[key])):
+                return None
+
+        if "authors" in filters:
+            if "authors" not in manifest:
+                return None
+            manifest_authors = manifest['authors']
+            if not isinstance(manifest_authors, list):
+                manifest_authors = [manifest_authors]
+            manifest_authors = [
+                a['name'] for a in manifest_authors
+                if isinstance(a, dict) and "name" in a
+            ]
+            filter_authors = filters['authors']
+            if not isinstance(filter_authors, list):
+                filter_authors = [filter_authors]
+            if not set(filter_authors) <= set(manifest_authors):
+                return None
+
+        return int(manifest['id'])
+
+    def install(  # pylint: disable=arguments-differ
+            self,
+            name,
+            requirements=None,
+            silent=False,
+            trigger_event=True,
+            interactive=False,
+            force=False):
+        _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
+        if not _url:
+            name = "id=%d" % self.search_lib_id(
+                {
+                    "name": _name,
+                    "requirements": _requirements
+                },
+                silent=silent,
+                interactive=interactive)
+            requirements = _requirements
+        pkg_dir = BasePkgManager.install(
+            self,
+            name,
+            requirements,
+            silent=silent,
+            trigger_event=trigger_event,
+            force=force)
+
+        if not pkg_dir:
+            return None
+
+        manifest = self.load_manifest(pkg_dir)
+        if "dependencies" not in manifest:
+            return pkg_dir
+
+        if not silent:
+            click.secho("Installing dependencies", fg="yellow")
+
+        for filters in self.normalize_dependencies(manifest['dependencies']):
+            assert "name" in filters
+            if any([s in filters.get("version", "") for s in ("\\", "/")]):
+                self.install(
+                    "{name}={version}".format(**filters),
+                    silent=silent,
+                    trigger_event=trigger_event,
+                    interactive=interactive,
+                    force=force)
+            else:
+                try:
+                    lib_id = self.search_lib_id(filters, silent, interactive)
+                except exception.LibNotFound as e:
+                    if not silent or is_builtin_lib(filters['name']):
+                        click.secho("Warning! %s" % e, fg="yellow")
+                    continue
+
+                if filters.get("version"):
+                    self.install(
+                        lib_id,
+                        filters.get("version"),
+                        silent=silent,
+                        trigger_event=trigger_event,
+                        interactive=interactive,
+                        force=force)
+                else:
+                    self.install(
+                        lib_id,
+                        silent=silent,
+                        trigger_event=trigger_event,
+                        interactive=interactive,
+                        force=force)
+        return pkg_dir
+
+
+@util.memoized
+def get_builtin_libs(storage_names=None):
+    items = []
+    storage_names = storage_names or []
+    pm = PlatformManager()
+    for manifest in pm.get_installed():
+        p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
+        for storage in p.get_lib_storages():
+            if storage_names and storage['name'] not in storage_names:
+                continue
+            lm = LibraryManager(storage['path'])
+            items.append({
+                "name": storage['name'],
+                "path": storage['path'],
+                "items": lm.get_installed()
+            })
+    return items
+
+
+@util.memoized
+def is_builtin_lib(name):
+    for storage in get_builtin_libs():
+        if any([l.get("name") == name for l in storage['items']]):
+            return True
+    return False

platformio/util.py

@@ -723,6 +723,20 @@ def pepver_to_semver(pepver):
     return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1)
 
 
+def items_to_list(items):
+    if not isinstance(items, list):
+        items = [i.strip() for i in items.split(",")]
+    return [i.lower() for i in items if i]
+
+
+def items_in_list(needle, haystack):
+    needle = items_to_list(needle)
+    haystack = items_to_list(haystack)
+    if "*" in needle or "*" in haystack:
+        return True
+    return set(needle) & set(haystack)
+
+
 def rmtree_(path):
 
     def _onerror(_, name, __):

tests/conftest.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+
 import pytest
 from click.testing import CliRunner