List built-in libraries based on development platforms with pio lib builtin command

HISTORY.rst

@@ -14,6 +14,8 @@ PlatformIO 3.0
   - Recent and popular keywords
   - Featured libraries (today, week, month)
 
+* List built-in libraries based on development platforms with
+  `pio lib builtin <http://docs.platformio.org/page/userguide/lib/cmd_builtin.html>`__ command
 * Show detailed info about a library using `pio lib show <http://docs.platformio.org/page/userguide/lib/cmd_show.html>`__
   command
   (`issue #430 <https://github.com/platformio/platformio-core/issues/430>`_)

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 3, "0a2")
+VERSION = (3, 3, "0a3")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/commands/lib.py

@@ -24,6 +24,7 @@ import click
 
 from platformio import exception, util
 from platformio.managers.lib import LibraryManager
+from platformio.managers.platform import PlatformFactory, PlatformManager
 from platformio.util import get_api_result
 
 
@@ -47,8 +48,9 @@ from platformio.util import get_api_result
     help="Manage custom library storage")
 @click.pass_context
 def cli(ctx, **options):
+    non_storage_cmds = ("search", "show", "register", "stats", "builtin")
     # skip commands that don't need storage folder
-    if ctx.invoked_subcommand in ("search", "show", "register", "stats") or \
+    if ctx.invoked_subcommand in non_storage_cmds or \
             (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")):
         return
     storage_dir = options['storage_dir']
@@ -123,10 +125,10 @@ def print_lib_item(item):
     click.echo("=" * len(item['name']))
     if "id" in item:
         click.secho("#ID: %d" % item['id'], bold=True)
-    click.echo(item.get("description", item.get("url", "")).encode("utf-8"))
+    click.echo(item.get("description", item.get("url", "")))
     click.echo()
 
-    for key in ("homepage", "license", "keywords"):
+    for key in ("version", "homepage", "license", "keywords"):
         if key not in item or not item[key]:
             continue
         if isinstance(item[key], list):
@@ -242,6 +244,38 @@ def lib_list(lm, json_output):
         print_lib_item(item)
 
 
+@cli.command("builtin", short_help="List built-in libraries")
+@click.option("--storage", multiple=True)
+@click.option("--json-output", is_flag=True)
+@click.pass_obj
+def lib_builtin(lm, storage, json_output):
+    items = []
+    storage_names = storage or []
+    del storage
+    pm = PlatformManager()
+    for manifest in pm.get_installed():
+        p = PlatformFactory.newPlatform(
+            pm.get_manifest_path(manifest['__pkg_dir']))
+        for storage in p.get_lib_storages():
+            if storage_names and storage['name'] not in storage_names:
+                continue
+            lm = LibraryManager(storage['path'])
+            items.append(dict(name=storage['name'], items=lm.get_installed()))
+
+    if json_output:
+        return click.echo(json.dumps(items))
+
+    for storage in items:
+        if not storage['items']:
+            continue
+        click.secho(storage['name'], fg="green")
+        click.echo("*" * len(storage['name']))
+        click.echo()
+
+        for item in sorted(storage['items'], key=lambda i: i['name']):
+            print_lib_item(item)
+
+
 @cli.command("show", short_help="Show detailed info about a library")
 @click.argument("library", metavar="[LIBRARY]")
 @click.option("--json-output", is_flag=True)

platformio/exception.py

@@ -78,7 +78,7 @@ class UnknownPackage(PlatformioException):
 
 class MissingPackageManifest(PlatformioException):
 
-    MESSAGE = "Could not find '{0}' manifest file in the package"
+    MESSAGE = "Could not find one of '{0}' manifest files in the package"
 
 
 class UndefinedPackageVersion(PlatformioException):

platformio/managers/lib.py

@@ -16,8 +16,9 @@
 
 import json
 import os
+import re
 from hashlib import md5
-from os.path import dirname, join
+from os.path import join
 
 import arrow
 import click
@@ -35,70 +36,99 @@ class LibraryManager(BasePkgManager):
         BasePkgManager.__init__(self, package_dir)
 
     @property
-    def manifest_name(self):
+    def manifest_names(self):
-        return ".library.json"
+        return [
+            ".library.json", "library.properties", "library.json",
+            "module.json"
+        ]
 
     def check_pkg_structure(self, pkg_dir):
         try:
             return BasePkgManager.check_pkg_structure(self, pkg_dir)
         except exception.MissingPackageManifest:
             # we will generate manifest automatically
+            # if library doesn't contain any
             pass
 
         manifest = {
             "name": "Library_" + md5(pkg_dir).hexdigest()[:5],
             "version": "0.0.0"
         }
-        manifest_path = self._find_any_manifest(pkg_dir)
+        for root, dirs, files in os.walk(pkg_dir):
-        if manifest_path:
+            if len(dirs) == 1 and not files:
-            _manifest = self._parse_manifest(manifest_path)
+                manifest['name'] = dirs[0]
-            pkg_dir = dirname(manifest_path)
+                continue
-            for key in ("name", "version"):
+            if dirs or files:
-                if key not in _manifest:
+                pkg_dir = root
-                    _manifest[key] = manifest[key]
+                break
-            manifest = _manifest
-        else:
-            for root, dirs, files in os.walk(pkg_dir):
-                if len(dirs) == 1 and not files:
-                    manifest['name'] = dirs[0]
-                    continue
-                if dirs or files:
-                    pkg_dir = root
-                    break
 
-        with open(join(pkg_dir, self.manifest_name), "w") as fp:
+        with open(join(pkg_dir, self.manifest_names[0]), "w") as fp:
             json.dump(manifest, fp)
 
         return pkg_dir
 
-    @staticmethod
+    def load_manifest(self, path):
-    def _find_any_manifest(pkg_dir):
+        manifest = BasePkgManager.load_manifest(self, path)
-        manifests = ("library.json", "library.properties", "module.json")
+        if not manifest:
-        for root, _, files in os.walk(pkg_dir):
+            return manifest
-            for manifest in manifests:
-                if manifest in files:
-                    return join(root, manifest)
-        return None
 
-    @staticmethod
+        # if Arudino library.properties
-    def _parse_manifest(path):
+        if "sentence" in manifest:
-        manifest = {}
-        if path.endswith(".json"):
-            return util.load_json(path)
-        elif path.endswith("library.properties"):
-            with open(path) as fp:
-                for line in fp.readlines():
-                    if "=" not in line:
-                        continue
-                    key, value = line.split("=", 1)
-                    manifest[key.strip()] = value.strip()
             manifest['frameworks'] = ["arduino"]
-            if "author" in manifest:
+
-                manifest['authors'] = [{"name": manifest['author']}]
+        if "author" in manifest:
-                del manifest['author']
+            manifest['authors'] = [{"name": manifest['author']}]
-            if "sentence" in manifest:
+            del manifest['author']
-                manifest['description'] = manifest['sentence']
+
-                del manifest['sentence']
+        if "sentence" in manifest:
+            manifest['description'] = manifest['sentence']
+            del manifest['sentence']
+
+        if "keywords" not in manifest:
+            keywords = []
+            for keyword in re.split(r"[\s/]+",
+                                    manifest.get("category", "Uncategorized")):
+                keyword = keyword.strip()
+                if not keyword:
+                    continue
+                keywords.append(keyword.lower())
+            manifest['keywords'] = keywords
+            if "category" in manifest:
+                del manifest['category']
+
+        # don't replace VCS URL
+        if "url" in manifest and "description" in manifest:
+            manifest['homepage'] = manifest['url']
+            del manifest['url']
+
+        if "architectures" in manifest:
+            platforms = []
+            platforms_map = {
+                "avr": "atmelavr",
+                "sam": "atmelsam",
+                "samd": "atmelsam",
+                "esp8266": "espressif8266",
+                "arc32": "intel_arc32"
+            }
+            for arch in manifest['architectures'].split(","):
+                arch = arch.strip()
+                if arch == "*":
+                    platforms = "*"
+                    break
+                if arch in platforms_map:
+                    platforms.append(platforms_map[arch])
+            manifest['platforms'] = platforms
+            del manifest['architectures']
+
+        # convert listed items via comma to array
+        for key in ("keywords", "frameworks", "platforms"):
+            if key not in manifest or \
+                    not isinstance(manifest[key], basestring):
+                continue
+            manifest[key] = [
+                i.strip() for i in manifest[key].split(",") if i.strip()
+            ]
+
         return manifest
 
     @staticmethod
@@ -301,7 +331,7 @@ class LibraryManager(BasePkgManager):
                 click.secho(
                     "Conflict: More than one library has been found "
                     "by request %s:" % json.dumps(filters),
-                    fg="red",
+                    fg="yellow",
                     err=True)
                 for item in result['items']:
                     commands.lib.print_lib_item(item)

platformio/managers/package.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import json
 import os
 import shutil
@@ -122,10 +123,14 @@ class PkgInstallerMixin(object):
     def get_manifest_path(self, pkg_dir):
         if not isdir(pkg_dir):
             return None
-        manifest_path = join(pkg_dir, self.manifest_name)
+        manifest_path = self.get_vcs_manifest_path(pkg_dir)
-        if isfile(manifest_path):
+        if manifest_path:
             return manifest_path
-        return self.get_vcs_manifest_path(pkg_dir)
+        for name in self.manifest_names:
+            manifest_path = join(pkg_dir, name)
+            if isfile(manifest_path):
+                return manifest_path
+        return None
 
     def manifest_exists(self, pkg_dir):
         return self.get_manifest_path(pkg_dir) is not None
@@ -135,15 +140,27 @@ class PkgInstallerMixin(object):
         pkg_dir = path
         if isdir(path):
             path = self.get_manifest_path(path)
+            if not path:
+                return None
         else:
             pkg_dir = dirname(pkg_dir)
-        if path:
+
-            if isfile(path) and path.endswith(self.VCS_MANIFEST_NAME):
+        if isfile(path) and path.endswith(self.VCS_MANIFEST_NAME):
-                pkg_dir = dirname(dirname(path))
+            pkg_dir = dirname(dirname(path))
+
+        if path.endswith(".json"):
             manifest = util.load_json(path)
-            manifest['__pkg_dir'] = pkg_dir
+        else:
-            return manifest
+            manifest = {}
-        return None
+            with codecs.open(path, encoding="utf-8") as fp:
+                for line in fp.readlines():
+                    if "=" not in line:
+                        continue
+                    key, value = line.split("=", 1)
+                    manifest[key.strip()] = value.strip()
+        manifest['__pkg_dir'] = pkg_dir
+
+        return manifest
 
     def check_pkg_structure(self, pkg_dir):
         if self.manifest_exists(pkg_dir):
@@ -153,7 +170,7 @@ class PkgInstallerMixin(object):
             if self.manifest_exists(root):
                 return root
 
-        raise exception.MissingPackageManifest(self.manifest_name)
+        raise exception.MissingPackageManifest(", ".join(self.manifest_names))
 
     def _install_from_piorepo(self, name, requirements):
         pkg_dir = None
@@ -273,7 +290,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         assert isdir(self.package_dir)
 
     @property
-    def manifest_name(self):
+    def manifest_names(self):
         raise NotImplementedError()
 
     def download(self, url, dest_dir, sha1=None):
@@ -381,7 +398,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             manifest = self.load_manifest(pkg_dir)
             if not manifest:
                 continue
-            assert set(["name", "version"]) <= set(manifest.keys())
+            assert "name" in manifest
             items.append(manifest)
         BasePkgManager._INSTALLED_CACHE[self.package_dir] = items
         return items
@@ -605,5 +622,5 @@ class PackageManager(BasePkgManager):
     FILE_CACHE_VALID = None  # disable package caching
 
     @property
-    def manifest_name(self):
+    def manifest_names(self):
-        return "package.json"
+        return ["package.json"]

platformio/managers/platform.py

@@ -41,8 +41,8 @@ class PlatformManager(BasePkgManager):
                                 repositories)
 
     @property
-    def manifest_name(self):
+    def manifest_names(self):
-        return "platform.json"
+        return ["platform.json"]
 
     def install(self,
                 name,
@@ -484,6 +484,19 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
                 "optional": False
             }
 
+    def get_lib_storages(self):
+        storages = []
+        for _, opts in (self.frameworks or {}).items():
+            if "package" not in opts:
+                continue
+            pkg_dir = self.get_package_dir(opts['package'])
+            if not pkg_dir or not isdir(join(pkg_dir, "libraries")):
+                continue
+            storages.append(
+                dict(
+                    name=opts['package'], path=join(pkg_dir, "libraries")))
+        return storages
+
 
 class PlatformBoardConfig(object):
 

platformio/telemetry.py

@@ -121,8 +121,8 @@ class MeasurementProtocol(TelemetryBase):
                        "settings", "account"):
             cmd_path = args[:2]
         if args[0] == "lib" and len(args) > 1:
-            lib_subcmds = ("install", "list", "register", "search", "show",
+            lib_subcmds = ("builtin", "install", "list", "register", "search",
-                           "uninstall", "update")
+                           "show", "stats", "uninstall", "update")
             sub_cmd = _first_arg_from_list(args[1:], lib_subcmds)
             if sub_cmd:
                 cmd_path.append(sub_cmd)