Implement package packer

platformio/commands/platform.py

@@ -20,6 +20,7 @@ from platformio import app, exception, util
 from platformio.commands.boards import print_boards
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.package.pack import PackagePacker
 
 
 @click.group(short_help="Platform Manager")
@@ -403,3 +404,13 @@ def platform_update(  # pylint: disable=too-many-locals
         click.echo()
 
     return True
+
+
+@cli.command(
+    "pack", short_help="Create a tarball from development platform/tool package"
+)
+@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]")
+def platform_pack(package):
+    p = PackagePacker(package)
+    tarball_path = p.pack()
+    click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green")

platformio/compat.py

@@ -39,7 +39,7 @@ def get_locale_encoding():
 
 
 def get_class_attributes(cls):
-    attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a)))
+    attributes = inspect.getmembers(cls, lambda a: not inspect.isroutine(a))
     return {
         a[0]: a[1]
         for a in attributes

platformio/fs.py

@@ -143,10 +143,10 @@ def path_endswith_ext(path, extensions):
     return False
 
 
-def match_src_files(src_dir, src_filter=None, src_exts=None):
+def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True):
     def _append_build_item(items, item, src_dir):
         if not src_exts or path_endswith_ext(item, src_exts):
-            items.add(item.replace(src_dir + os.sep, ""))
+            items.add(os.path.relpath(item, src_dir))
 
     src_filter = src_filter or ""
     if isinstance(src_filter, (list, tuple)):
@@ -159,7 +159,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None):
         items = set()
         for item in glob(os.path.join(glob_escape(src_dir), pattern)):
             if os.path.isdir(item):
-                for root, _, files in os.walk(item, followlinks=True):
+                for root, _, files in os.walk(item, followlinks=followlinks):
                     for f in files:
                         _append_build_item(items, os.path.join(root, f), src_dir)
             else:

platformio/managers/package.py

@@ -28,10 +28,8 @@ from platformio import __version__, app, exception, fs, util
 from platformio.compat import hashlib_encode_data
 from platformio.downloader import FileDownloader
 from platformio.lockfile import LockFile
-from platformio.package.manifest.parser import (
+from platformio.package.exception import ManifestException
-    ManifestParserError,
+from platformio.package.manifest.parser import ManifestParserFactory
-    ManifestParserFactory,
-)
 from platformio.unpacker import FileUnpacker
 from platformio.vcsclient import VCSClientFactory
 
@@ -347,7 +345,7 @@ class PkgInstallerMixin(object):
 
         try:
             manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict()
-        except ManifestParserError:
+        except ManifestException:
             pass
 
         if src_manifest:

platformio/package/exception.py

@@ -19,6 +19,10 @@ class ManifestException(PlatformioException):
     pass
 
 
+class UnknownManifestError(ManifestException):
+    pass
+
+
 class ManifestParserError(ManifestException):
     pass
 

platformio/package/manifest/parser.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import inspect
 import json
 import os
 import re
@@ -20,7 +21,7 @@ import requests
 
 from platformio.compat import get_class_attributes, string_types
 from platformio.fs import get_file_contents
-from platformio.package.exception import ManifestParserError
+from platformio.package.exception import ManifestParserError, UnknownManifestError
 from platformio.project.helpers import is_platformio_project
 
 try:
@@ -36,36 +37,36 @@ class ManifestFileType(object):
     MODULE_JSON = "module.json"
     PACKAGE_JSON = "package.json"
 
+    @classmethod
+    def items(cls):
+        return get_class_attributes(ManifestFileType)
+
     @classmethod
     def from_uri(cls, uri):
-        if uri.endswith(".properties"):
+        for t in sorted(cls.items().values()):
-            return ManifestFileType.LIBRARY_PROPERTIES
+            if uri.endswith(t):
-        if uri.endswith("platform.json"):
+                return t
-            return ManifestFileType.PLATFORM_JSON
+        return None
-        if uri.endswith("module.json"):
+
-            return ManifestFileType.MODULE_JSON
+    @classmethod
-        if uri.endswith("package.json"):
+    def from_dir(cls, path):
-            return ManifestFileType.PACKAGE_JSON
+        for t in sorted(cls.items().values()):
-        if uri.endswith("library.json"):
+            if os.path.isfile(os.path.join(path, t)):
-            return ManifestFileType.LIBRARY_JSON
+                return t
         return None
 
 
 class ManifestParserFactory(object):
-    @staticmethod
-    def type_to_clsname(t):
-        t = t.replace(".", " ")
-        t = t.title()
-        return "%sManifestParser" % t.replace(" ", "")
-
     @staticmethod
     def new_from_file(path, remote_url=False):
         if not path or not os.path.isfile(path):
-            raise ManifestParserError("Manifest file does not exist %s" % path)
+            raise UnknownManifestError("Manifest file does not exist %s" % path)
-        for t in get_class_attributes(ManifestFileType).values():
+        type_from_uri = ManifestFileType.from_uri(path)
-            if path.endswith(t):
+        if not type_from_uri:
-                return ManifestParserFactory.new(get_file_contents(path), t, remote_url)
+            raise UnknownManifestError("Unknown manifest file type %s" % path)
-        raise ManifestParserError("Unknown manifest file type %s" % path)
+        return ManifestParserFactory.new(
+            get_file_contents(path), type_from_uri, remote_url
+        )
 
     @staticmethod
     def new_from_dir(path, remote_url=None):
@@ -80,23 +81,17 @@ class ManifestParserFactory(object):
                 package_dir=path,
             )
 
-        file_order = [
+        type_from_dir = ManifestFileType.from_dir(path)
-            ManifestFileType.PLATFORM_JSON,
+        if not type_from_dir:
-            ManifestFileType.LIBRARY_JSON,
+            raise UnknownManifestError(
-            ManifestFileType.LIBRARY_PROPERTIES,
+                "Unknown manifest file type in %s directory" % path
-            ManifestFileType.MODULE_JSON,
-            ManifestFileType.PACKAGE_JSON,
-        ]
-        for t in file_order:
-            if not os.path.isfile(os.path.join(path, t)):
-                continue
-            return ManifestParserFactory.new(
-                get_file_contents(os.path.join(path, t)),
-                t,
-                remote_url=remote_url,
-                package_dir=path,
             )
-        raise ManifestParserError("Unknown manifest file type in %s directory" % path)
+        return ManifestParserFactory.new(
+            get_file_contents(os.path.join(path, type_from_dir)),
+            type_from_dir,
+            remote_url=remote_url,
+            package_dir=path,
+        )
 
     @staticmethod
     def new_from_url(remote_url):
@@ -109,12 +104,18 @@ class ManifestParserFactory(object):
         )
 
     @staticmethod
-    def new(contents, type, remote_url=None, package_dir=None):
+    def new(  # pylint: disable=redefined-builtin
-        # pylint: disable=redefined-builtin
+        contents, type, remote_url=None, package_dir=None
-        clsname = ManifestParserFactory.type_to_clsname(type)
+    ):
-        if clsname not in globals():
+        for _, cls in globals().items():
-            raise ManifestParserError("Unknown manifest file type %s" % clsname)
+            if (
-        return globals()[clsname](contents, remote_url, package_dir)
+                inspect.isclass(cls)
+                and issubclass(cls, BaseManifestParser)
+                and cls != BaseManifestParser
+                and cls.manifest_type == type
+            ):
+                return cls(contents, remote_url, package_dir)
+        raise UnknownManifestError("Unknown manifest file type %s" % type)
 
 
 class BaseManifestParser(object):
@@ -268,6 +269,8 @@ class BaseManifestParser(object):
 
 
 class LibraryJsonManifestParser(BaseManifestParser):
+    manifest_type = ManifestFileType.LIBRARY_JSON
+
     def parse(self, contents):
         data = json.loads(contents)
         data = self._process_renamed_fields(data)
@@ -349,6 +352,8 @@ class LibraryJsonManifestParser(BaseManifestParser):
 
 
 class ModuleJsonManifestParser(BaseManifestParser):
+    manifest_type = ManifestFileType.MODULE_JSON
+
     def parse(self, contents):
         data = json.loads(contents)
         data["frameworks"] = ["mbed"]
@@ -381,10 +386,12 @@ class ModuleJsonManifestParser(BaseManifestParser):
 
 
 class LibraryPropertiesManifestParser(BaseManifestParser):
+    manifest_type = ManifestFileType.LIBRARY_PROPERTIES
+
     def parse(self, contents):
         data = self._parse_properties(contents)
         repository = self._parse_repository(data)
-        homepage = data.get("url")
+        homepage = data.get("url") or None
         if repository and repository["url"] == homepage:
             homepage = None
         data.update(
@@ -529,6 +536,8 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
 
 
 class PlatformJsonManifestParser(BaseManifestParser):
+    manifest_type = ManifestFileType.PLATFORM_JSON
+
     def parse(self, contents):
         data = json.loads(contents)
         if "frameworks" in data:
@@ -543,6 +552,8 @@ class PlatformJsonManifestParser(BaseManifestParser):
 
 
 class PackageJsonManifestParser(BaseManifestParser):
+    manifest_type = ManifestFileType.PACKAGE_JSON
+
     def parse(self, contents):
         data = json.loads(contents)
         data = self._parse_system(data)

platformio/package/pack.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tarfile
+import tempfile
+
+from platformio import fs
+from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory
+from platformio.package.manifest.schema import ManifestSchema
+from platformio.unpacker import FileUnpacker
+
+
+class PackagePacker(object):
+    EXCLUDE_DEFAULT = ["._*", ".DS_Store", ".git", ".hg", ".svn", ".pio"]
+    INCLUDE_DEFAULT = ManifestFileType.items().values()
+
+    def __init__(self, package):
+        self.package = package
+
+    def pack(self, dst=None):
+        tmp_dir = tempfile.mkdtemp()
+        try:
+            src = self.package
+
+            # if zip/tar.gz -> unpack to tmp dir
+            if not os.path.isdir(src):
+                with FileUnpacker(src) as fu:
+                    assert fu.unpack(tmp_dir, silent=True)
+                src = tmp_dir
+
+            manifest = self.load_manifest(src)
+            filename = "{name}{system}-{version}.tar.gz".format(
+                name=manifest["name"],
+                system="-" + manifest["system"][0] if "system" in manifest else "",
+                version=manifest["version"],
+            )
+
+            if not dst:
+                dst = os.path.join(os.getcwd(), filename)
+            elif os.path.isdir(dst):
+                dst = os.path.join(dst, filename)
+
+            return self._create_tarball(
+                src,
+                dst,
+                include=manifest.get("export", {}).get("include"),
+                exclude=manifest.get("export", {}).get("exclude"),
+            )
+        finally:
+            shutil.rmtree(tmp_dir)
+
+    @staticmethod
+    def load_manifest(src):
+        mp = ManifestParserFactory.new_from_dir(src)
+        return ManifestSchema().load_manifest(mp.as_dict())
+
+    def _create_tarball(self, src, dst, include=None, exclude=None):
+        # remap root
+        if (
+            include
+            and len(include) == 1
+            and os.path.isdir(os.path.join(src, include[0]))
+        ):
+            src = os.path.join(src, include[0])
+            include = None
+
+        src_filters = self.compute_src_filters(include, exclude)
+        with tarfile.open(dst, "w:gz") as tar:
+            for f in fs.match_src_files(src, src_filters, followlinks=False):
+                tar.add(os.path.join(src, f), f)
+        return dst
+
+    def compute_src_filters(self, include, exclude):
+        result = ["+<%s>" % p for p in include or ["*"]]
+        result += ["-<%s>" % p for p in exclude or []]
+        result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT]
+        # automatically include manifests
+        result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT]
+        return result

platformio/unpacker.py

@@ -73,6 +73,7 @@ class TARArchive(ArchiveBase):
         ).startswith(base)
 
     def extract_item(self, item, dest_dir):
+        dest_dir = self.resolve_path(dest_dir)
         bad_conds = [
             self.is_bad_path(item.name, dest_dir),
             self.is_link(item) and self.is_bad_link(item, dest_dir),
@@ -137,10 +138,13 @@ class FileUnpacker(object):
         if self._unpacker:
             self._unpacker.close()
 
-    def unpack(self, dest_dir=".", with_progress=True, check_unpacked=True):
+    def unpack(
+        self, dest_dir=".", with_progress=True, check_unpacked=True, silent=False
+    ):
         assert self._unpacker
-        if not with_progress:
+        if not with_progress or silent:
-            click.echo("Unpacking...")
+            if not silent:
+                click.echo("Unpacking...")
             for item in self._unpacker.get_items():
                 self._unpacker.extract_item(item, dest_dir)
         else:

tests/package/test_pack.py

@@ -0,0 +1,116 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import tarfile
+
+import pytest
+
+from platformio import fs
+from platformio.package.exception import UnknownManifestError
+from platformio.package.pack import PackagePacker
+
+
+def test_base(tmpdir_factory):
+    pkg_dir = tmpdir_factory.mktemp("package")
+    pkg_dir.join("main.cpp").write("#include <stdio.h>")
+    p = PackagePacker(str(pkg_dir))
+    # test missed manifest
+    with pytest.raises(UnknownManifestError):
+        p.pack()
+    # minimal package
+    pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}')
+    pkg_dir.mkdir("include").join("main.h").write("#ifndef")
+    with fs.cd(str(pkg_dir)):
+        p.pack()
+    with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar:
+        assert set(tar.getnames()) == set(
+            ["include/main.h", "library.json", "main.cpp"]
+        )
+
+
+def test_filters(tmpdir_factory):
+    pkg_dir = tmpdir_factory.mktemp("package")
+    src_dir = pkg_dir.mkdir("src")
+    src_dir.join("main.cpp").write("#include <stdio.h>")
+    src_dir.mkdir("util").join("helpers.cpp").write("void")
+    pkg_dir.mkdir("include").join("main.h").write("#ifndef")
+    test_dir = pkg_dir.mkdir("tests")
+    test_dir.join("test_1.h").write("")
+    test_dir.join("test_2.h").write("")
+
+    # test include with remap of root
+    pkg_dir.join("library.json").write(
+        json.dumps(dict(name="bar", version="1.2.3", export={"include": "src"}))
+    )
+    p = PackagePacker(str(pkg_dir))
+    dst = os.path.join(str(pkg_dir), "tarball.tar.gz")
+    p.pack(dst)
+    with tarfile.open(dst, "r:gz") as tar:
+        assert set(tar.getnames()) == set(["util/helpers.cpp", "main.cpp"])
+
+    # test include "src" and "include"
+    pkg_dir.join("library.json").write(
+        json.dumps(
+            dict(name="bar", version="1.2.3", export={"include": ["src", "include"]})
+        )
+    )
+    p = PackagePacker(str(pkg_dir))
+    dst = os.path.join(str(pkg_dir), "tarball.tar.gz")
+    p.pack(dst)
+    with tarfile.open(dst, "r:gz") as tar:
+        assert set(tar.getnames()) == set(
+            ["include/main.h", "library.json", "src/main.cpp", "src/util/helpers.cpp"]
+        )
+
+    # test include & exclude
+    pkg_dir.join("library.json").write(
+        json.dumps(
+            dict(
+                name="bar",
+                version="1.2.3",
+                export={"include": ["src", "include"], "exclude": ["*/*.h"]},
+            )
+        )
+    )
+    p = PackagePacker(str(pkg_dir))
+    dst = os.path.join(str(pkg_dir), "tarball.tar.gz")
+    p.pack(dst)
+    with tarfile.open(dst, "r:gz") as tar:
+        assert set(tar.getnames()) == set(
+            ["library.json", "src/main.cpp", "src/util/helpers.cpp"]
+        )
+
+
+def test_symlinks(tmpdir_factory):
+    pkg_dir = tmpdir_factory.mktemp("package")
+    src_dir = pkg_dir.mkdir("src")
+    src_dir.join("main.cpp").write("#include <stdio.h>")
+    pkg_dir.mkdir("include").join("main.h").write("#ifndef")
+    src_dir.join("main.h").mksymlinkto(os.path.join("..", "include", "main.h"))
+    pkg_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}')
+    tarball = pkg_dir.join("bar.tar.gz")
+    with tarfile.open(str(tarball), "w:gz") as tar:
+        for item in pkg_dir.listdir():
+            tar.add(str(item), str(item.relto(pkg_dir)))
+
+    p = PackagePacker(str(tarball))
+    assert p.pack(str(pkg_dir)).endswith("bar-2.0.0.tar.gz")
+    with tarfile.open(os.path.join(str(pkg_dir), "bar-2.0.0.tar.gz"), "r:gz") as tar:
+        assert set(tar.getnames()) == set(
+            ["include/main.h", "library.json", "src/main.cpp", "src/main.h"]
+        )
+        m = tar.getmember("src/main.h")
+        assert m.issym()