diff --git a/docs b/docs index 9f450500..3622e358 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9f4505000aadd467731991f56accecfd5c71bd50 +Subproject commit 3622e3581951a4b193f7ce8f3aa5e05f59da29b5 diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 968f978b..c094b560 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -20,6 +20,7 @@ from platformio import app, exception, util from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.pack import PackagePacker @click.group(short_help="Platform Manager") @@ -403,3 +404,13 @@ def platform_update( # pylint: disable=too-many-locals click.echo() return True + + +@cli.command( + "pack", short_help="Create a tarball from development platform/tool package" +) +@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +def platform_pack(package): + p = PackagePacker(package) + tarball_path = p.pack() + click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") diff --git a/platformio/compat.py b/platformio/compat.py index a8ff28e2..9107f8b1 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -39,7 +39,7 @@ def get_locale_encoding(): def get_class_attributes(cls): - attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a))) + attributes = inspect.getmembers(cls, lambda a: not inspect.isroutine(a)) return { a[0]: a[1] for a in attributes diff --git a/platformio/fs.py b/platformio/fs.py index ca561b10..57809ba6 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -143,10 +143,10 @@ def path_endswith_ext(path, extensions): return False -def match_src_files(src_dir, src_filter=None, src_exts=None): +def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True): def _append_build_item(items, item, src_dir): if not src_exts or path_endswith_ext(item, src_exts): - items.add(item.replace(src_dir + os.sep, "")) + items.add(os.path.relpath(item, src_dir)) src_filter = src_filter or "" if isinstance(src_filter, (list, tuple)): @@ -159,7 +159,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None): items = set() for item in glob(os.path.join(glob_escape(src_dir), pattern)): if os.path.isdir(item): - for root, _, files in os.walk(item, followlinks=True): + for root, _, files in os.walk(item, followlinks=followlinks): for f in files: _append_build_item(items, os.path.join(root, f), src_dir) else: diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 6df5276d..d6481541 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -28,10 +28,8 @@ from platformio import __version__, app, exception, fs, util from platformio.compat import hashlib_encode_data from platformio.downloader import FileDownloader from platformio.lockfile import LockFile -from platformio.package.manifest.parser import ( - ManifestParserError, - ManifestParserFactory, -) +from platformio.package.exception import ManifestException +from platformio.package.manifest.parser import ManifestParserFactory from platformio.unpacker import FileUnpacker from platformio.vcsclient import VCSClientFactory @@ -347,7 +345,7 @@ class PkgInstallerMixin(object): try: manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict() - except ManifestParserError: + except ManifestException: pass if src_manifest: diff --git a/platformio/package/exception.py b/platformio/package/exception.py index f2597dd7..3927505d 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -19,6 +19,10 @@ class ManifestException(PlatformioException): pass +class UnknownManifestError(ManifestException): + pass + + class ManifestParserError(ManifestException): pass diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 028d84a7..71f63c79 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import json import os import re @@ -20,7 +21,7 @@ import requests from platformio.compat import get_class_attributes, string_types from platformio.fs import get_file_contents -from platformio.package.exception import ManifestParserError +from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project try: @@ -36,36 +37,36 @@ class ManifestFileType(object): MODULE_JSON = "module.json" PACKAGE_JSON = "package.json" + @classmethod + def items(cls): + return get_class_attributes(ManifestFileType) + @classmethod def from_uri(cls, uri): - if uri.endswith(".properties"): - return ManifestFileType.LIBRARY_PROPERTIES - if uri.endswith("platform.json"): - return ManifestFileType.PLATFORM_JSON - if uri.endswith("module.json"): - return ManifestFileType.MODULE_JSON - if uri.endswith("package.json"): - return ManifestFileType.PACKAGE_JSON - if uri.endswith("library.json"): - return ManifestFileType.LIBRARY_JSON + for t in sorted(cls.items().values()): + if uri.endswith(t): + return t + return None + + @classmethod + def from_dir(cls, path): + for t in sorted(cls.items().values()): + if os.path.isfile(os.path.join(path, t)): + return t return None class ManifestParserFactory(object): - @staticmethod - def type_to_clsname(t): - t = t.replace(".", " ") - t = t.title() - return "%sManifestParser" % t.replace(" ", "") - @staticmethod def new_from_file(path, remote_url=False): if not path or not os.path.isfile(path): - raise ManifestParserError("Manifest file does not exist %s" % path) - for t in get_class_attributes(ManifestFileType).values(): - if path.endswith(t): - return ManifestParserFactory.new(get_file_contents(path), t, remote_url) - raise ManifestParserError("Unknown manifest file type %s" % path) + raise UnknownManifestError("Manifest file does not exist %s" % path) + type_from_uri = ManifestFileType.from_uri(path) + if not type_from_uri: + raise UnknownManifestError("Unknown manifest file type %s" % path) + return ManifestParserFactory.new( + get_file_contents(path), type_from_uri, remote_url + ) @staticmethod def new_from_dir(path, remote_url=None): @@ -80,23 +81,17 @@ class ManifestParserFactory(object): package_dir=path, ) - file_order = [ - ManifestFileType.PLATFORM_JSON, - ManifestFileType.LIBRARY_JSON, - ManifestFileType.LIBRARY_PROPERTIES, - ManifestFileType.MODULE_JSON, - ManifestFileType.PACKAGE_JSON, - ] - for t in file_order: - if not os.path.isfile(os.path.join(path, t)): - continue - return ManifestParserFactory.new( - get_file_contents(os.path.join(path, t)), - t, - remote_url=remote_url, - package_dir=path, + type_from_dir = ManifestFileType.from_dir(path) + if not type_from_dir: + raise UnknownManifestError( + "Unknown manifest file type in %s directory" % path ) - raise ManifestParserError("Unknown manifest file type in %s directory" % path) + return ManifestParserFactory.new( + get_file_contents(os.path.join(path, type_from_dir)), + type_from_dir, + remote_url=remote_url, + package_dir=path, + ) @staticmethod def new_from_url(remote_url): @@ -109,12 +104,18 @@ class ManifestParserFactory(object): ) @staticmethod - def new(contents, type, remote_url=None, package_dir=None): - # pylint: disable=redefined-builtin - clsname = ManifestParserFactory.type_to_clsname(type) - if clsname not in globals(): - raise ManifestParserError("Unknown manifest file type %s" % clsname) - return globals()[clsname](contents, remote_url, package_dir) + def new( # pylint: disable=redefined-builtin + contents, type, remote_url=None, package_dir=None + ): + for _, cls in globals().items(): + if ( + inspect.isclass(cls) + and issubclass(cls, BaseManifestParser) + and cls != BaseManifestParser + and cls.manifest_type == type + ): + return cls(contents, remote_url, package_dir) + raise UnknownManifestError("Unknown manifest file type %s" % type) class BaseManifestParser(object): @@ -268,6 +269,8 @@ class BaseManifestParser(object): class LibraryJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.LIBRARY_JSON + def parse(self, contents): data = json.loads(contents) data = self._process_renamed_fields(data) @@ -349,6 +352,8 @@ class LibraryJsonManifestParser(BaseManifestParser): class ModuleJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.MODULE_JSON + def parse(self, contents): data = json.loads(contents) data["frameworks"] = ["mbed"] @@ -381,10 +386,12 @@ class ModuleJsonManifestParser(BaseManifestParser): class LibraryPropertiesManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.LIBRARY_PROPERTIES + def parse(self, contents): data = self._parse_properties(contents) repository = self._parse_repository(data) - homepage = data.get("url") + homepage = data.get("url") or None if repository and repository["url"] == homepage: homepage = None data.update( @@ -529,6 +536,8 @@ class LibraryPropertiesManifestParser(BaseManifestParser): class PlatformJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.PLATFORM_JSON + def parse(self, contents): data = json.loads(contents) if "frameworks" in data: @@ -543,6 +552,8 @@ class PlatformJsonManifestParser(BaseManifestParser): class PackageJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.PACKAGE_JSON + def parse(self, contents): data = json.loads(contents) data = self._parse_system(data) diff --git a/platformio/package/pack.py b/platformio/package/pack.py new file mode 100644 index 00000000..c58b84a3 --- /dev/null +++ b/platformio/package/pack.py @@ -0,0 +1,92 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tarfile +import tempfile + +from platformio import fs +from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory +from platformio.package.manifest.schema import ManifestSchema +from platformio.unpacker import FileUnpacker + + +class PackagePacker(object): + EXCLUDE_DEFAULT = ["._*", ".DS_Store", ".git", ".hg", ".svn", ".pio"] + INCLUDE_DEFAULT = ManifestFileType.items().values() + + def __init__(self, package): + self.package = package + + def pack(self, dst=None): + tmp_dir = tempfile.mkdtemp() + try: + src = self.package + + # if zip/tar.gz -> unpack to tmp dir + if not os.path.isdir(src): + with FileUnpacker(src) as fu: + assert fu.unpack(tmp_dir, silent=True) + src = tmp_dir + + manifest = self.load_manifest(src) + filename = "{name}{system}-{version}.tar.gz".format( + name=manifest["name"], + system="-" + manifest["system"][0] if "system" in manifest else "", + version=manifest["version"], + ) + + if not dst: + dst = os.path.join(os.getcwd(), filename) + elif os.path.isdir(dst): + dst = os.path.join(dst, filename) + + return self._create_tarball( + src, + dst, + include=manifest.get("export", {}).get("include"), + exclude=manifest.get("export", {}).get("exclude"), + ) + finally: + shutil.rmtree(tmp_dir) + + @staticmethod + def load_manifest(src): + mp = ManifestParserFactory.new_from_dir(src) + return ManifestSchema().load_manifest(mp.as_dict()) + + def _create_tarball(self, src, dst, include=None, exclude=None): + # remap root + if ( + include + and len(include) == 1 + and os.path.isdir(os.path.join(src, include[0])) + ): + src = os.path.join(src, include[0]) + include = None + + src_filters = self.compute_src_filters(include, exclude) + with tarfile.open(dst, "w:gz") as tar: + for f in fs.match_src_files(src, src_filters, followlinks=False): + tar.add(os.path.join(src, f), f) + return dst + + def compute_src_filters(self, include, exclude): + result = ["+<%s>" % p for p in include or ["*"]] + result += ["-<%s>" % p for p in exclude or []] + result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT] + # automatically include manifests + result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT] + return result diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 980b43db..7fce466d 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -73,6 +73,7 @@ class TARArchive(ArchiveBase): ).startswith(base) def extract_item(self, item, dest_dir): + dest_dir = self.resolve_path(dest_dir) bad_conds = [ self.is_bad_path(item.name, dest_dir), self.is_link(item) and self.is_bad_link(item, dest_dir), @@ -137,10 +138,13 @@ class FileUnpacker(object): if self._unpacker: self._unpacker.close() - def unpack(self, dest_dir=".", with_progress=True, check_unpacked=True): + def unpack( + self, dest_dir=".", with_progress=True, check_unpacked=True, silent=False + ): assert self._unpacker - if not with_progress: - click.echo("Unpacking...") + if not with_progress or silent: + if not silent: + click.echo("Unpacking...") for item in self._unpacker.get_items(): self._unpacker.extract_item(item, dest_dir) else: diff --git a/tests/test_pkgmanifest.py b/tests/package/test_manifest.py similarity index 100% rename from tests/test_pkgmanifest.py rename to tests/package/test_manifest.py diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py new file mode 100644 index 00000000..91565213 --- /dev/null +++ b/tests/package/test_pack.py @@ -0,0 +1,116 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import tarfile + +import pytest + +from platformio import fs +from platformio.package.exception import UnknownManifestError +from platformio.package.pack import PackagePacker + + +def test_base(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("main.cpp").write("#include ") + p = PackagePacker(str(pkg_dir)) + # test missed manifest + with pytest.raises(UnknownManifestError): + p.pack() + # minimal package + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + with fs.cd(str(pkg_dir)): + p.pack() + with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["include/main.h", "library.json", "main.cpp"] + ) + + +def test_filters(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + src_dir = pkg_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + src_dir.mkdir("util").join("helpers.cpp").write("void") + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + test_dir = pkg_dir.mkdir("tests") + test_dir.join("test_1.h").write("") + test_dir.join("test_2.h").write("") + + # test include with remap of root + pkg_dir.join("library.json").write( + json.dumps(dict(name="bar", version="1.2.3", export={"include": "src"})) + ) + p = PackagePacker(str(pkg_dir)) + dst = os.path.join(str(pkg_dir), "tarball.tar.gz") + p.pack(dst) + with tarfile.open(dst, "r:gz") as tar: + assert set(tar.getnames()) == set(["util/helpers.cpp", "main.cpp"]) + + # test include "src" and "include" + pkg_dir.join("library.json").write( + json.dumps( + dict(name="bar", version="1.2.3", export={"include": ["src", "include"]}) + ) + ) + p = PackagePacker(str(pkg_dir)) + dst = os.path.join(str(pkg_dir), "tarball.tar.gz") + p.pack(dst) + with tarfile.open(dst, "r:gz") as tar: + assert set(tar.getnames()) == set( + ["include/main.h", "library.json", "src/main.cpp", "src/util/helpers.cpp"] + ) + + # test include & exclude + pkg_dir.join("library.json").write( + json.dumps( + dict( + name="bar", + version="1.2.3", + export={"include": ["src", "include"], "exclude": ["*/*.h"]}, + ) + ) + ) + p = PackagePacker(str(pkg_dir)) + dst = os.path.join(str(pkg_dir), "tarball.tar.gz") + p.pack(dst) + with tarfile.open(dst, "r:gz") as tar: + assert set(tar.getnames()) == set( + ["library.json", "src/main.cpp", "src/util/helpers.cpp"] + ) + + +def test_symlinks(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + src_dir = pkg_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + src_dir.join("main.h").mksymlinkto(os.path.join("..", "include", "main.h")) + pkg_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') + tarball = pkg_dir.join("bar.tar.gz") + with tarfile.open(str(tarball), "w:gz") as tar: + for item in pkg_dir.listdir(): + tar.add(str(item), str(item.relto(pkg_dir))) + + p = PackagePacker(str(tarball)) + assert p.pack(str(pkg_dir)).endswith("bar-2.0.0.tar.gz") + with tarfile.open(os.path.join(str(pkg_dir), "bar-2.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["include/main.h", "library.json", "src/main.cpp", "src/main.h"] + ) + m = tar.getmember("src/main.h") + assert m.issym()