platformio-core/platformio/package/pack.py

# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
import re
import shutil
import tarfile
import tempfile

from platformio import fs
from platformio.compat import ensure_python3
from platformio.package.exception import PackageException
from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory
from platformio.package.manifest.schema import ManifestSchema
from platformio.package.meta import PackageItem
from platformio.package.unpack import FileUnpacker


class PackagePacker(object):
    INCLUDE_DEFAULT = ManifestFileType.items().values()
    EXCLUDE_DEFAULT = [
        # PlatformIO internal files
        PackageItem.METAFILE_NAME,
        ".pio/",
        "**/.pio/",
        # Hidden files
        "._*",
        "__*",
        ".DS_Store",
        ".vscode",
        ".cache",
        "**/.cache",
        # VCS
        ".git/",
        ".hg/",
        ".svn/",
        # Tests
        "tests?",
        # Docs
        "doc",
        "docs",
        "mkdocs",
        "**/*.[pP][dD][fF]",
        "**/*.[dD][oO][cC]?",
        "**/*.[pP][pP][tT]?",
        "**/*.[dD][oO][xX]",
        "**/*.[hH][tT][mM]?",
        "**/*.[tT][eE][xX]",
        "**/*.[jJ][sS]",
        "**/*.[cC][sS][sS]",
        # Binary files
        "**/*.[jJ][pP][gG]",
        "**/*.[jJ][pP][eE][gG]",
        "**/*.[pP][nN][gG]",
        "**/*.[gG][iI][fF]",
        "**/*.[zZ][iI][pP]",
        "**/*.[gG][zZ]",
        "**/*.3[gG][pP]",
        "**/*.[mM][oO][vV]",
        "**/*.[mM][pP][34]",
        "**/*.[pP][sS][dD]",
        "**/*.[wW][aA][wW]",
    ]
    EXCLUDE_LIBRARY_EXTRA = [
        "assets",
        "extra",
        "resources",
        "html",
        "media",
        "doxygen",
        "**/build/",
        "**/*.flat",
        "**/*.[jJ][aA][rR]",
        "**/*.[eE][xX][eE]",
        "**/*.[bB][iI][nN]",
        "**/*.[hH][eE][xX]",
        "**/*.[dD][bB]",
        "**/*.[dD][aA][tT]",
        "**/*.[dD][lL][lL]",
    ]

    def __init__(self, package, manifest_uri=None):
        assert ensure_python3()
        self.package = package
        self.manifest_uri = manifest_uri

    @staticmethod
    def get_archive_name(name, version, system=None):
        return re.sub(
            r"[^\da-zA-Z\-\._\+]+",
            "",
            "{name}{system}-{version}.tar.gz".format(
                name=name,
                system=("-" + system) if system else "",
                version=version,
            ),
        )

    def pack(self, dst=None):
        tmp_dir = tempfile.mkdtemp()
        try:
            src = self.package

            # if zip/tar.gz -> unpack to tmp dir
            if not os.path.isdir(src):
                with FileUnpacker(src) as fu:
                    assert fu.unpack(tmp_dir, silent=True)
                src = tmp_dir

            src = self.find_source_root(src)

            manifest = self.load_manifest(src)
            filename = self.get_archive_name(
                manifest["name"],
                manifest["version"],
                manifest["system"][0] if "system" in manifest else None,
            )

            if not dst:
                dst = os.path.join(os.getcwd(), filename)
            elif os.path.isdir(dst):
                dst = os.path.join(dst, filename)

            return self._create_tarball(src, dst, manifest)
        finally:
            shutil.rmtree(tmp_dir)

    @staticmethod
    def load_manifest(src):
        mp = ManifestParserFactory.new_from_dir(src)
        return ManifestSchema().load_manifest(mp.as_dict())

    def find_source_root(self, src):
        if self.manifest_uri:
            mp = (
                ManifestParserFactory.new_from_file(self.manifest_uri[5:])
                if self.manifest_uri.startswith("file:")
                else ManifestParserFactory.new_from_url(self.manifest_uri)
            )
            manifest = ManifestSchema().load_manifest(mp.as_dict())
            include = manifest.get("export", {}).get("include", [])
            if len(include) == 1:
                if not os.path.isdir(os.path.join(src, include[0])):
                    raise PackageException(
                        "Non existing `include` directory `%s` in a package"
                        % include[0]
                    )
                return os.path.join(src, include[0])

        for root, _, __ in os.walk(src):
            if ManifestFileType.from_dir(root):
                return root

        return src

    def _create_tarball(self, src, dst, manifest):
        include = manifest.get("export", {}).get("include")
        exclude = manifest.get("export", {}).get("exclude")
        # remap root
        if (
            include
            and len(include) == 1
            and os.path.isdir(os.path.join(src, include[0]))
        ):
            src = os.path.join(src, include[0])
            with open(os.path.join(src, "library.json"), "w") as fp:
                manifest_updated = manifest.copy()
                del manifest_updated["export"]["include"]
                json.dump(manifest_updated, fp, indent=2, ensure_ascii=False)
            include = None

        src_filters = self.compute_src_filters(src, include, exclude)
        with tarfile.open(dst, "w:gz") as tar:
            for f in fs.match_src_files(src, src_filters, followlinks=False):
                tar.add(os.path.join(src, f), f)
        return dst

    def compute_src_filters(self, src, include, exclude):
        exclude_default = self.EXCLUDE_DEFAULT[:]
        # extend with library extra filters
        if any(
            os.path.isfile(os.path.join(src, name))
            for name in (
                ManifestFileType.LIBRARY_JSON,
                ManifestFileType.LIBRARY_PROPERTIES,
                ManifestFileType.MODULE_JSON,
            )
        ):
            exclude_default.extend(self.EXCLUDE_LIBRARY_EXTRA)

        result = ["+<%s>" % p for p in include or ["*", ".*"]]
        result += ["-<%s>" % p for p in exclude or []]
        result += ["-<%s>" % p for p in exclude_default]
        # automatically include manifests
        result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT]
        return result