From bbd694c5ea34315b5acc67be5e7dd6e9259a6b1a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 17:54:59 +0300 Subject: [PATCH] ManifestParse: automatically generate examples from package dir --- platformio/package/manifest/parser.py | 92 ++++++++++++++++++++++++--- tests/test_pkgmanifest.py | 92 ++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 12 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 291fbae6..46895ecd 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -21,6 +21,7 @@ import requests from platformio.compat import get_class_attributes, string_types from platformio.exception import PlatformioException from platformio.fs import get_file_contents +from platformio.project.helpers import is_platformio_project try: from urllib.parse import urlparse @@ -86,7 +87,10 @@ class ManifestParserFactory(object): if not os.path.isfile(os.path.join(path, t)): continue return ManifestParserFactory.new( - get_file_contents(os.path.join(path, t)), t, remote_url + get_file_contents(os.path.join(path, t)), + t, + remote_url=remote_url, + package_dir=path, ) raise ManifestException("Unknown manifest file type in %s directory" % path) @@ -99,18 +103,20 @@ class ManifestParserFactory(object): ) @staticmethod - def new(contents, type, remote_url=None): + def new(contents, type, remote_url=None, package_dir=None): # pylint: disable=redefined-builtin clsname = ManifestParserFactory.type_to_clsname(type) if clsname not in globals(): raise ManifestException("Unknown manifest file type %s" % clsname) - return globals()[clsname](contents, remote_url) + return globals()[clsname](contents, remote_url, package_dir) class BaseManifestParser(object): - def __init__(self, contents, remote_url=None): + def __init__(self, contents, remote_url=None, package_dir=None): self.remote_url = remote_url + self.package_dir = package_dir self._data = self.parse(contents) + self._data = self.parse_examples(self._data) def parse(self, contents): raise NotImplementedError @@ -119,7 +125,7 @@ class BaseManifestParser(object): return self._data @staticmethod - def _cleanup_author(author): + def cleanup_author(author): if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) return author @@ -136,6 +142,74 @@ class BaseManifestParser(object): email = raw[raw.index(ldel) + 1 : raw.index(rdel)] return (name.strip(), email.strip() if email else None) + def parse_examples(self, data): + examples = data.get("examples") + if ( + not examples + or not isinstance(examples, list) + or not all(isinstance(v, dict) for v in examples) + ): + data["examples"] = None + if not examples and self.package_dir: + data["examples"] = self.parse_examples_from_dir(self.package_dir) + if "examples" in data and not data["examples"]: + del data["examples"] + return data + + @staticmethod + def parse_examples_from_dir(package_dir): + assert os.path.isdir(package_dir) + examples_dir = os.path.join(package_dir, "examples") + if not os.path.isdir(examples_dir): + return None + + allowed_exts = ( + ".c", + ".cc", + ".cpp", + ".h", + ".hpp", + ".asm", + ".ASM", + ".s", + ".S", + ".ino", + ".pde", + ) + + result = {} + last_pio_project = None + for root, _, files in os.walk(examples_dir): + if is_platformio_project(root): + last_pio_project = root + result[last_pio_project] = dict( + name=os.path.relpath(root, examples_dir), + base=os.path.relpath(root, package_dir), + files=files, + ) + continue + if last_pio_project: + if root.startswith(last_pio_project): + result[last_pio_project]["files"].extend( + [ + os.path.relpath(os.path.join(root, f), last_pio_project) + for f in files + ] + ) + continue + last_pio_project = None + + matched_files = [f for f in files if f.endswith(allowed_exts)] + if not matched_files: + continue + result[root] = dict( + name=os.path.relpath(root, examples_dir), + base=os.path.relpath(root, package_dir), + files=matched_files, + ) + + return list(result.values()) or None + class LibraryJsonManifestParser(BaseManifestParser): def parse(self, contents): @@ -193,7 +267,7 @@ class LibraryJsonManifestParser(BaseManifestParser): # normalize Union[dict, list] fields if not isinstance(raw, list): raw = [raw] - return [self._cleanup_author(author) for author in raw] + return [self.cleanup_author(author) for author in raw] @staticmethod def _parse_platforms(raw): @@ -243,7 +317,7 @@ class ModuleJsonManifestParser(BaseManifestParser): if not name: continue result.append( - self._cleanup_author(dict(name=name, email=email, maintainer=False)) + self.cleanup_author(dict(name=name, email=email, maintainer=False)) ) return result @@ -341,7 +415,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): if not name: continue authors.append( - self._cleanup_author(dict(name=name, email=email, maintainer=False)) + self.cleanup_author(dict(name=name, email=email, maintainer=False)) ) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) @@ -357,7 +431,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): item["email"] = email if not found: authors.append( - self._cleanup_author(dict(name=name, email=email, maintainer=True)) + self.cleanup_author(dict(name=name, email=email, maintainer=True)) ) return authors diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index ec19b29d..64e6b88e 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -425,6 +425,94 @@ def test_package_json_model(): assert mp.as_dict()["system"] == ["darwin_x86_64"] +def test_examples_from_dir(tmpdir_factory): + package_dir = tmpdir_factory.mktemp("project") + package_dir.join("library.json").write('{"name": "pkg", "version": "1.0.0"}') + examples_dir = package_dir.mkdir("examples") + + # PlatformIO project #1 + pio_dir = examples_dir.mkdir("PlatformIO").mkdir("hello") + pio_dir.join("platformio.ini").write("") + pio_dir.mkdir("include").join("main.h").write("") + pio_dir.mkdir("src").join("main.cpp").write("") + + # wiring examples + examples_dir.mkdir("SomeSketchIno").join("SomeSketchIno.ino").write("") + examples_dir.mkdir("SomeSketchPde").join("SomeSketchPde.pde").write("") + + # custom examples + demo_dir = examples_dir.mkdir("demo") + demo_dir.join("demo.cpp").write("") + demo_dir.join("demo.h").write("") + demo_dir.join("util.h").write("") + + # PlatformIO project #2 + pio_dir = examples_dir.mkdir("world") + pio_dir.join("platformio.ini").write("") + pio_dir.join("README").write("") + pio_dir.join("extra.py").write("") + pio_dir.mkdir("include").join("world.h").write("") + pio_dir.mkdir("src").join("world.c").write("") + + # invalid example + examples_dir.mkdir("invalid-example").join("hello.json") + + # Do testing + + data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() + assert isinstance(data["examples"], list) + assert len(data["examples"]) == 5 + + def _sort_examples(items): + for i, item in enumerate(items): + items[i]["files"] = sorted(item["files"]) + return sorted(items, key=lambda item: item["name"]) + + data["examples"] = _sort_examples(data["examples"]) + model = ManifestModel(**data) + assert model == ManifestModel( + **{ + "version": "1.0.0", + "name": "pkg", + "examples": _sort_examples( + [ + { + "name": "PlatformIO/hello", + "base": "examples/PlatformIO/hello", + "files": ["platformio.ini", "include/main.h", "src/main.cpp"], + }, + { + "name": "SomeSketchIno", + "base": "examples/SomeSketchIno", + "files": ["SomeSketchIno.ino"], + }, + { + "name": "SomeSketchPde", + "base": "examples/SomeSketchPde", + "files": ["SomeSketchPde.pde"], + }, + { + "name": "demo", + "base": "examples/demo", + "files": ["demo.h", "util.h", "demo.cpp"], + }, + { + "name": "world", + "base": "examples/world", + "files": [ + "platformio.ini", + "include/world.h", + "src/world.c", + "README", + "extra.py", + ], + }, + ] + ), + } + ) + + def test_broken_models(): # non-strict mode assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4 @@ -453,9 +541,7 @@ def test_broken_models(): ) # broken value for DataModel - with pytest.raises( - DataFieldException, match="Value should be type of dict, not `" - ): + with pytest.raises(DataFieldException, match="Value should be type of dict"): assert StrictManifestModel( name="MyPackage", description="MyDescription",