ManifestParse: automatically generate examples from package dir

platformio/package/manifest/parser.py

@@ -21,6 +21,7 @@ import requests
 from platformio.compat import get_class_attributes, string_types
 from platformio.exception import PlatformioException
 from platformio.fs import get_file_contents
+from platformio.project.helpers import is_platformio_project
 
 try:
     from urllib.parse import urlparse
@@ -86,7 +87,10 @@ class ManifestParserFactory(object):
             if not os.path.isfile(os.path.join(path, t)):
                 continue
             return ManifestParserFactory.new(
-                get_file_contents(os.path.join(path, t)), t, remote_url
+                get_file_contents(os.path.join(path, t)),
+                t,
+                remote_url=remote_url,
+                package_dir=path,
             )
         raise ManifestException("Unknown manifest file type in %s directory" % path)
 
@@ -99,18 +103,20 @@ class ManifestParserFactory(object):
         )
 
     @staticmethod
-    def new(contents, type, remote_url=None):
+    def new(contents, type, remote_url=None, package_dir=None):
         # pylint: disable=redefined-builtin
         clsname = ManifestParserFactory.type_to_clsname(type)
         if clsname not in globals():
             raise ManifestException("Unknown manifest file type %s" % clsname)
-        return globals()[clsname](contents, remote_url)
+        return globals()[clsname](contents, remote_url, package_dir)
 
 
 class BaseManifestParser(object):
-    def __init__(self, contents, remote_url=None):
+    def __init__(self, contents, remote_url=None, package_dir=None):
         self.remote_url = remote_url
+        self.package_dir = package_dir
         self._data = self.parse(contents)
+        self._data = self.parse_examples(self._data)
 
     def parse(self, contents):
         raise NotImplementedError
@@ -119,7 +125,7 @@ class BaseManifestParser(object):
         return self._data
 
     @staticmethod
-    def _cleanup_author(author):
+    def cleanup_author(author):
         if author.get("email"):
             author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"])
         return author
@@ -136,6 +142,74 @@ class BaseManifestParser(object):
                 email = raw[raw.index(ldel) + 1 : raw.index(rdel)]
         return (name.strip(), email.strip() if email else None)
 
+    def parse_examples(self, data):
+        examples = data.get("examples")
+        if (
+            not examples
+            or not isinstance(examples, list)
+            or not all(isinstance(v, dict) for v in examples)
+        ):
+            data["examples"] = None
+        if not examples and self.package_dir:
+            data["examples"] = self.parse_examples_from_dir(self.package_dir)
+        if "examples" in data and not data["examples"]:
+            del data["examples"]
+        return data
+
+    @staticmethod
+    def parse_examples_from_dir(package_dir):
+        assert os.path.isdir(package_dir)
+        examples_dir = os.path.join(package_dir, "examples")
+        if not os.path.isdir(examples_dir):
+            return None
+
+        allowed_exts = (
+            ".c",
+            ".cc",
+            ".cpp",
+            ".h",
+            ".hpp",
+            ".asm",
+            ".ASM",
+            ".s",
+            ".S",
+            ".ino",
+            ".pde",
+        )
+
+        result = {}
+        last_pio_project = None
+        for root, _, files in os.walk(examples_dir):
+            if is_platformio_project(root):
+                last_pio_project = root
+                result[last_pio_project] = dict(
+                    name=os.path.relpath(root, examples_dir),
+                    base=os.path.relpath(root, package_dir),
+                    files=files,
+                )
+                continue
+            if last_pio_project:
+                if root.startswith(last_pio_project):
+                    result[last_pio_project]["files"].extend(
+                        [
+                            os.path.relpath(os.path.join(root, f), last_pio_project)
+                            for f in files
+                        ]
+                    )
+                    continue
+                last_pio_project = None
+
+            matched_files = [f for f in files if f.endswith(allowed_exts)]
+            if not matched_files:
+                continue
+            result[root] = dict(
+                name=os.path.relpath(root, examples_dir),
+                base=os.path.relpath(root, package_dir),
+                files=matched_files,
+            )
+
+        return list(result.values()) or None
+
 
 class LibraryJsonManifestParser(BaseManifestParser):
     def parse(self, contents):
@@ -193,7 +267,7 @@ class LibraryJsonManifestParser(BaseManifestParser):
         # normalize Union[dict, list] fields
         if not isinstance(raw, list):
             raw = [raw]
-        return [self._cleanup_author(author) for author in raw]
+        return [self.cleanup_author(author) for author in raw]
 
     @staticmethod
     def _parse_platforms(raw):
@@ -243,7 +317,7 @@ class ModuleJsonManifestParser(BaseManifestParser):
             if not name:
                 continue
             result.append(
-                self._cleanup_author(dict(name=name, email=email, maintainer=False))
+                self.cleanup_author(dict(name=name, email=email, maintainer=False))
             )
         return result
 
@@ -341,7 +415,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
             if not name:
                 continue
             authors.append(
-                self._cleanup_author(dict(name=name, email=email, maintainer=False))
+                self.cleanup_author(dict(name=name, email=email, maintainer=False))
             )
         for author in properties.get("maintainer", "").split(","):
             name, email = self.parse_author_name_and_email(author)
@@ -357,7 +431,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
                     item["email"] = email
             if not found:
                 authors.append(
-                    self._cleanup_author(dict(name=name, email=email, maintainer=True))
+                    self.cleanup_author(dict(name=name, email=email, maintainer=True))
                 )
         return authors
 

tests/test_pkgmanifest.py

@@ -425,6 +425,94 @@ def test_package_json_model():
     assert mp.as_dict()["system"] == ["darwin_x86_64"]
 
 
+def test_examples_from_dir(tmpdir_factory):
+    package_dir = tmpdir_factory.mktemp("project")
+    package_dir.join("library.json").write('{"name": "pkg", "version": "1.0.0"}')
+    examples_dir = package_dir.mkdir("examples")
+
+    # PlatformIO project #1
+    pio_dir = examples_dir.mkdir("PlatformIO").mkdir("hello")
+    pio_dir.join("platformio.ini").write("")
+    pio_dir.mkdir("include").join("main.h").write("")
+    pio_dir.mkdir("src").join("main.cpp").write("")
+
+    # wiring examples
+    examples_dir.mkdir("SomeSketchIno").join("SomeSketchIno.ino").write("")
+    examples_dir.mkdir("SomeSketchPde").join("SomeSketchPde.pde").write("")
+
+    # custom examples
+    demo_dir = examples_dir.mkdir("demo")
+    demo_dir.join("demo.cpp").write("")
+    demo_dir.join("demo.h").write("")
+    demo_dir.join("util.h").write("")
+
+    # PlatformIO project #2
+    pio_dir = examples_dir.mkdir("world")
+    pio_dir.join("platformio.ini").write("")
+    pio_dir.join("README").write("")
+    pio_dir.join("extra.py").write("")
+    pio_dir.mkdir("include").join("world.h").write("")
+    pio_dir.mkdir("src").join("world.c").write("")
+
+    # invalid example
+    examples_dir.mkdir("invalid-example").join("hello.json")
+
+    # Do testing
+
+    data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict()
+    assert isinstance(data["examples"], list)
+    assert len(data["examples"]) == 5
+
+    def _sort_examples(items):
+        for i, item in enumerate(items):
+            items[i]["files"] = sorted(item["files"])
+        return sorted(items, key=lambda item: item["name"])
+
+    data["examples"] = _sort_examples(data["examples"])
+    model = ManifestModel(**data)
+    assert model == ManifestModel(
+        **{
+            "version": "1.0.0",
+            "name": "pkg",
+            "examples": _sort_examples(
+                [
+                    {
+                        "name": "PlatformIO/hello",
+                        "base": "examples/PlatformIO/hello",
+                        "files": ["platformio.ini", "include/main.h", "src/main.cpp"],
+                    },
+                    {
+                        "name": "SomeSketchIno",
+                        "base": "examples/SomeSketchIno",
+                        "files": ["SomeSketchIno.ino"],
+                    },
+                    {
+                        "name": "SomeSketchPde",
+                        "base": "examples/SomeSketchPde",
+                        "files": ["SomeSketchPde.pde"],
+                    },
+                    {
+                        "name": "demo",
+                        "base": "examples/demo",
+                        "files": ["demo.h", "util.h", "demo.cpp"],
+                    },
+                    {
+                        "name": "world",
+                        "base": "examples/world",
+                        "files": [
+                            "platformio.ini",
+                            "include/world.h",
+                            "src/world.c",
+                            "README",
+                            "extra.py",
+                        ],
+                    },
+                ]
+            ),
+        }
+    )
+
+
 def test_broken_models():
     # non-strict mode
     assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4
@@ -453,9 +541,7 @@ def test_broken_models():
         )
 
     # broken value for DataModel
-    with pytest.raises(
+    with pytest.raises(DataFieldException, match="Value should be type of dict"):
-        DataFieldException, match="Value should be type of dict, not `<type 'str'>"
-    ):
         assert StrictManifestModel(
             name="MyPackage",
             description="MyDescription",