Introduce DataModel, package manifest parser and base manifest model

platformio/builder/tools/piomisc.py

@@ -322,7 +322,7 @@ def ConfigureDebugFlags(env):
     env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"])
 
     debug_flags = ["-Og", "-g3", "-ggdb3"]
-    for scope in ("ASFLAGS", "CCFLAGS",):
+    for scope in ("ASFLAGS", "CCFLAGS"):
         _cleanup_debug_flags(scope)
         env.Append(**{scope: debug_flags})
 

platformio/commands/lib.py

@@ -14,8 +14,8 @@
 
 # pylint: disable=too-many-branches, too-many-locals
 
+import os
 import time
-from os.path import isdir, join
 
 import click
 import semantic_version
@@ -25,6 +25,7 @@ from platformio import exception, util
 from platformio.commands import PlatformioCLI
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
+from platformio.package.manifest.parser import ManifestFactory
 from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import get_project_dir, is_platformio_project
@@ -104,13 +105,13 @@ def cli(ctx, **options):
         if not is_platformio_project(storage_dir):
             ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
             continue
-        config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini"))
+        config = ProjectConfig.get_instance(os.path.join(storage_dir, "platformio.ini"))
         config.validate(options["environment"], silent=in_silence)
         libdeps_dir = config.get_optional_dir("libdeps")
         for env in config.envs():
             if options["environment"] and env not in options["environment"]:
                 continue
-            storage_dir = join(libdeps_dir, env)
+            storage_dir = os.path.join(libdeps_dir, env)
             ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
             ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get(
                 "env:" + env, "lib_deps", []
@@ -169,7 +170,7 @@ def lib_install(  # pylint: disable=too-many-arguments
     input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
     project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
     for input_dir in input_dirs:
-        config = ProjectConfig.get_instance(join(input_dir, "platformio.ini"))
+        config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini"))
         config.validate(project_environments)
         for env in config.envs():
             if project_environments and env not in project_environments:
@@ -231,7 +232,7 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output):
         if only_check and json_output:
             result = []
             for library in _libraries:
-                pkg_dir = library if isdir(library) else None
+                pkg_dir = library if os.path.isdir(library) else None
                 requirements = None
                 url = None
                 if not pkg_dir:
@@ -492,6 +493,9 @@ def lib_register(config_url):
     if not config_url.startswith("http://") and not config_url.startswith("https://"):
         raise exception.InvalidLibConfURL(config_url)
 
+    manifest = ManifestFactory.new_from_url(config_url)
+    assert set(["name", "version"]) & set(list(manifest.as_dict()))
+
     result = util.get_api_result("/lib/register", data=dict(config_url=config_url))
     if "message" in result and result["message"]:
         click.secho(

platformio/compat.py

@@ -15,6 +15,7 @@
 # pylint: disable=unused-import, no-name-in-module, import-error,
 # pylint: disable=no-member, undefined-variable
 
+import inspect
 import json
 import os
 import re
@@ -29,6 +30,15 @@ def get_filesystem_encoding():
     return sys.getfilesystemencoding() or sys.getdefaultencoding()
 
 
+def get_class_attributes(cls):
+    attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a)))
+    return {
+        a[0]: a[1]
+        for a in attributes
+        if not (a[0].startswith("__") and a[0].endswith("__"))
+    }
+
+
 if PY2:
     import imp
 

platformio/datamodel.py

@@ -0,0 +1,154 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import re
+
+from platformio.compat import get_class_attributes, string_types
+from platformio.exception import PlatformioException
+
+# pylint: disable=too-many-instance-attributes
+# pylint: disable=redefined-builtin, too-many-arguments
+
+
+class DataModelException(PlatformioException):
+    pass
+
+
+class DataField(object):
+    def __init__(
+        self,
+        default=None,
+        type=str,
+        required=False,
+        min_length=None,
+        max_length=None,
+        regex=None,
+        validate_factory=None,
+        title=None,
+    ):
+        self.default = default
+        self.type = type
+        self.required = required
+        self.min_length = min_length
+        self.max_length = max_length
+        self.regex = regex
+        self.validate_factory = validate_factory
+        self.title = title
+
+        self._value = None
+
+    def __repr__(self):
+        return '<DataField %s="%s">' % (
+            self.title,
+            self.default if self._value is None else self._value,
+        )
+
+    def validate(self, value, parent, attr):
+        if self.title is None:
+            self.title = attr.title()
+        try:
+            if self.required and value is None:
+                raise ValueError("Required field, value is None")
+            if self.validate_factory is not None:
+                value = self.validate_factory(value)
+            if value is None:
+                return self.default
+            if issubclass(self.type, (str, list, bool)):
+                return getattr(self, "_validate_%s_value" % self.type.__name__)(value)
+        except (AssertionError, ValueError) as e:
+            raise DataModelException(
+                "%s for %s.%s" % (str(e), parent.__class__.__name__, attr)
+            )
+        return value
+
+    def _validate_str_value(self, value):
+        if not isinstance(value, string_types):
+            value = str(value)
+        assert self.min_length is None or len(value) >= self.min_length, (
+            "Minimum allowed length is %d characters" % self.min_length
+        )
+        assert self.max_length is None or len(value) <= self.max_length, (
+            "Maximum allowed length is %d characters" % self.max_length
+        )
+        assert self.regex is None or re.match(
+            self.regex, value
+        ), "Value `%s` does not match RegExp `%s` pattern" % (value, self.regex)
+        return value
+
+    @staticmethod
+    def _validate_bool_value(value):
+        if isinstance(value, bool):
+            return value
+        return str(value).lower() in ("true", "yes", "1")
+
+
+class DataModel(object):
+    __PRIVATE_ATTRIBUTES__ = ("__PRIVATE_ATTRIBUTES__", "_init_type", "as_dict")
+
+    def __init__(self, data=None):
+        data = data or {}
+        assert isinstance(data, dict)
+
+        for attr, scheme_or_model in get_class_attributes(self).items():
+            if attr in self.__PRIVATE_ATTRIBUTES__:
+                continue
+            if isinstance(scheme_or_model, list):
+                assert len(scheme_or_model) == 1
+                if data.get(attr) is None:
+                    setattr(self, attr, None)
+                    continue
+
+                if not isinstance(data.get(attr), list):
+                    raise DataModelException("Value should be a list for %s" % (attr))
+                setattr(
+                    self,
+                    attr,
+                    [
+                        self._init_type(scheme_or_model[0], v, attr)
+                        for v in data.get(attr)
+                    ],
+                )
+            else:
+                setattr(
+                    self, attr, self._init_type(scheme_or_model, data.get(attr), attr)
+                )
+
+    def __repr__(self):
+        attrs = []
+        for name, value in get_class_attributes(self).items():
+            if name in self.__PRIVATE_ATTRIBUTES__:
+                continue
+            attrs.append('%s="%s"' % (name, value))
+        return "<%s %s>" % (self.__class__.__name__, " ".join(attrs))
+
+    def _init_type(self, type_, value, attr):
+        if inspect.isclass(type_) and issubclass(type_, DataModel):
+            return type_(value)
+        if isinstance(type_, DataField):
+            return type_.validate(value, parent=self, attr=attr)
+        raise DataModelException("Undeclared or unknown data type for %s" % attr)
+
+    def as_dict(self):
+        result = {}
+        for name, value in get_class_attributes(self).items():
+            if name in self.__PRIVATE_ATTRIBUTES__:
+                continue
+            if isinstance(value, DataModel):
+                result[name] = value.as_dict()
+            elif value and isinstance(value, list) and isinstance(value[0], DataModel):
+                result[name] = value[0].as_dict()
+            else:
+                result[name] = value
+        return result

platformio/package/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/package/manifest/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/package/manifest/model.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import semantic_version
+
+from platformio.datamodel import DataField, DataModel
+
+
+class AuthorModel(DataModel):
+    name = DataField(max_length=50, required=True)
+    email = DataField(max_length=50)
+    maintainer = DataField(default=False, type=bool)
+    url = DataField(max_length=255)
+
+
+class RepositoryModel(DataModel):
+    type = DataField(max_length=3, required=True)
+    url = DataField(max_length=255, required=True)
+    branch = DataField(max_length=50)
+
+
+class ExportModel(DataModel):
+    include = [DataField()]
+    exclude = [DataField()]
+
+
+class ManifestModel(DataModel):
+
+    name = DataField(max_length=100, required=True)
+    version = DataField(
+        required=True,
+        max_length=50,
+        validate_factory=lambda v: v if semantic_version.Version.coerce(v) else None,
+    )
+
+    description = DataField(max_length=1000)
+    keywords = [DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")]
+    authors = [AuthorModel]
+
+    homepage = DataField(max_length=255)
+    license = DataField(max_length=255)
+    platforms = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")]
+    frameworks = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")]
+
+    repository = RepositoryModel
+    export = ExportModel

platformio/package/manifest/parser.py

@@ -0,0 +1,369 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+
+import requests
+
+from platformio.compat import get_class_attributes, string_types
+from platformio.exception import PlatformioException
+from platformio.fs import get_file_contents
+from platformio.package.manifest.model import ManifestModel
+
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+
+
+class ManifestException(PlatformioException):
+    pass
+
+
+class ManifestParserException(ManifestException):
+    pass
+
+
+class ManifestFileType(object):
+    PLATFORM_JSON = "platform.json"
+    LIBRARY_JSON = "library.json"
+    LIBRARY_PROPERTIES = "library.properties"
+    MODULE_JSON = "module.json"
+    PACKAGE_JSON = "package.json"
+
+    @classmethod
+    def from_uri(cls, uri):
+        if uri.endswith(".properties"):
+            return ManifestFileType.LIBRARY_PROPERTIES
+        if uri.endswith("platform.json"):
+            return ManifestFileType.PLATFORM_JSON
+        if uri.endswith("module.json"):
+            return ManifestFileType.MODULE_JSON
+        if uri.endswith("package.json"):
+            return ManifestFileType.PACKAGE_JSON
+        return ManifestFileType.LIBRARY_JSON
+
+
+class ManifestFactory(object):
+    @staticmethod
+    def type_to_clsname(type_):
+        type_ = type_.replace(".", " ")
+        type_ = type_.title()
+        return "%sManifestParser" % type_.replace(" ", "")
+
+    @staticmethod
+    def new_from_file(path):
+        if not path or not os.path.isfile(path):
+            raise ManifestException("Manifest file does not exist %s" % path)
+        for type_ in get_class_attributes(ManifestFileType).values():
+            if path.endswith(type_):
+                return ManifestFactory.new(get_file_contents(path), type_)
+        raise ManifestException("Unknown manifest file type %s" % path)
+
+    @staticmethod
+    def new_from_url(remote_url):
+        r = requests.get(remote_url)
+        r.raise_for_status()
+        return ManifestFactory.new(
+            r.text, ManifestFileType.from_uri(remote_url), remote_url
+        )
+
+    @staticmethod
+    def new(contents, type_, remote_url=None):
+        clsname = ManifestFactory.type_to_clsname(type_)
+        if clsname not in globals():
+            raise ManifestException("Unknown manifest file type %s" % clsname)
+        mp = globals()[clsname](contents, remote_url)
+        return ManifestModel(mp.as_dict())
+
+
+class BaseManifestParser(object):
+    def __init__(self, contents, remote_url=None):
+        self.remote_url = remote_url
+        self._data = self.parse(contents)
+
+    def parse(self, contents):
+        raise NotImplementedError
+
+    def as_dict(self):
+        return self._data
+
+    @staticmethod
+    def _cleanup_author(author):
+        if author.get("email"):
+            author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"])
+        return author
+
+    @staticmethod
+    def parse_author_name_and_email(raw):
+        if raw == "None" or "://" in raw:
+            return (None, None)
+        name = raw
+        email = None
+        for ldel, rdel in [("<", ">"), ("(", ")")]:
+            if ldel in raw and rdel in raw:
+                name = raw[: raw.index(ldel)]
+                email = raw[raw.index(ldel) + 1 : raw.index(rdel)]
+        return (name.strip(), email.strip() if email else None)
+
+
+class LibraryJsonManifestParser(BaseManifestParser):
+    def parse(self, contents):
+        data = json.loads(contents)
+        data = self._process_renamed_fields(data)
+
+        # normalize Union[str, list] fields
+        for k in ("keywords", "platforms", "frameworks"):
+            if k in data:
+                data[k] = self._str_to_list(data[k], sep=",")
+
+        if "authors" in data:
+            data["authors"] = self._parse_authors(data["authors"])
+        if "platforms" in data:
+            data["platforms"] = self._parse_platforms(data["platforms"]) or None
+
+        return data
+
+    @staticmethod
+    def _str_to_list(value, sep=",", lowercase=True):
+        if isinstance(value, string_types):
+            value = value.split(sep)
+        assert isinstance(value, list)
+        result = []
+        for item in value:
+            item = item.strip()
+            if not item:
+                continue
+            if lowercase:
+                item = item.lower()
+            result.append(item)
+        return result
+
+    @staticmethod
+    def _process_renamed_fields(data):
+        if "url" in data:
+            data["homepage"] = data["url"]
+            del data["url"]
+
+        for key in ("include", "exclude"):
+            if key not in data:
+                continue
+            if "export" not in data:
+                data["export"] = {}
+            data["export"][key] = (
+                data[key] if isinstance(data[key], list) else [data[key]]
+            )
+            del data[key]
+
+        return data
+
+    def _parse_authors(self, raw):
+        if not raw:
+            return None
+        # normalize Union[dict, list] fields
+        if not isinstance(raw, list):
+            raw = [raw]
+        return [self._cleanup_author(author) for author in raw]
+
+    @staticmethod
+    def _parse_platforms(raw):
+        assert isinstance(raw, list)
+        result = []
+        # renamed platforms
+        for item in raw:
+            if item == "espressif":
+                item = "espressif8266"
+            result.append(item)
+        return result
+
+
+class ModuleJsonManifestParser(BaseManifestParser):
+    def parse(self, contents):
+        data = json.loads(contents)
+        return dict(
+            name=data["name"],
+            version=data["version"],
+            keywords=data.get("keywords"),
+            description=data["description"],
+            frameworks=["mbed"],
+            platforms=["*"],
+            homepage=data.get("homepage"),
+            export={"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]},
+            authors=self._parse_authors(data.get("author")),
+            license=self._parse_license(data.get("licenses")),
+        )
+
+    def _parse_authors(self, raw):
+        if not raw:
+            return None
+        result = []
+        for author in raw.split(","):
+            name, email = self.parse_author_name_and_email(author)
+            if not name:
+                continue
+            result.append(
+                self._cleanup_author(dict(name=name, email=email, maintainer=False))
+            )
+        return result
+
+    @staticmethod
+    def _parse_license(raw):
+        if not raw or not isinstance(raw, list):
+            return None
+        return raw[0].get("type")
+
+
+class LibraryPropertiesManifestParser(BaseManifestParser):
+    def parse(self, contents):
+        properties = self._parse_properties(contents)
+        repository = self._parse_repository(properties)
+        homepage = properties.get("url")
+        if repository and repository["url"] == homepage:
+            homepage = None
+        return dict(
+            name=properties["name"],
+            version=properties["version"],
+            description=properties["sentence"],
+            frameworks=["arduino"],
+            platforms=self._process_platforms(properties) or ["*"],
+            keywords=self._parse_keywords(properties),
+            authors=self._parse_authors(properties) or None,
+            homepage=homepage,
+            repository=repository or None,
+            export=self._parse_export(),
+        )
+
+    @staticmethod
+    def _parse_properties(contents):
+        data = {}
+        for line in contents.splitlines():
+            line = line.strip()
+            if not line or "=" not in line:
+                continue
+            # skip comments
+            if line.startswith("#"):
+                continue
+            key, value = line.split("=", 1)
+            data[key.strip()] = value.strip()
+
+        required_fields = set(["name", "version", "author", "sentence"])
+        if not set(data.keys()) >= required_fields:
+            raise ManifestParserException(
+                "Missing fields: " + ",".join(required_fields - set(data.keys()))
+            )
+        return data
+
+    @staticmethod
+    def _parse_keywords(properties):
+        result = []
+        for item in re.split(r"[\s/]+", properties.get("category", "uncategorized")):
+            item = item.strip()
+            if not item:
+                continue
+            result.append(item.lower())
+        return result
+
+    @staticmethod
+    def _process_platforms(properties):
+        result = []
+        platforms_map = {
+            "avr": "atmelavr",
+            "sam": "atmelsam",
+            "samd": "atmelsam",
+            "esp8266": "espressif8266",
+            "esp32": "espressif32",
+            "arc32": "intel_arc32",
+            "stm32": "ststm32",
+        }
+        for arch in properties.get("architectures", "").split(","):
+            if "particle-" in arch:
+                raise ManifestParserException("Particle is not supported yet")
+            arch = arch.strip()
+            if not arch:
+                continue
+            if arch == "*":
+                return ["*"]
+            if arch in platforms_map:
+                result.append(platforms_map[arch])
+        return result
+
+    def _parse_authors(self, properties):
+        authors = []
+        for author in properties["author"].split(","):
+            name, email = self.parse_author_name_and_email(author)
+            if not name:
+                continue
+            authors.append(
+                self._cleanup_author(dict(name=name, email=email, maintainer=False))
+            )
+        for author in properties.get("maintainer", "").split(","):
+            name, email = self.parse_author_name_and_email(author)
+            if not name:
+                continue
+            found = False
+            for item in authors:
+                if item["name"].lower() != name.lower():
+                    continue
+                found = True
+                item["maintainer"] = True
+                if not item["email"]:
+                    item["email"] = email
+            if not found:
+                authors.append(
+                    self._cleanup_author(dict(name=name, email=email, maintainer=True))
+                )
+        return authors
+
+    def _parse_repository(self, properties):
+        if self.remote_url:
+            repo_parse = urlparse(self.remote_url)
+            repo_path_tokens = repo_parse.path[1:].split("/")[:-1]
+            if "github" in repo_parse.netloc:
+                return dict(
+                    type="git",
+                    url="%s://github.com/%s"
+                    % (repo_parse.scheme, "/".join(repo_path_tokens[:2])),
+                )
+            if "raw" in repo_path_tokens:
+                return dict(
+                    type="git",
+                    url="%s://%s/%s"
+                    % (
+                        repo_parse.scheme,
+                        repo_parse.netloc,
+                        "/".join(repo_path_tokens[: repo_path_tokens.index("raw")]),
+                    ),
+                )
+        if properties.get("url", "").startswith("https://github.com"):
+            return dict(type="git", url=properties["url"])
+        return None
+
+    def _parse_export(self):
+        include = None
+        if self.remote_url:
+            repo_parse = urlparse(self.remote_url)
+            repo_path_tokens = repo_parse.path[1:].split("/")[:-1]
+            if "github" in repo_parse.netloc:
+                include = "/".join(repo_path_tokens[3:]) or None
+            elif "raw" in repo_path_tokens:
+                include = (
+                    "/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :])
+                    or None
+                )
+        return {
+            "include": include,
+            "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
+        }

tests/test_pkgmanifest.py

@@ -13,32 +13,196 @@
 # limitations under the License.
 
 import pytest
-import requests
+
+from platformio.package.manifest import parser
 
 
-def validate_response(r):
+def test_library_json_parser():
-    assert r.status_code == 200, r.url
+    contents = """
-    assert int(r.headers["Content-Length"]) > 0, r.url
+{
-    assert r.headers["Content-Type"] in ("application/gzip", "application/octet-stream")
+    "name": "TestPackage",
+    "keywords": "kw1, KW2, kw3",
+    "platforms": ["atmelavr", "espressif"],
+    "url": "http://old.url.format",
+    "exclude": [".gitignore", "tests"],
+    "include": "mylib"
+}
+"""
+    mp = parser.LibraryJsonManifestParser(contents)
+    assert sorted(mp.as_dict().items()) == sorted(
+        {
+            "name": "TestPackage",
+            "platforms": ["atmelavr", "espressif8266"],
+            "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]},
+            "keywords": ["kw1", "kw2", "kw3"],
+            "homepage": "http://old.url.format",
+        }.items()
+    )
 
 
-def test_packages():
+def test_module_json_parser():
-    pkgs_manifest = requests.get(
+    contents = """
-        "https://dl.bintray.com/platformio/dl-packages/manifest.json"
+{
-    ).json()
+  "author": "Name Surname <name@surname.com>",
-    assert isinstance(pkgs_manifest, dict)
+  "description": "This is Yotta library",
-    items = []
+  "homepage": "https://yottabuild.org",
-    for _, variants in pkgs_manifest.items():
+  "keywords": [
-        for item in variants:
+    "mbed",
-            items.append(item)
+    "Yotta"
+  ],
+  "licenses": [
+    {
+      "type": "Apache-2.0",
+      "url": "https://spdx.org/licenses/Apache-2.0"
+    }
+  ],
+  "name": "YottaLibrary",
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:username/repo.git"
+  },
+  "version": "1.2.3"
+}
+"""
+    mp = parser.ModuleJsonManifestParser(contents)
+    assert sorted(mp.as_dict().items()) == sorted(
+        {
+            "name": "YottaLibrary",
+            "description": "This is Yotta library",
+            "homepage": "https://yottabuild.org",
+            "keywords": ["mbed", "Yotta"],
+            "license": "Apache-2.0",
+            "platforms": ["*"],
+            "frameworks": ["mbed"],
+            "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]},
+            "authors": [
+                {
+                    "maintainer": False,
+                    "email": "name@surname.com",
+                    "name": "Name Surname",
+                }
+            ],
+            "version": "1.2.3",
+        }.items()
+    )
 
-    for item in items:
-        assert item["url"].endswith(".tar.gz"), item
 
-        r = requests.head(item["url"], allow_redirects=True)
+def test_library_properties_parser():
-        validate_response(r)
+    # test missed fields
+    with pytest.raises(parser.ManifestParserException):
+        parser.LibraryPropertiesManifestParser("name=TestPackage")
 
-        if "X-Checksum-Sha1" not in r.headers:
+    # Base
-            return pytest.skip("X-Checksum-Sha1 is not provided")
+    contents = """
+name=TestPackage
+version=1.2.3
+author=SomeAuthor <info AT author.com>
+sentence=This is Arduino library
+"""
+    mp = parser.LibraryPropertiesManifestParser(contents)
+    assert sorted(mp.as_dict().items()) == sorted(
+        {
+            "name": "TestPackage",
+            "version": "1.2.3",
+            "description": "This is Arduino library",
+            "repository": None,
+            "platforms": ["*"],
+            "frameworks": ["arduino"],
+            "export": {
+                "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
+                "include": None,
+            },
+            "authors": [
+                {"maintainer": False, "email": "info@author.com", "name": "SomeAuthor"}
+            ],
+            "keywords": ["uncategorized"],
+            "homepage": None,
+        }.items()
+    )
 
-        assert item["sha1"] == r.headers.get("X-Checksum-Sha1")[0:40], item
+    # Platforms ALL
+    mp = parser.LibraryPropertiesManifestParser("architectures=*\n" + contents)
+    assert mp.as_dict()["platforms"] == ["*"]
+    # Platforms specific
+    mp = parser.LibraryPropertiesManifestParser("architectures=avr, esp32\n" + contents)
+    assert mp.as_dict()["platforms"] == ["atmelavr", "espressif32"]
+
+    # Remote URL
+    mp = parser.LibraryPropertiesManifestParser(
+        contents,
+        remote_url=(
+            "https://raw.githubusercontent.com/username/reponame/master/"
+            "libraries/TestPackage/library.properties"
+        ),
+    )
+    assert mp.as_dict()["export"] == {
+        "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
+        "include": "libraries/TestPackage",
+    }
+
+    # Hope page
+    mp = parser.LibraryPropertiesManifestParser(
+        "url=https://github.com/username/reponame.git\n" + contents
+    )
+    assert mp.as_dict()["homepage"] is None
+    assert mp.as_dict()["repository"] == {
+        "type": "git",
+        "url": "https://github.com/username/reponame.git",
+    }
+
+
+def test_library_json_model():
+    contents = """
+{
+  "name": "ArduinoJson",
+  "keywords": "JSON, rest, http, web",
+  "description": "An elegant and efficient JSON library for embedded systems",
+  "homepage": "https://arduinojson.org",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bblanchon/ArduinoJson.git"
+  },
+  "version": "6.12.0",
+  "authors": {
+    "name": "Benoit Blanchon",
+    "url": "https://blog.benoitblanchon.fr"
+  },
+  "exclude": [
+    "fuzzing",
+    "scripts",
+    "test",
+    "third-party"
+  ],
+  "frameworks": "arduino",
+  "platforms": "*",
+  "license": "MIT"
+}
+"""
+    model = parser.ManifestFactory.new(contents, parser.ManifestFileType.LIBRARY_JSON)
+    assert sorted(model.as_dict().items()) == sorted(
+        {
+            "name": "ArduinoJson",
+            "keywords": ["json", "rest", "http", "web"],
+            "description": "An elegant and efficient JSON library for embedded systems",
+            "homepage": "https://arduinojson.org",
+            "repository": {
+                "url": "https://github.com/bblanchon/ArduinoJson.git",
+                "type": "git",
+                "branch": None,
+            },
+            "version": "6.12.0",
+            "authors": {
+                "url": "https://blog.benoitblanchon.fr",
+                "maintainer": False,
+                "email": None,
+                "name": "Benoit Blanchon",
+            },
+            "export": {
+                "include": None,
+                "exclude": ["fuzzing", "scripts", "test", "third-party"],
+            },
+            "frameworks": ["arduino"],
+            "platforms": ["*"],
+            "license": "MIT",
+        }.items()
+    )