From 5f55c183734c9dcb3734875b6f9c2b2dc3e1f55b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Sep 2019 17:59:06 +0300 Subject: [PATCH] Introduce DataModel, package manifest parser and base manifest model --- platformio/builder/tools/piomisc.py | 2 +- platformio/commands/lib.py | 14 +- platformio/compat.py | 10 + platformio/datamodel.py | 154 ++++++++++ platformio/package/__init__.py | 13 + platformio/package/manifest/__init__.py | 13 + platformio/package/manifest/model.py | 57 ++++ platformio/package/manifest/parser.py | 369 ++++++++++++++++++++++++ tests/test_pkgmanifest.py | 206 +++++++++++-- 9 files changed, 811 insertions(+), 27 deletions(-) create mode 100644 platformio/datamodel.py create mode 100644 platformio/package/__init__.py create mode 100644 platformio/package/manifest/__init__.py create mode 100644 platformio/package/manifest/model.py create mode 100644 platformio/package/manifest/parser.py diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 91184381..bea63a31 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -322,7 +322,7 @@ def ConfigureDebugFlags(env): env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) debug_flags = ["-Og", "-g3", "-ggdb3"] - for scope in ("ASFLAGS", "CCFLAGS",): + for scope in ("ASFLAGS", "CCFLAGS"): _cleanup_debug_flags(scope) env.Append(**{scope: debug_flags}) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 83e6cff4..8f61769f 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -14,8 +14,8 @@ # pylint: disable=too-many-branches, too-many-locals +import os import time -from os.path import isdir, join import click import semantic_version @@ -25,6 +25,7 @@ from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib +from platformio.package.manifest.parser import ManifestFactory from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -104,13 +105,13 @@ def cli(ctx, **options): if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue - config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini")) + config = ProjectConfig.get_instance(os.path.join(storage_dir, "platformio.ini")) config.validate(options["environment"], silent=in_silence) libdeps_dir = config.get_optional_dir("libdeps") for env in config.envs(): if options["environment"] and env not in options["environment"]: continue - storage_dir = join(libdeps_dir, env) + storage_dir = os.path.join(libdeps_dir, env) ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( "env:" + env, "lib_deps", [] @@ -169,7 +170,7 @@ def lib_install( # pylint: disable=too-many-arguments input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] for input_dir in input_dirs: - config = ProjectConfig.get_instance(join(input_dir, "platformio.ini")) + config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini")) config.validate(project_environments) for env in config.envs(): if project_environments and env not in project_environments: @@ -231,7 +232,7 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): if only_check and json_output: result = [] for library in _libraries: - pkg_dir = library if isdir(library) else None + pkg_dir = library if os.path.isdir(library) else None requirements = None url = None if not pkg_dir: @@ -492,6 +493,9 @@ def lib_register(config_url): if not config_url.startswith("http://") and not config_url.startswith("https://"): raise exception.InvalidLibConfURL(config_url) + manifest = ManifestFactory.new_from_url(config_url) + assert set(["name", "version"]) & set(list(manifest.as_dict())) + result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: click.secho( diff --git a/platformio/compat.py b/platformio/compat.py index 64d437a1..15942556 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -15,6 +15,7 @@ # pylint: disable=unused-import, no-name-in-module, import-error, # pylint: disable=no-member, undefined-variable +import inspect import json import os import re @@ -29,6 +30,15 @@ def get_filesystem_encoding(): return sys.getfilesystemencoding() or sys.getdefaultencoding() +def get_class_attributes(cls): + attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a))) + return { + a[0]: a[1] + for a in attributes + if not (a[0].startswith("__") and a[0].endswith("__")) + } + + if PY2: import imp diff --git a/platformio/datamodel.py b/platformio/datamodel.py new file mode 100644 index 00000000..39547d59 --- /dev/null +++ b/platformio/datamodel.py @@ -0,0 +1,154 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import re + +from platformio.compat import get_class_attributes, string_types +from platformio.exception import PlatformioException + +# pylint: disable=too-many-instance-attributes +# pylint: disable=redefined-builtin, too-many-arguments + + +class DataModelException(PlatformioException): + pass + + +class DataField(object): + def __init__( + self, + default=None, + type=str, + required=False, + min_length=None, + max_length=None, + regex=None, + validate_factory=None, + title=None, + ): + self.default = default + self.type = type + self.required = required + self.min_length = min_length + self.max_length = max_length + self.regex = regex + self.validate_factory = validate_factory + self.title = title + + self._value = None + + def __repr__(self): + return '' % ( + self.title, + self.default if self._value is None else self._value, + ) + + def validate(self, value, parent, attr): + if self.title is None: + self.title = attr.title() + try: + if self.required and value is None: + raise ValueError("Required field, value is None") + if self.validate_factory is not None: + value = self.validate_factory(value) + if value is None: + return self.default + if issubclass(self.type, (str, list, bool)): + return getattr(self, "_validate_%s_value" % self.type.__name__)(value) + except (AssertionError, ValueError) as e: + raise DataModelException( + "%s for %s.%s" % (str(e), parent.__class__.__name__, attr) + ) + return value + + def _validate_str_value(self, value): + if not isinstance(value, string_types): + value = str(value) + assert self.min_length is None or len(value) >= self.min_length, ( + "Minimum allowed length is %d characters" % self.min_length + ) + assert self.max_length is None or len(value) <= self.max_length, ( + "Maximum allowed length is %d characters" % self.max_length + ) + assert self.regex is None or re.match( + self.regex, value + ), "Value `%s` does not match RegExp `%s` pattern" % (value, self.regex) + return value + + @staticmethod + def _validate_bool_value(value): + if isinstance(value, bool): + return value + return str(value).lower() in ("true", "yes", "1") + + +class DataModel(object): + __PRIVATE_ATTRIBUTES__ = ("__PRIVATE_ATTRIBUTES__", "_init_type", "as_dict") + + def __init__(self, data=None): + data = data or {} + assert isinstance(data, dict) + + for attr, scheme_or_model in get_class_attributes(self).items(): + if attr in self.__PRIVATE_ATTRIBUTES__: + continue + if isinstance(scheme_or_model, list): + assert len(scheme_or_model) == 1 + if data.get(attr) is None: + setattr(self, attr, None) + continue + + if not isinstance(data.get(attr), list): + raise DataModelException("Value should be a list for %s" % (attr)) + setattr( + self, + attr, + [ + self._init_type(scheme_or_model[0], v, attr) + for v in data.get(attr) + ], + ) + else: + setattr( + self, attr, self._init_type(scheme_or_model, data.get(attr), attr) + ) + + def __repr__(self): + attrs = [] + for name, value in get_class_attributes(self).items(): + if name in self.__PRIVATE_ATTRIBUTES__: + continue + attrs.append('%s="%s"' % (name, value)) + return "<%s %s>" % (self.__class__.__name__, " ".join(attrs)) + + def _init_type(self, type_, value, attr): + if inspect.isclass(type_) and issubclass(type_, DataModel): + return type_(value) + if isinstance(type_, DataField): + return type_.validate(value, parent=self, attr=attr) + raise DataModelException("Undeclared or unknown data type for %s" % attr) + + def as_dict(self): + result = {} + for name, value in get_class_attributes(self).items(): + if name in self.__PRIVATE_ATTRIBUTES__: + continue + if isinstance(value, DataModel): + result[name] = value.as_dict() + elif value and isinstance(value, list) and isinstance(value[0], DataModel): + result[name] = value[0].as_dict() + else: + result[name] = value + return result diff --git a/platformio/package/__init__.py b/platformio/package/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/package/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manifest/__init__.py b/platformio/package/manifest/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/package/manifest/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py new file mode 100644 index 00000000..63bad52e --- /dev/null +++ b/platformio/package/manifest/model.py @@ -0,0 +1,57 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import semantic_version + +from platformio.datamodel import DataField, DataModel + + +class AuthorModel(DataModel): + name = DataField(max_length=50, required=True) + email = DataField(max_length=50) + maintainer = DataField(default=False, type=bool) + url = DataField(max_length=255) + + +class RepositoryModel(DataModel): + type = DataField(max_length=3, required=True) + url = DataField(max_length=255, required=True) + branch = DataField(max_length=50) + + +class ExportModel(DataModel): + include = [DataField()] + exclude = [DataField()] + + +class ManifestModel(DataModel): + + name = DataField(max_length=100, required=True) + version = DataField( + required=True, + max_length=50, + validate_factory=lambda v: v if semantic_version.Version.coerce(v) else None, + ) + + description = DataField(max_length=1000) + keywords = [DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")] + authors = [AuthorModel] + + homepage = DataField(max_length=255) + license = DataField(max_length=255) + platforms = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")] + frameworks = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")] + + repository = RepositoryModel + export = ExportModel diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py new file mode 100644 index 00000000..faf576c1 --- /dev/null +++ b/platformio/package/manifest/parser.py @@ -0,0 +1,369 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re + +import requests + +from platformio.compat import get_class_attributes, string_types +from platformio.exception import PlatformioException +from platformio.fs import get_file_contents +from platformio.package.manifest.model import ManifestModel + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class ManifestException(PlatformioException): + pass + + +class ManifestParserException(ManifestException): + pass + + +class ManifestFileType(object): + PLATFORM_JSON = "platform.json" + LIBRARY_JSON = "library.json" + LIBRARY_PROPERTIES = "library.properties" + MODULE_JSON = "module.json" + PACKAGE_JSON = "package.json" + + @classmethod + def from_uri(cls, uri): + if uri.endswith(".properties"): + return ManifestFileType.LIBRARY_PROPERTIES + if uri.endswith("platform.json"): + return ManifestFileType.PLATFORM_JSON + if uri.endswith("module.json"): + return ManifestFileType.MODULE_JSON + if uri.endswith("package.json"): + return ManifestFileType.PACKAGE_JSON + return ManifestFileType.LIBRARY_JSON + + +class ManifestFactory(object): + @staticmethod + def type_to_clsname(type_): + type_ = type_.replace(".", " ") + type_ = type_.title() + return "%sManifestParser" % type_.replace(" ", "") + + @staticmethod + def new_from_file(path): + if not path or not os.path.isfile(path): + raise ManifestException("Manifest file does not exist %s" % path) + for type_ in get_class_attributes(ManifestFileType).values(): + if path.endswith(type_): + return ManifestFactory.new(get_file_contents(path), type_) + raise ManifestException("Unknown manifest file type %s" % path) + + @staticmethod + def new_from_url(remote_url): + r = requests.get(remote_url) + r.raise_for_status() + return ManifestFactory.new( + r.text, ManifestFileType.from_uri(remote_url), remote_url + ) + + @staticmethod + def new(contents, type_, remote_url=None): + clsname = ManifestFactory.type_to_clsname(type_) + if clsname not in globals(): + raise ManifestException("Unknown manifest file type %s" % clsname) + mp = globals()[clsname](contents, remote_url) + return ManifestModel(mp.as_dict()) + + +class BaseManifestParser(object): + def __init__(self, contents, remote_url=None): + self.remote_url = remote_url + self._data = self.parse(contents) + + def parse(self, contents): + raise NotImplementedError + + def as_dict(self): + return self._data + + @staticmethod + def _cleanup_author(author): + if author.get("email"): + author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) + return author + + @staticmethod + def parse_author_name_and_email(raw): + if raw == "None" or "://" in raw: + return (None, None) + name = raw + email = None + for ldel, rdel in [("<", ">"), ("(", ")")]: + if ldel in raw and rdel in raw: + name = raw[: raw.index(ldel)] + email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + return (name.strip(), email.strip() if email else None) + + +class LibraryJsonManifestParser(BaseManifestParser): + def parse(self, contents): + data = json.loads(contents) + data = self._process_renamed_fields(data) + + # normalize Union[str, list] fields + for k in ("keywords", "platforms", "frameworks"): + if k in data: + data[k] = self._str_to_list(data[k], sep=",") + + if "authors" in data: + data["authors"] = self._parse_authors(data["authors"]) + if "platforms" in data: + data["platforms"] = self._parse_platforms(data["platforms"]) or None + + return data + + @staticmethod + def _str_to_list(value, sep=",", lowercase=True): + if isinstance(value, string_types): + value = value.split(sep) + assert isinstance(value, list) + result = [] + for item in value: + item = item.strip() + if not item: + continue + if lowercase: + item = item.lower() + result.append(item) + return result + + @staticmethod + def _process_renamed_fields(data): + if "url" in data: + data["homepage"] = data["url"] + del data["url"] + + for key in ("include", "exclude"): + if key not in data: + continue + if "export" not in data: + data["export"] = {} + data["export"][key] = ( + data[key] if isinstance(data[key], list) else [data[key]] + ) + del data[key] + + return data + + def _parse_authors(self, raw): + if not raw: + return None + # normalize Union[dict, list] fields + if not isinstance(raw, list): + raw = [raw] + return [self._cleanup_author(author) for author in raw] + + @staticmethod + def _parse_platforms(raw): + assert isinstance(raw, list) + result = [] + # renamed platforms + for item in raw: + if item == "espressif": + item = "espressif8266" + result.append(item) + return result + + +class ModuleJsonManifestParser(BaseManifestParser): + def parse(self, contents): + data = json.loads(contents) + return dict( + name=data["name"], + version=data["version"], + keywords=data.get("keywords"), + description=data["description"], + frameworks=["mbed"], + platforms=["*"], + homepage=data.get("homepage"), + export={"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, + authors=self._parse_authors(data.get("author")), + license=self._parse_license(data.get("licenses")), + ) + + def _parse_authors(self, raw): + if not raw: + return None + result = [] + for author in raw.split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + result.append( + self._cleanup_author(dict(name=name, email=email, maintainer=False)) + ) + return result + + @staticmethod + def _parse_license(raw): + if not raw or not isinstance(raw, list): + return None + return raw[0].get("type") + + +class LibraryPropertiesManifestParser(BaseManifestParser): + def parse(self, contents): + properties = self._parse_properties(contents) + repository = self._parse_repository(properties) + homepage = properties.get("url") + if repository and repository["url"] == homepage: + homepage = None + return dict( + name=properties["name"], + version=properties["version"], + description=properties["sentence"], + frameworks=["arduino"], + platforms=self._process_platforms(properties) or ["*"], + keywords=self._parse_keywords(properties), + authors=self._parse_authors(properties) or None, + homepage=homepage, + repository=repository or None, + export=self._parse_export(), + ) + + @staticmethod + def _parse_properties(contents): + data = {} + for line in contents.splitlines(): + line = line.strip() + if not line or "=" not in line: + continue + # skip comments + if line.startswith("#"): + continue + key, value = line.split("=", 1) + data[key.strip()] = value.strip() + + required_fields = set(["name", "version", "author", "sentence"]) + if not set(data.keys()) >= required_fields: + raise ManifestParserException( + "Missing fields: " + ",".join(required_fields - set(data.keys())) + ) + return data + + @staticmethod + def _parse_keywords(properties): + result = [] + for item in re.split(r"[\s/]+", properties.get("category", "uncategorized")): + item = item.strip() + if not item: + continue + result.append(item.lower()) + return result + + @staticmethod + def _process_platforms(properties): + result = [] + platforms_map = { + "avr": "atmelavr", + "sam": "atmelsam", + "samd": "atmelsam", + "esp8266": "espressif8266", + "esp32": "espressif32", + "arc32": "intel_arc32", + "stm32": "ststm32", + } + for arch in properties.get("architectures", "").split(","): + if "particle-" in arch: + raise ManifestParserException("Particle is not supported yet") + arch = arch.strip() + if not arch: + continue + if arch == "*": + return ["*"] + if arch in platforms_map: + result.append(platforms_map[arch]) + return result + + def _parse_authors(self, properties): + authors = [] + for author in properties["author"].split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + authors.append( + self._cleanup_author(dict(name=name, email=email, maintainer=False)) + ) + for author in properties.get("maintainer", "").split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + found = False + for item in authors: + if item["name"].lower() != name.lower(): + continue + found = True + item["maintainer"] = True + if not item["email"]: + item["email"] = email + if not found: + authors.append( + self._cleanup_author(dict(name=name, email=email, maintainer=True)) + ) + return authors + + def _parse_repository(self, properties): + if self.remote_url: + repo_parse = urlparse(self.remote_url) + repo_path_tokens = repo_parse.path[1:].split("/")[:-1] + if "github" in repo_parse.netloc: + return dict( + type="git", + url="%s://github.com/%s" + % (repo_parse.scheme, "/".join(repo_path_tokens[:2])), + ) + if "raw" in repo_path_tokens: + return dict( + type="git", + url="%s://%s/%s" + % ( + repo_parse.scheme, + repo_parse.netloc, + "/".join(repo_path_tokens[: repo_path_tokens.index("raw")]), + ), + ) + if properties.get("url", "").startswith("https://github.com"): + return dict(type="git", url=properties["url"]) + return None + + def _parse_export(self): + include = None + if self.remote_url: + repo_parse = urlparse(self.remote_url) + repo_path_tokens = repo_parse.path[1:].split("/")[:-1] + if "github" in repo_parse.netloc: + include = "/".join(repo_path_tokens[3:]) or None + elif "raw" in repo_path_tokens: + include = ( + "/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :]) + or None + ) + return { + "include": include, + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], + } diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index b2c1dc9c..8ca36add 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -13,32 +13,196 @@ # limitations under the License. import pytest -import requests + +from platformio.package.manifest import parser -def validate_response(r): - assert r.status_code == 200, r.url - assert int(r.headers["Content-Length"]) > 0, r.url - assert r.headers["Content-Type"] in ("application/gzip", "application/octet-stream") +def test_library_json_parser(): + contents = """ +{ + "name": "TestPackage", + "keywords": "kw1, KW2, kw3", + "platforms": ["atmelavr", "espressif"], + "url": "http://old.url.format", + "exclude": [".gitignore", "tests"], + "include": "mylib" +} +""" + mp = parser.LibraryJsonManifestParser(contents) + assert sorted(mp.as_dict().items()) == sorted( + { + "name": "TestPackage", + "platforms": ["atmelavr", "espressif8266"], + "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, + "keywords": ["kw1", "kw2", "kw3"], + "homepage": "http://old.url.format", + }.items() + ) -def test_packages(): - pkgs_manifest = requests.get( - "https://dl.bintray.com/platformio/dl-packages/manifest.json" - ).json() - assert isinstance(pkgs_manifest, dict) - items = [] - for _, variants in pkgs_manifest.items(): - for item in variants: - items.append(item) +def test_module_json_parser(): + contents = """ +{ + "author": "Name Surname ", + "description": "This is Yotta library", + "homepage": "https://yottabuild.org", + "keywords": [ + "mbed", + "Yotta" + ], + "licenses": [ + { + "type": "Apache-2.0", + "url": "https://spdx.org/licenses/Apache-2.0" + } + ], + "name": "YottaLibrary", + "repository": { + "type": "git", + "url": "git@github.com:username/repo.git" + }, + "version": "1.2.3" +} +""" + mp = parser.ModuleJsonManifestParser(contents) + assert sorted(mp.as_dict().items()) == sorted( + { + "name": "YottaLibrary", + "description": "This is Yotta library", + "homepage": "https://yottabuild.org", + "keywords": ["mbed", "Yotta"], + "license": "Apache-2.0", + "platforms": ["*"], + "frameworks": ["mbed"], + "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, + "authors": [ + { + "maintainer": False, + "email": "name@surname.com", + "name": "Name Surname", + } + ], + "version": "1.2.3", + }.items() + ) - for item in items: - assert item["url"].endswith(".tar.gz"), item - r = requests.head(item["url"], allow_redirects=True) - validate_response(r) +def test_library_properties_parser(): + # test missed fields + with pytest.raises(parser.ManifestParserException): + parser.LibraryPropertiesManifestParser("name=TestPackage") - if "X-Checksum-Sha1" not in r.headers: - return pytest.skip("X-Checksum-Sha1 is not provided") + # Base + contents = """ +name=TestPackage +version=1.2.3 +author=SomeAuthor +sentence=This is Arduino library +""" + mp = parser.LibraryPropertiesManifestParser(contents) + assert sorted(mp.as_dict().items()) == sorted( + { + "name": "TestPackage", + "version": "1.2.3", + "description": "This is Arduino library", + "repository": None, + "platforms": ["*"], + "frameworks": ["arduino"], + "export": { + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], + "include": None, + }, + "authors": [ + {"maintainer": False, "email": "info@author.com", "name": "SomeAuthor"} + ], + "keywords": ["uncategorized"], + "homepage": None, + }.items() + ) - assert item["sha1"] == r.headers.get("X-Checksum-Sha1")[0:40], item + # Platforms ALL + mp = parser.LibraryPropertiesManifestParser("architectures=*\n" + contents) + assert mp.as_dict()["platforms"] == ["*"] + # Platforms specific + mp = parser.LibraryPropertiesManifestParser("architectures=avr, esp32\n" + contents) + assert mp.as_dict()["platforms"] == ["atmelavr", "espressif32"] + + # Remote URL + mp = parser.LibraryPropertiesManifestParser( + contents, + remote_url=( + "https://raw.githubusercontent.com/username/reponame/master/" + "libraries/TestPackage/library.properties" + ), + ) + assert mp.as_dict()["export"] == { + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], + "include": "libraries/TestPackage", + } + + # Hope page + mp = parser.LibraryPropertiesManifestParser( + "url=https://github.com/username/reponame.git\n" + contents + ) + assert mp.as_dict()["homepage"] is None + assert mp.as_dict()["repository"] == { + "type": "git", + "url": "https://github.com/username/reponame.git", + } + + +def test_library_json_model(): + contents = """ +{ + "name": "ArduinoJson", + "keywords": "JSON, rest, http, web", + "description": "An elegant and efficient JSON library for embedded systems", + "homepage": "https://arduinojson.org", + "repository": { + "type": "git", + "url": "https://github.com/bblanchon/ArduinoJson.git" + }, + "version": "6.12.0", + "authors": { + "name": "Benoit Blanchon", + "url": "https://blog.benoitblanchon.fr" + }, + "exclude": [ + "fuzzing", + "scripts", + "test", + "third-party" + ], + "frameworks": "arduino", + "platforms": "*", + "license": "MIT" +} +""" + model = parser.ManifestFactory.new(contents, parser.ManifestFileType.LIBRARY_JSON) + assert sorted(model.as_dict().items()) == sorted( + { + "name": "ArduinoJson", + "keywords": ["json", "rest", "http", "web"], + "description": "An elegant and efficient JSON library for embedded systems", + "homepage": "https://arduinojson.org", + "repository": { + "url": "https://github.com/bblanchon/ArduinoJson.git", + "type": "git", + "branch": None, + }, + "version": "6.12.0", + "authors": { + "url": "https://blog.benoitblanchon.fr", + "maintainer": False, + "email": None, + "name": "Benoit Blanchon", + }, + "export": { + "include": None, + "exclude": ["fuzzing", "scripts", "test", "third-party"], + }, + "frameworks": ["arduino"], + "platforms": ["*"], + "license": "MIT", + }.items() + )