From 27fc19d6b39f0c9fd40fee802de9b80768fb4ad2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 00:17:16 +0300 Subject: [PATCH] Switch to Marshmallow ODM framework --- docs | 2 +- platformio/commands/lib.py | 8 +- platformio/datamodel.py | 303 -------------------------- platformio/package/exception.py | 36 +++ platformio/package/manifest/model.py | 92 -------- platformio/package/manifest/parser.py | 51 ++--- platformio/package/manifest/schema.py | 181 +++++++++++++++ setup.py | 1 + tests/test_pkgmanifest.py | 268 +++++++++-------------- tox.ini | 1 + 10 files changed, 358 insertions(+), 585 deletions(-) delete mode 100644 platformio/datamodel.py create mode 100644 platformio/package/exception.py delete mode 100644 platformio/package/manifest/model.py create mode 100644 platformio/package/manifest/schema.py diff --git a/docs b/docs index 3c565d17..e8b93616 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3c565d175d608db0428aeafbca41c380876590a6 +Subproject commit e8b9361615d00180d9cb01af0b401885da091df8 diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 3f52cb8b..58cb85ed 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -25,8 +25,8 @@ from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib -from platformio.package.manifest.model import StrictManifestModel from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -495,7 +495,11 @@ def lib_register(config_url): raise exception.InvalidLibConfURL(config_url) # Validate manifest - StrictManifestModel(**ManifestParserFactory.new_from_url(config_url).as_dict()) + data, error = ManifestSchema(strict=False).load( + ManifestParserFactory.new_from_url(config_url).as_dict() + ) + if error: + raise ManifestValidationError(error, data) result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: diff --git a/platformio/datamodel.py b/platformio/datamodel.py deleted file mode 100644 index 600de738..00000000 --- a/platformio/datamodel.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import re - -from platformio.compat import get_class_attributes, string_types -from platformio.exception import PlatformioException - -# pylint: disable=too-many-instance-attributes -# pylint: disable=redefined-builtin, too-many-arguments - - -class DataModelException(PlatformioException): - pass - - -class DataFieldException(DataModelException): - def __init__(self, field, message): - self.field = field - self.message = message - super(DataFieldException, self).__init__() - - def __str__(self): - return "%s for `%s.%s` field" % ( - self.message, - self.field.parent.__class__.__name__, - self.field.name, - ) - - def __repr__(self): - return str(self) - - -class ListOfType(object): - def __init__(self, type): - self.type = type - - -class DictOfType(object): - def __init__(self, type): - self.type = type - - -class DataField(object): - def __init__( - self, - default=None, - type=str, - required=False, - min_length=None, - max_length=None, - regex=None, - validate_factory=None, - title=None, - ): - self.default = default - self.type = type - self.required = required - self.min_length = min_length - self.max_length = max_length - self.regex = regex - self.validate_factory = validate_factory - self.title = title - - self._parent = None - self._name = None - self.value = None - - def __repr__(self): - return '' % ( - self.title, - self.default if self.value is None else self.value, - ) - - @property - def parent(self): - return self._parent - - @parent.setter - def parent(self, value): - self._parent = value - - @property - def name(self): - return self._name - - @name.setter - def name(self, value): - self._name = value - self.title = self.title or value.title() - - def validate(self, value): - try: - if self.required and value is None: - raise ValueError("Missed value") - if self.validate_factory is not None: - return self.validate_factory(self, value) or self.default - if value is None: - return self.default - if inspect.isclass(self.type) and issubclass(self.type, DataModel): - return self.type(**self.ensure_value_is_dict(value)) - if inspect.isclass(self.type) and issubclass(self.type, (str, bool)): - return getattr(self, "_validate_%s_value" % self.type.__name__)(value) - except ValueError as e: - raise DataFieldException(self, str(e)) - return value - - @staticmethod - def ensure_value_is_dict(value): - if not isinstance(value, dict): - raise ValueError("Value should be type of dict, not `%s`" % type(value)) - return value - - def _validate_str_value(self, value): - if not isinstance(value, string_types): - value = str(value) - if self.min_length and len(value) < self.min_length: - raise ValueError( - "Minimum allowed length is %d characters" % self.min_length - ) - if self.max_length and len(value) > self.max_length: - raise ValueError( - "Maximum allowed length is %d characters" % self.max_length - ) - if self.regex and not re.match(self.regex, value): - raise ValueError( - "Value `%s` does not match RegExp `%s` pattern" % (value, self.regex) - ) - return value - - @staticmethod - def _validate_bool_value(value): - if isinstance(value, bool): - return value - return str(value).lower() in ("true", "yes", "1") - - -class DataModel(object): - - _field_names = None - _exceptions = None - - def __init__(self, **kwargs): - self._field_names = [] - self._exceptions = set() - for name, field in get_class_attributes(self).items(): - if not isinstance(field, DataField): - continue - field.parent = self - field.name = name - self._field_names.append(name) - - raw_value = kwargs.get(name) - value = None - try: - if isinstance(field.type, ListOfType): - value = self._validate_list_of_type(field, name, raw_value) - elif isinstance(field.type, DictOfType): - value = self._validate_dict_of_type(field, name, raw_value) - else: - value = field.validate(raw_value) - except DataFieldException as e: - self._exceptions.add(e) - if isinstance(self, StrictDataModel): - raise e - finally: - setattr(self, name, value) - - def _validate_list_of_type(self, field, name, value): - data_type = field.type.type - # check if ListOfType is not required - value = field.validate(value) - if not value: - return None - if not isinstance(value, list): - raise DataFieldException(field, "Value should be a list") - - if isinstance(data_type, DataField): - result = [] - data_type.parent = self - data_type.name = name - for v in value: - try: - result.append(data_type.validate(v)) - except DataFieldException as e: - self._exceptions.add(e) - if isinstance(self, StrictDataModel): - raise e - return result - - assert issubclass(data_type, DataModel) - - result = [] - for v in value: - try: - if not isinstance(v, dict): - raise DataFieldException( - field, "Value `%s` should be type of dictionary" % v - ) - m = data_type(**v) - me = m.get_exceptions() - if not me: - result.append(m) - else: - self._exceptions |= set(me) - except DataFieldException as e: - self._exceptions.add(e) - if isinstance(self, StrictDataModel): - raise e - return result - - def _validate_dict_of_type(self, field, _, value): - data_type = field.type.type - assert issubclass(data_type, DataModel) - - # check if DictOfType is not required - value = field.validate(value) - if not value: - return None - if not isinstance(value, dict): - raise DataFieldException( - field, "Value `%s` should be type of dictionary" % value - ) - result = {} - for k, v in value.items(): - try: - if not isinstance(v, dict): - raise DataFieldException( - field, "Value `%s` should be type of dictionary" % v - ) - m = data_type(**v) - me = m.get_exceptions() - if not me: - result[k] = m - else: - self._exceptions |= set(me) - except DataFieldException as e: - self._exceptions.add(e) - if isinstance(self, StrictDataModel): - raise e - return result - - def __eq__(self, other): - assert isinstance(other, DataModel) - if self.get_field_names() != other.get_field_names(): - return False - return self.as_dict() == other.as_dict() - - def __repr__(self): - fields = [] - for name in self._field_names: - fields.append('%s="%s"' % (name, getattr(self, name))) - return "<%s %s>" % (self.__class__.__name__, " ".join(fields)) - - def get_field_names(self): - return self._field_names - - def get_exceptions(self): - result = list(self._exceptions) - for name in self._field_names: - value = getattr(self, name) - if isinstance(value, DataModel): - result.extend(value.get_exceptions()) - continue - if not isinstance(value, (dict, list)): - continue - for v in value.values() if isinstance(value, dict) else value: - if not isinstance(v, DataModel): - continue - result.extend(v.get_exceptions()) - return result - - def as_dict(self): - result = {} - for name in self._field_names: - value = getattr(self, name) - if isinstance(value, DataModel): - value = getattr(self, name).as_dict() - if isinstance(value, dict): - for k, v in value.items(): - if not isinstance(v, DataModel): - continue - value[k] = v.as_dict() - elif isinstance(value, list): - value = [v.as_dict() if isinstance(v, DataModel) else v for v in value] - result[name] = value - return result - - -class StrictDataModel(DataModel): - pass diff --git a/platformio/package/exception.py b/platformio/package/exception.py new file mode 100644 index 00000000..81753ad1 --- /dev/null +++ b/platformio/package/exception.py @@ -0,0 +1,36 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException + + +class ManifestException(PlatformioException): + pass + + +class ManifestParserError(ManifestException): + pass + + +class ManifestValidationError(ManifestException): + def __init__(self, error, data): + super(ManifestValidationError, self).__init__() + self.error = error + self.data = data + + def __str__(self): + return ( + "Invalid manifest fields: %s. \nPlease check specification -> " + "http://docs.platformio.org/page/librarymanager/config.html" % self.error + ) diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py deleted file mode 100644 index d5428ec7..00000000 --- a/platformio/package/manifest/model.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import semantic_version - -from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel - - -def validate_semver_field(_, value): - value = str(value) - if "." not in value: - raise ValueError("Invalid semantic versioning format") - return value if semantic_version.Version.coerce(value) else None - - -class AuthorModel(DataModel): - name = DataField(max_length=50, required=True) - email = DataField(max_length=50) - maintainer = DataField(default=False, type=bool) - url = DataField(max_length=255) - - -class StrictAuthorModel(AuthorModel, StrictDataModel): - pass - - -class RepositoryModel(DataModel): - type = DataField(max_length=3, required=True) - url = DataField(max_length=255, required=True) - branch = DataField(max_length=50) - - -class ExportModel(DataModel): - include = DataField(type=ListOfType(DataField())) - exclude = DataField(type=ListOfType(DataField())) - - -class ExampleModel(DataModel): - name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/]+$", required=True) - base = DataField(required=True) - files = DataField(type=ListOfType(DataField()), required=True) - - -class ManifestModel(DataModel): - - # Required fields - name = DataField(max_length=100, required=True) - version = DataField( - max_length=50, validate_factory=validate_semver_field, required=True - ) - description = DataField(max_length=1000, required=True) - keywords = DataField( - type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\-\+\. ]+$")), - required=True, - ) - authors = DataField(type=ListOfType(AuthorModel), required=True) - - homepage = DataField(max_length=255) - license = DataField(max_length=255) - platforms = DataField( - type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$")) - ) - frameworks = DataField( - type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$")) - ) - - repository = DataField(type=RepositoryModel) - export = DataField(type=ExportModel) - examples = DataField(type=ListOfType(ExampleModel)) - - # platform.json specific - title = DataField(max_length=100) - - # package.json specific - system = DataField( - type=ListOfType(DataField(max_length=50, regex=r"^[a-z\d\-_]+$")) - ) - - -class StrictManifestModel(ManifestModel, StrictDataModel): - authors = DataField(type=ListOfType(StrictAuthorModel), required=True) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 7970fecd..aa0e1c7e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -19,8 +19,8 @@ import re import requests from platformio.compat import get_class_attributes, string_types -from platformio.exception import PlatformioException from platformio.fs import get_file_contents +from platformio.package.exception import ManifestParserError from platformio.project.helpers import is_platformio_project try: @@ -29,14 +29,6 @@ except ImportError: from urlparse import urlparse -class ManifestException(PlatformioException): - pass - - -class ManifestParserException(ManifestException): - pass - - class ManifestFileType(object): PLATFORM_JSON = "platform.json" LIBRARY_JSON = "library.json" @@ -67,11 +59,11 @@ class ManifestParserFactory(object): @staticmethod def new_from_file(path, remote_url=False): if not path or not os.path.isfile(path): - raise ManifestException("Manifest file does not exist %s" % path) + raise ManifestParserError("Manifest file does not exist %s" % path) for t in get_class_attributes(ManifestFileType).values(): if path.endswith(t): return ManifestParserFactory.new(get_file_contents(path), t, remote_url) - raise ManifestException("Unknown manifest file type %s" % path) + raise ManifestParserError("Unknown manifest file type %s" % path) @staticmethod def new_from_dir(path, remote_url=None): @@ -102,7 +94,7 @@ class ManifestParserFactory(object): remote_url=remote_url, package_dir=path, ) - raise ManifestException("Unknown manifest file type in %s directory" % path) + raise ManifestParserError("Unknown manifest file type in %s directory" % path) @staticmethod def new_from_url(remote_url): @@ -119,7 +111,7 @@ class ManifestParserFactory(object): # pylint: disable=redefined-builtin clsname = ManifestParserFactory.type_to_clsname(type) if clsname not in globals(): - raise ManifestException("Unknown manifest file type %s" % clsname) + raise ManifestParserError("Unknown manifest file type %s" % clsname) return globals()[clsname](contents, remote_url, package_dir) @@ -130,9 +122,14 @@ class BaseManifestParser(object): try: self._data = self.parse(contents) except Exception as e: - raise ManifestParserException("Could not parse manifest -> %s" % e) + raise ManifestParserError("Could not parse manifest -> %s" % e) self._data = self.parse_examples(self._data) + # remove None fields + for key in list(self._data.keys()): + if self._data[key] is None: + del self._data[key] + def parse(self, contents): raise NotImplementedError @@ -141,8 +138,12 @@ class BaseManifestParser(object): @staticmethod def cleanup_author(author): + assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) + for key in list(author.keys()): + if author[key] is None: + del author[key] return author @staticmethod @@ -351,9 +352,7 @@ class ModuleJsonManifestParser(BaseManifestParser): name, email = self.parse_author_name_and_email(author) if not name: continue - result.append( - self.cleanup_author(dict(name=name, email=email, maintainer=False)) - ) + result.append(self.cleanup_author(dict(name=name, email=email))) return result @staticmethod @@ -431,7 +430,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): } for arch in properties.get("architectures", "").split(","): if "particle-" in arch: - raise ManifestParserException("Particle is not supported yet") + raise ManifestParserError("Particle is not supported yet") arch = arch.strip() if not arch: continue @@ -449,20 +448,18 @@ class LibraryPropertiesManifestParser(BaseManifestParser): name, email = self.parse_author_name_and_email(author) if not name: continue - authors.append( - self.cleanup_author(dict(name=name, email=email, maintainer=False)) - ) + authors.append(self.cleanup_author(dict(name=name, email=email))) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) if not name: continue found = False for item in authors: - if item["name"].lower() != name.lower(): + if item.get("name", "").lower() != name.lower(): continue found = True item["maintainer"] = True - if not item["email"]: + if not item.get("email"): item["email"] = email if not found: authors.append( @@ -495,6 +492,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): return None def _parse_export(self): + result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]} include = None if self.remote_url: repo_parse = urlparse(self.remote_url) @@ -506,10 +504,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser): "/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :]) or None ) - return { - "include": include, - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - } + if include: + result["include"] = include + return result class PlatformJsonManifestParser(BaseManifestParser): diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py new file mode 100644 index 00000000..75471323 --- /dev/null +++ b/platformio/package/manifest/schema.py @@ -0,0 +1,181 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests +import semantic_version +from marshmallow import Schema, ValidationError, fields, validate, validates + +from platformio.package.exception import ManifestValidationError +from platformio.util import memoized + + +class StrictSchema(Schema): + def handle_error(self, error, data): + # skip broken records + if self.many: + error.data = [ + item for idx, item in enumerate(data) if idx not in error.messages + ] + else: + error.data = None + raise error + + +class StrictListField(fields.List): + def _deserialize(self, value, attr, data): + try: + return super(StrictListField, self)._deserialize(value, attr, data) + except ValidationError as exc: + exc.data = [item for item in exc.data if item is not None] + raise exc + + +class AuthorSchema(StrictSchema): + name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + email = fields.Email(validate=validate.Length(min=1, max=50)) + maintainer = fields.Bool(default=False) + url = fields.Url(validate=validate.Length(min=1, max=255)) + + +class RepositorySchema(StrictSchema): + type = fields.Str( + required=True, + validate=validate.OneOf( + ["git", "hg", "svn"], + error="Invalid repository type, please use one of [git, hg, svn]", + ), + ) + url = fields.Str(required=True, validate=validate.Length(min=1, max=255)) + branch = fields.Str(validate=validate.Length(min=1, max=50)) + + +class ExportSchema(Schema): + include = StrictListField(fields.Str) + exclude = StrictListField(fields.Str) + + +class ExampleSchema(StrictSchema): + name = fields.Str( + required=True, + validate=[ + validate.Length(min=1, max=100), + validate.Regexp( + r"^[a-zA-Z\d\-\_/]+$", error="Only [a-zA-Z0-9-_/] chars are allowed" + ), + ], + ) + base = fields.Str(required=True) + files = StrictListField(fields.Str, required=True) + + +class ManifestSchema(Schema): + # Required fields + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + + # Optional fields + + authors = fields.Nested(AuthorSchema, many=True) + description = fields.Str(validate=validate.Length(min=1, max=1000)) + homepage = fields.Url(validate=validate.Length(min=1, max=255)) + license = fields.Str(validate=validate.Length(min=1, max=255)) + repository = fields.Nested(RepositorySchema) + export = fields.Nested(ExportSchema) + examples = fields.Nested(ExampleSchema, many=True) + + keywords = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed" + ), + ] + ) + ) + + platforms = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + frameworks = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + + # platform.json specific + title = fields.Str(validate=validate.Length(min=1, max=100)) + + # package.json specific + system = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed" + ), + ] + ) + ) + + def handle_error(self, error, data): + if self.strict: + raise ManifestValidationError(error, data) + + @validates("version") + def validate_version(self, value): # pylint: disable=no-self-use + try: + value = str(value) + assert "." in value + semantic_version.Version.coerce(value) + except (AssertionError, ValueError): + raise ValidationError( + "Invalid semantic versioning format, see https://semver.org/" + ) + + @validates("license") + def validate_license(self, value): + try: + spdx = self.load_spdx_licenses() + except requests.exceptions.RequestException: + raise ValidationError("Could not load SPDX licenses for validation") + for item in spdx.get("licenses", []): + if item.get("licenseId") == value: + return + raise ValidationError( + "Invalid SPDX license identifier. See valid identifiers at " + "https://spdx.org/licenses/" + ) + + @staticmethod + @memoized(expire="1h") + def load_spdx_licenses(): + r = requests.get( + "https://raw.githubusercontent.com/spdx/license-list-data" + "/v3.6/json/licenses.json" + ) + r.raise_for_status() + return r.json() diff --git a/setup.py b/setup.py index 60b0a115..64a83fbf 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ install_requires = [ "semantic_version>=2.8.1,<3", "tabulate>=0.8.3,<1", "pyelftools>=0.25,<1", + "marshmallow>=2.20.5,<3" ] setup( diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index db531b93..95a9af43 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import jsondiff import pytest -from platformio import datamodel from platformio.compat import WINDOWS -from platformio.package.manifest import model, parser +from platformio.package.manifest import parser +from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError def test_library_json_parser(): @@ -31,14 +32,15 @@ def test_library_json_parser(): } """ mp = parser.LibraryJsonManifestParser(contents) - assert sorted(mp.as_dict().items()) == sorted( + assert not jsondiff.diff( + mp.as_dict(), { "name": "TestPackage", "platforms": ["atmelavr", "espressif8266"], "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, "keywords": ["kw1", "kw2", "kw3"], "homepage": "http://old.url.format", - }.items() + }, ) contents = """ @@ -52,13 +54,14 @@ def test_library_json_parser(): } """ mp = parser.LibraryJsonManifestParser(contents) - assert sorted(mp.as_dict().items()) == sorted( + assert not jsondiff.diff( + mp.as_dict(), { "keywords": ["sound", "audio", "music", "sd", "card", "playback"], "frameworks": ["arduino"], "export": {"exclude": ["audio_samples"]}, "platforms": ["atmelavr"], - }.items() + }, ) @@ -87,7 +90,8 @@ def test_module_json_parser(): } """ mp = parser.ModuleJsonManifestParser(contents) - assert sorted(mp.as_dict().items()) == sorted( + assert not jsondiff.diff( + mp.as_dict(), { "name": "YottaLibrary", "description": "This is Yotta library", @@ -97,15 +101,9 @@ def test_module_json_parser(): "platforms": ["*"], "frameworks": ["mbed"], "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, - "authors": [ - { - "maintainer": False, - "email": "name@surname.com", - "name": "Name Surname", - } - ], + "authors": [{"email": "name@surname.com", "name": "Name Surname"}], "version": "1.2.3", - }.items() + }, ) @@ -118,24 +116,20 @@ author=SomeAuthor sentence=This is Arduino library """ mp = parser.LibraryPropertiesManifestParser(contents) - assert sorted(mp.as_dict().items()) == sorted( + assert not jsondiff.diff( + mp.as_dict(), { "name": "TestPackage", "version": "1.2.3", "description": "This is Arduino library", - "repository": None, "platforms": ["*"], "frameworks": ["arduino"], "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - "include": None, + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, - "authors": [ - {"maintainer": False, "email": "info@author.com", "name": "SomeAuthor"} - ], + "authors": [{"email": "info@author.com", "name": "SomeAuthor"}], "keywords": ["uncategorized"], - "homepage": None, - }.items() + }, ) # Platforms ALL @@ -170,7 +164,6 @@ sentence=This is Arduino library data = parser.LibraryPropertiesManifestParser( "url=https://github.com/username/reponame.git\n" + contents ).as_dict() - assert data["homepage"] is None assert data["repository"] == { "type": "git", "url": "https://github.com/username/reponame.git", @@ -216,15 +209,20 @@ def test_library_json_model(): ] } """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_JSON ).as_dict() - m = model.StrictManifestModel(**data) - assert m.repository.url == "https://github.com/bblanchon/ArduinoJson.git" - assert m.examples[1].base == "examples/JsonHttpClient" - assert m.examples[1].files == ["JsonHttpClient.ino"] - assert m == model.StrictManifestModel( - **{ + + data, errors = ManifestSchema(strict=True).load(raw_data) + assert not errors + + assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git" + assert data["examples"][1]["base"] == "examples/JsonHttpClient" + assert data["examples"][1]["files"] == ["JsonHttpClient.ino"] + + assert not jsondiff.diff( + data, + { "name": "ArduinoJson", "keywords": ["json", "rest", "http", "web"], "description": "An elegant and efficient JSON library for embedded systems", @@ -232,21 +230,12 @@ def test_library_json_model(): "repository": { "url": "https://github.com/bblanchon/ArduinoJson.git", "type": "git", - "branch": None, }, "version": "6.12.0", "authors": [ - { - "name": "Benoit Blanchon", - "url": "https://blog.benoitblanchon.fr", - "maintainer": False, - "email": None, - } + {"name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr"} ], - "export": { - "exclude": ["fuzzing", "scripts", "test", "third-party"], - "include": None, - }, + "export": {"exclude": ["fuzzing", "scripts", "test", "third-party"]}, "frameworks": ["arduino"], "platforms": ["*"], "license": "MIT", @@ -262,7 +251,7 @@ def test_library_json_model(): "files": ["JsonHttpClient.ino"], }, ], - } + }, ) @@ -278,42 +267,33 @@ category=Display url=https://github.com/olikraus/u8glib architectures=avr,sam """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES ).as_dict() - m = model.StrictManifestModel(**data) - assert not m.get_exceptions() - assert m == model.StrictManifestModel( - **{ - "license": None, + + data, errors = ManifestSchema(strict=True).load(raw_data) + assert not errors + + assert not jsondiff.diff( + data, + { "description": ( "A library for monochrome TFTs and OLEDs. Supported display " "controller: SSD1306, SSD1309, SSD1322, SSD1325" ), - "repository": { - "url": "https://github.com/olikraus/u8glib", - "type": "git", - "branch": None, - }, + "repository": {"url": "https://github.com/olikraus/u8glib", "type": "git"}, "frameworks": ["arduino"], "platforms": ["atmelavr", "atmelsam"], "version": "1.19.1", "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - "include": None, + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, "authors": [ - { - "url": None, - "maintainer": True, - "email": "olikraus@gmail.com", - "name": "oliver", - } + {"maintainer": True, "email": "olikraus@gmail.com", "name": "oliver"} ], "keywords": ["display"], - "homepage": None, "name": "U8glib", - } + }, ) # Broken fields @@ -330,7 +310,7 @@ architectures=* dot_a_linkage=false includes=MozziGuts.h """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES, remote_url=( @@ -338,10 +318,13 @@ includes=MozziGuts.h "master/library.properties" ), ).as_dict() - m = model.ManifestModel(**data) - assert m.get_exceptions() - assert m == model.ManifestModel( - **{ + + data, errors = ManifestSchema(strict=False).load(raw_data) + assert errors["authors"] + + assert not jsondiff.diff( + data, + { "name": "Mozzi", "version": "1.0.3", "description": ( @@ -353,8 +336,7 @@ includes=MozziGuts.h "platforms": ["*"], "frameworks": ["arduino"], "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - "include": None, + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, "authors": [ { @@ -365,7 +347,7 @@ includes=MozziGuts.h ], "keywords": ["signal", "input", "output"], "homepage": "https://sensorium.github.io/Mozzi/", - } + }, ) @@ -419,14 +401,16 @@ def test_platform_json_model(): } } """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PLATFORM_JSON ).as_dict() - data["frameworks"] = sorted(data["frameworks"]) - m = model.ManifestModel(**data) - assert m.frameworks == ["arduino", "simba"] - assert m == model.ManifestModel( - **{ + + data, errors = ManifestSchema(strict=False).load(raw_data) + assert not errors + + assert not jsondiff.diff( + data, + { "name": "atmelavr", "title": "Atmel AVR", "description": ( @@ -441,11 +425,10 @@ def test_platform_json_model(): "repository": { "url": "https://github.com/platformio/platform-atmelavr.git", "type": "git", - "branch": None, }, "frameworks": ["arduino", "simba"], "version": "1.15.0", - } + }, ) @@ -458,19 +441,21 @@ def test_package_json_model(): "version": "3.30101.0" } """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PACKAGE_JSON ).as_dict() - m = model.ManifestModel(**data) - assert m.system is None - assert m.homepage == "http://www.scons.org" - assert m == model.ManifestModel( - **{ + + data, errors = ManifestSchema(strict=False).load(raw_data) + assert not errors + + assert not jsondiff.diff( + data, + { "name": "tool-scons", "description": "SCons software construction tool", "homepage": "http://www.scons.org", "version": "3.30101.0", - } + }, ) mp = parser.ManifestParserFactory.new( @@ -543,20 +528,23 @@ def test_examples_from_dir(tmpdir_factory): # Do testing - data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() - assert isinstance(data["examples"], list) - assert len(data["examples"]) == 6 + raw_data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() + assert isinstance(raw_data["examples"], list) + assert len(raw_data["examples"]) == 6 def _sort_examples(items): for i, item in enumerate(items): items[i]["files"] = sorted(item["files"]) return sorted(items, key=lambda item: item["name"]) - data["examples"] = _sort_examples(data["examples"]) - m = model.ManifestModel(**data) - assert m.examples[3].name == "PlatformIO/hello" - assert m == model.ManifestModel( - **{ + raw_data["examples"] = _sort_examples(raw_data["examples"]) + + data, errors = ManifestSchema(strict=True).load(raw_data) + assert not errors + + assert not jsondiff.diff( + data, + { "version": "1.0.0", "name": "pkg", "examples": _sort_examples( @@ -599,84 +587,44 @@ def test_examples_from_dir(tmpdir_factory): }, ] ), - } + }, ) -def test_dict_of_type(): - class TestModel(datamodel.DataModel): - examples = datamodel.DataField(type=datamodel.DictOfType(model.ExampleModel)) - - class StrictTestModel(TestModel, datamodel.StrictDataModel): - pass - - # valid - m = TestModel( - examples={ - "valid": dict(name="Valid", base="valid", files=["valid.h"]), - "invalid": "test", - } - ) - assert list(m.examples.keys()) == ["valid"] - - # invalid - with pytest.raises(datamodel.DataFieldException): - StrictTestModel(examples=[dict(name="Valid", base="valid", files=["valid.h"])]) - - with pytest.raises(datamodel.DataFieldException): - StrictTestModel( - examples={ - "valid": dict(name="Valid", base="valid", files=["valid.h"]), - "invalid": "test", - } - ) - - def test_broken_models(): # non-strict mode - assert len(model.ManifestModel(name="MyPackage").get_exceptions()) == 4 - assert ( - model.ManifestModel(name="MyPackage", version="broken_version").version is None - ) + data, errors = ManifestSchema(strict=False).load(dict(name="MyPackage")) + assert set(errors.keys()) == set(["version"]) + assert data.get("version") is None # invalid keywords - m = model.ManifestModel(keywords=["kw1", "*^[]"]) - assert any( - "Value `*^[]` does not match RegExp" in str(e) for e in m.get_exceptions() - ) - assert m.keywords == ["kw1"] + data, errors = ManifestSchema(strict=False).load(dict(keywords=["kw1", "*^[]"])) + assert errors + assert data["keywords"] == ["kw1"] # strict mode - with pytest.raises(datamodel.DataFieldException) as excinfo: - assert model.StrictManifestModel(name="MyPackage") - assert excinfo.match(r"Missed value for `StrictManifestModel.[a-z]+` field") + with pytest.raises( + ManifestValidationError, match="Missing data for required field" + ): + ManifestSchema(strict=True).load(dict(name="MyPackage")) # broken SemVer with pytest.raises( - datamodel.DataFieldException, - match=( - "Invalid semantic versioning format for " - "`StrictManifestModel.version` field" - ), + ManifestValidationError, match=("Invalid semantic versioning format") ): - assert model.StrictManifestModel( - name="MyPackage", - description="MyDescription", - keywords=["a", "b"], - authors=[{"name": "Author"}], - version="broken_version", + ManifestSchema(strict=True).load( + dict(name="MyPackage", version="broken_version") ) - # broken value for DataModel - with pytest.raises( - datamodel.DataFieldException, - match=("Value `should be dict here` should be type of dictionary"), - ): - assert model.StrictManifestModel( - name="MyPackage", - description="MyDescription", - keywords=["a", "b"], - authors=["should be dict here"], - version="1.2.3", + # broken value for Nested + with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"): + ManifestSchema(strict=True).load( + dict( + name="MyPackage", + description="MyDescription", + keywords=["a", "b"], + authors=["should be dict here"], + version="1.2.3", + ) ) diff --git a/tox.ini b/tox.ini index 3512bfb6..29384ca2 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = pylint pytest pytest-xdist + jsondiff commands = {envpython} --version pylint --rcfile=./.pylintrc ./platformio