From df6a8da2908be2a49394fc4f802d6ce837633003 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Oct 2019 16:13:25 +0300 Subject: [PATCH] DataModel: add support for silent validation and "get_exceptions" API --- platformio/datamodel.py | 76 +++++++++++++++++++++------ platformio/package/manifest/model.py | 17 +++--- platformio/package/manifest/parser.py | 17 ++++-- tests/test_pkgmanifest.py | 71 ++++++++++++++++--------- 4 files changed, 128 insertions(+), 53 deletions(-) diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 0231913b..5390484c 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -26,6 +26,20 @@ class DataModelException(PlatformioException): pass +class DataFieldException(DataModelException): + def __init__(self, field, message): + self.field = field + self.message = message + super(DataFieldException, self).__init__() + + def __str__(self): + return "%s for `%s.%s` field" % ( + self.message, + self.field.parent.__class__.__name__, + self.field.name, + ) + + class ListOfType(object): def __init__(self, type): self.type = type @@ -52,19 +66,19 @@ class DataField(object): self.validate_factory = validate_factory self.title = title - self._parent = None - self._name = None - self._value = None + self.parent = None + self.name = None + self.value = None def __repr__(self): return '' % ( self.title, - self.default if self._value is None else self._value, + self.default if self.value is None else self.value, ) def validate(self, parent, name, value): - self._parent = parent - self._name = name + self.parent = parent + self.name = name self.title = self.title or name.title() try: @@ -81,16 +95,14 @@ class DataField(object): if issubclass(self.type, (str, bool)): return getattr(self, "_validate_%s_value" % self.type.__name__)(value) except ValueError as e: - raise DataModelException( - "%s for `%s.%s` field" % (str(e), parent.__class__.__name__, name) - ) + raise DataFieldException(self, str(e)) return value def _validate_list_of_type(self, list_of_type, value): if not isinstance(value, list): raise ValueError("Value should be a list") if isinstance(list_of_type, DataField): - return [list_of_type.validate(self._parent, self._name, v) for v in value] + return [list_of_type.validate(self.parent, self.name, v) for v in value] assert issubclass(list_of_type, DataModel) return [list_of_type(**v).as_dict() for v in value] @@ -119,19 +131,53 @@ class DataField(object): class DataModel(object): + + _field_names = None + _exceptions = None + def __init__(self, **kwargs): - self._known_attributes = [] + self._field_names = [] + self._exceptions = [] for name, field in get_class_attributes(self).items(): if not isinstance(field, DataField): continue - self._known_attributes.append(name) - setattr(self, name, field.validate(self, name, kwargs.get(name))) + self._field_names.append(name) + value = None + try: + value = field.validate(self, name, kwargs.get(name)) + except DataFieldException as e: + self._exceptions.append(e) + if isinstance(self, StrictDataModel): + raise e + finally: + setattr(self, name, value) + + def __eq__(self, other): + assert isinstance(other, DataModel) + if self.get_field_names() != other.get_field_names(): + return False + if self.get_exceptions() != other.get_exceptions(): + return False + for name in self._field_names: + if getattr(self, name) != getattr(other, name): + return False + return True def __repr__(self): fields = [] - for name in self._known_attributes: + for name in self._field_names: fields.append('%s="%s"' % (name, getattr(self, name))) return "<%s %s>" % (self.__class__.__name__, " ".join(fields)) + def get_field_names(self): + return self._field_names + + def get_exceptions(self): + return self._exceptions + def as_dict(self): - return {name: getattr(self, name) for name in self._known_attributes} + return {name: getattr(self, name) for name in self._field_names} + + +class StrictDataModel(DataModel): + pass diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index b55795d3..c87f3417 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -14,7 +14,7 @@ import semantic_version -from platformio.datamodel import DataField, DataModel, ListOfType +from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel def validate_semver_field(_, value): @@ -48,12 +48,12 @@ class ManifestModel(DataModel): version = DataField( max_length=50, validate_factory=validate_semver_field, required=True ) - - description = DataField(max_length=1000) + description = DataField(max_length=1000, required=True) keywords = DataField( - type=ListOfType(DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")) + type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\- ]+$")), + required=True, ) - authors = DataField(type=ListOfType(AuthorModel)) + authors = DataField(type=ListOfType(AuthorModel), required=True) homepage = DataField(max_length=255) license = DataField(max_length=255) @@ -68,8 +68,5 @@ class ManifestModel(DataModel): export = DataField(type=ExportModel) -class StrictManifestModel(ManifestModel): - def __init__(self, *args, **kwargs): - for name in ("description", "keywords", "authors"): - getattr(self, name).required = True - super(StrictManifestModel, self).__init__(*args, **kwargs) +class StrictManifestModel(ManifestModel, StrictDataModel): + pass diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index ec2e713c..c2489a97 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -151,6 +151,8 @@ class LibraryJsonManifestParser(BaseManifestParser): data["authors"] = self._parse_authors(data["authors"]) if "platforms" in data: data["platforms"] = self._parse_platforms(data["platforms"]) or None + if "export" in data: + data["export"] = self._parse_export(data["export"]) return data @@ -180,9 +182,7 @@ class LibraryJsonManifestParser(BaseManifestParser): continue if "export" not in data: data["export"] = {} - data["export"][key] = ( - data[key] if isinstance(data[key], list) else [data[key]] - ) + data["export"][key] = data[key] del data[key] return data @@ -206,6 +206,17 @@ class LibraryJsonManifestParser(BaseManifestParser): result.append(item) return result + @staticmethod + def _parse_export(raw): + if not isinstance(raw, dict): + return None + result = {} + for k in ("include", "exclude"): + if k not in raw: + continue + result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]] + return result + class ModuleJsonManifestParser(BaseManifestParser): def parse(self, contents): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 9a71d384..6bfe93e0 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -14,7 +14,7 @@ import pytest -from platformio.datamodel import DataModelException +from platformio.datamodel import DataFieldException from platformio.package.manifest import parser from platformio.package.manifest.model import ManifestModel, StrictManifestModel @@ -41,6 +41,26 @@ def test_library_json_parser(): }.items() ) + contents = """ +{ + "keywords": ["sound", "audio", "music", "SD", "card", "playback"], + "frameworks": "arduino", + "platforms": "atmelavr", + "export": { + "exclude": "audio_samples" + } +} +""" + mp = parser.LibraryJsonManifestParser(contents) + assert sorted(mp.as_dict().items()) == sorted( + { + "keywords": ["sound", "audio", "music", "sd", "card", "playback"], + "frameworks": ["arduino"], + "export": {"exclude": ["audio_samples"]}, + "platforms": ["atmelavr"], + }.items() + ) + def test_module_json_parser(): contents = """ @@ -180,8 +200,8 @@ def test_library_json_valid_model(): contents, parser.ManifestFileType.LIBRARY_JSON ) model = ManifestModel(**data.as_dict()) - assert sorted(model.as_dict().items()) == sorted( - { + assert model == ManifestModel( + **{ "name": "ArduinoJson", "keywords": ["json", "rest", "http", "web"], "description": "An elegant and efficient JSON library for embedded systems", @@ -207,7 +227,7 @@ def test_library_json_valid_model(): "frameworks": ["arduino"], "platforms": ["*"], "license": "MIT", - }.items() + } ) @@ -227,8 +247,9 @@ architectures=avr,sam contents, parser.ManifestFileType.LIBRARY_PROPERTIES ) model = ManifestModel(**data.as_dict()) - assert sorted(model.as_dict().items()) == sorted( - { + assert not model.get_exceptions() + assert model == ManifestModel( + **{ "license": None, "description": ( "A library for monochrome TFTs and OLEDs. Supported display " @@ -257,30 +278,30 @@ architectures=avr,sam "keywords": ["display"], "homepage": None, "name": "U8glib", - }.items() + } ) def test_broken_model(): - # "version" field is required - with pytest.raises( - DataModelException, match="Missed value for `ManifestModel.version` field" - ): - assert ManifestModel(name="MyPackage") + # non-strict mode + assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4 + assert ManifestModel(name="MyPackage", version="broken_version").version is None + + # strict mode + + with pytest.raises(DataFieldException) as excinfo: + assert StrictManifestModel(name="MyPackage") + assert excinfo.match(r"Missed value for `StrictManifestModel.[a-z]+` field") # broken SemVer with pytest.raises( - DataModelException, - match="Invalid semantic versioning format for `ManifestModel.version` field", + DataFieldException, + match="Invalid semantic versioning format for `StrictManifestModel.version` field", ): - assert ManifestModel(name="MyPackage", version="broken_version") - - # the only name and version fields are required for base ManifestModel - assert ManifestModel(name="MyPackage", version="1.0") - - # check strict model - with pytest.raises( - DataModelException, - match="Missed value for `StrictManifestModel.description` field", - ): - assert StrictManifestModel(name="MyPackage", version="1.0") + assert StrictManifestModel( + name="MyPackage", + description="MyDescription", + keywords=["a", "b"], + authors=[{"name": "Author"}], + version="broken_version", + )