DataModel: add support for silent validation and "get_exceptions" API

platformio/datamodel.py

@@ -26,6 +26,20 @@ class DataModelException(PlatformioException):
     pass
 
 
+class DataFieldException(DataModelException):
+    def __init__(self, field, message):
+        self.field = field
+        self.message = message
+        super(DataFieldException, self).__init__()
+
+    def __str__(self):
+        return "%s for `%s.%s` field" % (
+            self.message,
+            self.field.parent.__class__.__name__,
+            self.field.name,
+        )
+
+
 class ListOfType(object):
     def __init__(self, type):
         self.type = type
@@ -52,19 +66,19 @@ class DataField(object):
         self.validate_factory = validate_factory
         self.title = title
 
-        self._parent = None
+        self.parent = None
-        self._name = None
+        self.name = None
-        self._value = None
+        self.value = None
 
     def __repr__(self):
         return '<DataField %s="%s">' % (
             self.title,
-            self.default if self._value is None else self._value,
+            self.default if self.value is None else self.value,
         )
 
     def validate(self, parent, name, value):
-        self._parent = parent
+        self.parent = parent
-        self._name = name
+        self.name = name
         self.title = self.title or name.title()
 
         try:
@@ -81,16 +95,14 @@ class DataField(object):
             if issubclass(self.type, (str, bool)):
                 return getattr(self, "_validate_%s_value" % self.type.__name__)(value)
         except ValueError as e:
-            raise DataModelException(
+            raise DataFieldException(self, str(e))
-                "%s for `%s.%s` field" % (str(e), parent.__class__.__name__, name)
-            )
         return value
 
     def _validate_list_of_type(self, list_of_type, value):
         if not isinstance(value, list):
             raise ValueError("Value should be a list")
         if isinstance(list_of_type, DataField):
-            return [list_of_type.validate(self._parent, self._name, v) for v in value]
+            return [list_of_type.validate(self.parent, self.name, v) for v in value]
         assert issubclass(list_of_type, DataModel)
         return [list_of_type(**v).as_dict() for v in value]
 
@@ -119,19 +131,53 @@ class DataField(object):
 
 
 class DataModel(object):
+
+    _field_names = None
+    _exceptions = None
+
     def __init__(self, **kwargs):
-        self._known_attributes = []
+        self._field_names = []
+        self._exceptions = []
         for name, field in get_class_attributes(self).items():
             if not isinstance(field, DataField):
                 continue
-            self._known_attributes.append(name)
+            self._field_names.append(name)
-            setattr(self, name, field.validate(self, name, kwargs.get(name)))
+            value = None
+            try:
+                value = field.validate(self, name, kwargs.get(name))
+            except DataFieldException as e:
+                self._exceptions.append(e)
+                if isinstance(self, StrictDataModel):
+                    raise e
+            finally:
+                setattr(self, name, value)
+
+    def __eq__(self, other):
+        assert isinstance(other, DataModel)
+        if self.get_field_names() != other.get_field_names():
+            return False
+        if self.get_exceptions() != other.get_exceptions():
+            return False
+        for name in self._field_names:
+            if getattr(self, name) != getattr(other, name):
+                return False
+        return True
 
     def __repr__(self):
         fields = []
-        for name in self._known_attributes:
+        for name in self._field_names:
             fields.append('%s="%s"' % (name, getattr(self, name)))
         return "<%s %s>" % (self.__class__.__name__, " ".join(fields))
 
+    def get_field_names(self):
+        return self._field_names
+
+    def get_exceptions(self):
+        return self._exceptions
+
     def as_dict(self):
-        return {name: getattr(self, name) for name in self._known_attributes}
+        return {name: getattr(self, name) for name in self._field_names}
+
+
+class StrictDataModel(DataModel):
+    pass

platformio/package/manifest/model.py

@@ -14,7 +14,7 @@
 
 import semantic_version
 
-from platformio.datamodel import DataField, DataModel, ListOfType
+from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel
 
 
 def validate_semver_field(_, value):
@@ -48,12 +48,12 @@ class ManifestModel(DataModel):
     version = DataField(
         max_length=50, validate_factory=validate_semver_field, required=True
     )
-
+    description = DataField(max_length=1000, required=True)
-    description = DataField(max_length=1000)
     keywords = DataField(
-        type=ListOfType(DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$"))
+        type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\- ]+$")),
+        required=True,
     )
-    authors = DataField(type=ListOfType(AuthorModel))
+    authors = DataField(type=ListOfType(AuthorModel), required=True)
 
     homepage = DataField(max_length=255)
     license = DataField(max_length=255)
@@ -68,8 +68,5 @@ class ManifestModel(DataModel):
     export = DataField(type=ExportModel)
 
 
-class StrictManifestModel(ManifestModel):
+class StrictManifestModel(ManifestModel, StrictDataModel):
-    def __init__(self, *args, **kwargs):
+    pass
-        for name in ("description", "keywords", "authors"):
-            getattr(self, name).required = True
-        super(StrictManifestModel, self).__init__(*args, **kwargs)

platformio/package/manifest/parser.py

@@ -151,6 +151,8 @@ class LibraryJsonManifestParser(BaseManifestParser):
             data["authors"] = self._parse_authors(data["authors"])
         if "platforms" in data:
             data["platforms"] = self._parse_platforms(data["platforms"]) or None
+        if "export" in data:
+            data["export"] = self._parse_export(data["export"])
 
         return data
 
@@ -180,9 +182,7 @@ class LibraryJsonManifestParser(BaseManifestParser):
                 continue
             if "export" not in data:
                 data["export"] = {}
-            data["export"][key] = (
+            data["export"][key] = data[key]
-                data[key] if isinstance(data[key], list) else [data[key]]
-            )
             del data[key]
 
         return data
@@ -206,6 +206,17 @@ class LibraryJsonManifestParser(BaseManifestParser):
             result.append(item)
         return result
 
+    @staticmethod
+    def _parse_export(raw):
+        if not isinstance(raw, dict):
+            return None
+        result = {}
+        for k in ("include", "exclude"):
+            if k not in raw:
+                continue
+            result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]]
+        return result
+
 
 class ModuleJsonManifestParser(BaseManifestParser):
     def parse(self, contents):

tests/test_pkgmanifest.py

@@ -14,7 +14,7 @@
 
 import pytest
 
-from platformio.datamodel import DataModelException
+from platformio.datamodel import DataFieldException
 from platformio.package.manifest import parser
 from platformio.package.manifest.model import ManifestModel, StrictManifestModel
 
@@ -41,6 +41,26 @@ def test_library_json_parser():
         }.items()
     )
 
+    contents = """
+{
+    "keywords": ["sound", "audio", "music", "SD", "card", "playback"],
+    "frameworks": "arduino",
+    "platforms": "atmelavr",
+    "export": {
+        "exclude": "audio_samples"
+    }
+}
+"""
+    mp = parser.LibraryJsonManifestParser(contents)
+    assert sorted(mp.as_dict().items()) == sorted(
+        {
+            "keywords": ["sound", "audio", "music", "sd", "card", "playback"],
+            "frameworks": ["arduino"],
+            "export": {"exclude": ["audio_samples"]},
+            "platforms": ["atmelavr"],
+        }.items()
+    )
+
 
 def test_module_json_parser():
     contents = """
@@ -180,8 +200,8 @@ def test_library_json_valid_model():
         contents, parser.ManifestFileType.LIBRARY_JSON
     )
     model = ManifestModel(**data.as_dict())
-    assert sorted(model.as_dict().items()) == sorted(
+    assert model == ManifestModel(
-        {
+        **{
             "name": "ArduinoJson",
             "keywords": ["json", "rest", "http", "web"],
             "description": "An elegant and efficient JSON library for embedded systems",
@@ -207,7 +227,7 @@ def test_library_json_valid_model():
             "frameworks": ["arduino"],
             "platforms": ["*"],
             "license": "MIT",
-        }.items()
+        }
     )
 
 
@@ -227,8 +247,9 @@ architectures=avr,sam
         contents, parser.ManifestFileType.LIBRARY_PROPERTIES
     )
     model = ManifestModel(**data.as_dict())
-    assert sorted(model.as_dict().items()) == sorted(
+    assert not model.get_exceptions()
-        {
+    assert model == ManifestModel(
+        **{
             "license": None,
             "description": (
                 "A library for monochrome TFTs and OLEDs. Supported display "
@@ -257,30 +278,30 @@ architectures=avr,sam
             "keywords": ["display"],
             "homepage": None,
             "name": "U8glib",
-        }.items()
+        }
     )
 
 
 def test_broken_model():
-    # "version" field is required
+    # non-strict mode
-    with pytest.raises(
+    assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4
-        DataModelException, match="Missed value for `ManifestModel.version` field"
+    assert ManifestModel(name="MyPackage", version="broken_version").version is None
-    ):
+
-        assert ManifestModel(name="MyPackage")
+    # strict mode
+
+    with pytest.raises(DataFieldException) as excinfo:
+        assert StrictManifestModel(name="MyPackage")
+    assert excinfo.match(r"Missed value for `StrictManifestModel.[a-z]+` field")
 
     # broken SemVer
     with pytest.raises(
-        DataModelException,
+        DataFieldException,
-        match="Invalid semantic versioning format for `ManifestModel.version` field",
+        match="Invalid semantic versioning format for `StrictManifestModel.version` field",
     ):
-        assert ManifestModel(name="MyPackage", version="broken_version")
+        assert StrictManifestModel(
-
+            name="MyPackage",
-    # the only name and version fields are required for base ManifestModel
+            description="MyDescription",
-    assert ManifestModel(name="MyPackage", version="1.0")
+            keywords=["a", "b"],
-
+            authors=[{"name": "Author"}],
-    # check strict model
+            version="broken_version",
-    with pytest.raises(
+        )
-        DataModelException,
-        match="Missed value for `StrictManifestModel.description` field",
-    ):
-        assert StrictManifestModel(name="MyPackage", version="1.0")