DataModel: add support for silent validation and "get_exceptions" API

platformio/datamodel.py

@@ -26,6 +26,20 @@ class DataModelException(PlatformioException):
     pass
 
 
+class DataFieldException(DataModelException):
+    def __init__(self, field, message):
+        self.field = field
+        self.message = message
+        super(DataFieldException, self).__init__()
+
+    def __str__(self):
+        return "%s for `%s.%s` field" % (
+            self.message,
+            self.field.parent.__class__.__name__,
+            self.field.name,
+        )
+
+
 class ListOfType(object):
     def __init__(self, type):
         self.type = type
@@ -52,19 +66,19 @@ class DataField(object):
         self.validate_factory = validate_factory
         self.title = title
 
-        self._parent = None
-        self._name = None
-        self._value = None
+        self.parent = None
+        self.name = None
+        self.value = None
 
     def __repr__(self):
         return '<DataField %s="%s">' % (
             self.title,
-            self.default if self._value is None else self._value,
+            self.default if self.value is None else self.value,
         )
 
     def validate(self, parent, name, value):
-        self._parent = parent
-        self._name = name
+        self.parent = parent
+        self.name = name
         self.title = self.title or name.title()
 
         try:
@@ -81,16 +95,14 @@ class DataField(object):
             if issubclass(self.type, (str, bool)):
                 return getattr(self, "_validate_%s_value" % self.type.__name__)(value)
         except ValueError as e:
-            raise DataModelException(
-                "%s for `%s.%s` field" % (str(e), parent.__class__.__name__, name)
-            )
+            raise DataFieldException(self, str(e))
         return value
 
     def _validate_list_of_type(self, list_of_type, value):
         if not isinstance(value, list):
             raise ValueError("Value should be a list")
         if isinstance(list_of_type, DataField):
-            return [list_of_type.validate(self._parent, self._name, v) for v in value]
+            return [list_of_type.validate(self.parent, self.name, v) for v in value]
         assert issubclass(list_of_type, DataModel)
         return [list_of_type(**v).as_dict() for v in value]
 
@@ -119,19 +131,53 @@ class DataField(object):
 
 
 class DataModel(object):
+
+    _field_names = None
+    _exceptions = None
+
     def __init__(self, **kwargs):
-        self._known_attributes = []
+        self._field_names = []
+        self._exceptions = []
         for name, field in get_class_attributes(self).items():
             if not isinstance(field, DataField):
                 continue
-            self._known_attributes.append(name)
-            setattr(self, name, field.validate(self, name, kwargs.get(name)))
+            self._field_names.append(name)
+            value = None
+            try:
+                value = field.validate(self, name, kwargs.get(name))
+            except DataFieldException as e:
+                self._exceptions.append(e)
+                if isinstance(self, StrictDataModel):
+                    raise e
+            finally:
+                setattr(self, name, value)
+
+    def __eq__(self, other):
+        assert isinstance(other, DataModel)
+        if self.get_field_names() != other.get_field_names():
+            return False
+        if self.get_exceptions() != other.get_exceptions():
+            return False
+        for name in self._field_names:
+            if getattr(self, name) != getattr(other, name):
+                return False
+        return True
 
     def __repr__(self):
         fields = []
-        for name in self._known_attributes:
+        for name in self._field_names:
             fields.append('%s="%s"' % (name, getattr(self, name)))
         return "<%s %s>" % (self.__class__.__name__, " ".join(fields))
 
+    def get_field_names(self):
+        return self._field_names
+
+    def get_exceptions(self):
+        return self._exceptions
+
     def as_dict(self):
-        return {name: getattr(self, name) for name in self._known_attributes}
+        return {name: getattr(self, name) for name in self._field_names}
+
+
+class StrictDataModel(DataModel):
+    pass

platformio/package/manifest/model.py

@@ -14,7 +14,7 @@
 
 import semantic_version
 
-from platformio.datamodel import DataField, DataModel, ListOfType
+from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel
 
 
 def validate_semver_field(_, value):
@@ -48,12 +48,12 @@ class ManifestModel(DataModel):
     version = DataField(
         max_length=50, validate_factory=validate_semver_field, required=True
     )
-
-    description = DataField(max_length=1000)
+    description = DataField(max_length=1000, required=True)
     keywords = DataField(
-        type=ListOfType(DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$"))
+        type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\- ]+$")),
+        required=True,
     )
-    authors = DataField(type=ListOfType(AuthorModel))
+    authors = DataField(type=ListOfType(AuthorModel), required=True)
 
     homepage = DataField(max_length=255)
     license = DataField(max_length=255)
@@ -68,8 +68,5 @@ class ManifestModel(DataModel):
     export = DataField(type=ExportModel)
 
 
-class StrictManifestModel(ManifestModel):
-    def __init__(self, *args, **kwargs):
-        for name in ("description", "keywords", "authors"):
-            getattr(self, name).required = True
-        super(StrictManifestModel, self).__init__(*args, **kwargs)
+class StrictManifestModel(ManifestModel, StrictDataModel):
+    pass

platformio/package/manifest/parser.py

@@ -151,6 +151,8 @@ class LibraryJsonManifestParser(BaseManifestParser):
             data["authors"] = self._parse_authors(data["authors"])
         if "platforms" in data:
             data["platforms"] = self._parse_platforms(data["platforms"]) or None
+        if "export" in data:
+            data["export"] = self._parse_export(data["export"])
 
         return data
 
@@ -180,9 +182,7 @@ class LibraryJsonManifestParser(BaseManifestParser):
                 continue
             if "export" not in data:
                 data["export"] = {}
-            data["export"][key] = (
-                data[key] if isinstance(data[key], list) else [data[key]]
-            )
+            data["export"][key] = data[key]
             del data[key]
 
         return data
@@ -206,6 +206,17 @@ class LibraryJsonManifestParser(BaseManifestParser):
             result.append(item)
         return result
 
+    @staticmethod
+    def _parse_export(raw):
+        if not isinstance(raw, dict):
+            return None
+        result = {}
+        for k in ("include", "exclude"):
+            if k not in raw:
+                continue
+            result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]]
+        return result
+
 
 class ModuleJsonManifestParser(BaseManifestParser):
     def parse(self, contents):