Switch to Marshmallow ODM framework

This commit is contained in:
Ivan Kravets committed 2019-10-17 00:17:16 +03:00
1 parent 9cfccc5cd4
commit 27fc19d6b3
10 files changed
+358 -585

No files matched your search

+6 -2
View File
@@ -25,8 +25,8 @@ from platformio import exception, util
from platformio.commands import PlatformioCLI
from platformio.compat import dump_json_to_unicode
from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
from platformio.package.manifest.model import StrictManifestModel
from platformio.package.manifest.parser import ManifestParserFactory
from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError
from platformio.proc import is_ci
from platformio.project.config import ProjectConfig
from platformio.project.helpers import get_project_dir, is_platformio_project
@@ -495,7 +495,11 @@ def lib_register(config_url):
raise exception.InvalidLibConfURL(config_url)
# Validate manifest
StrictManifestModel(**ManifestParserFactory.new_from_url(config_url).as_dict())
data, error = ManifestSchema(strict=False).load(
ManifestParserFactory.new_from_url(config_url).as_dict()
)
if error:
raise ManifestValidationError(error, data)
result = util.get_api_result("/lib/register", data=dict(config_url=config_url))
if "message" in result and result["message"]:
-303
View File
@@ -1,303 +0,0 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
import re
from platformio.compat import get_class_attributes, string_types
from platformio.exception import PlatformioException
# pylint: disable=too-many-instance-attributes
# pylint: disable=redefined-builtin, too-many-arguments
class DataModelException(PlatformioException):
pass
class DataFieldException(DataModelException):
def __init__(self, field, message):
self.field = field
self.message = message
super(DataFieldException, self).__init__()
def __str__(self):
return "%s for `%s.%s` field" % (
self.message,
self.field.parent.__class__.__name__,
self.field.name,
)
def __repr__(self):
return str(self)
class ListOfType(object):
def __init__(self, type):
self.type = type
class DictOfType(object):
def __init__(self, type):
self.type = type
class DataField(object):
def __init__(
self,
default=None,
type=str,
required=False,
min_length=None,
max_length=None,
regex=None,
validate_factory=None,
title=None,
):
self.default = default
self.type = type
self.required = required
self.min_length = min_length
self.max_length = max_length
self.regex = regex
self.validate_factory = validate_factory
self.title = title
self._parent = None
self._name = None
self.value = None
def __repr__(self):
return '<DataField %s="%s">' % (
self.title,
self.default if self.value is None else self.value,
)
@property
def parent(self):
return self._parent
@parent.setter
def parent(self, value):
self._parent = value
@property
def name(self):
return self._name
@name.setter
def name(self, value):
self._name = value
self.title = self.title or value.title()
def validate(self, value):
try:
if self.required and value is None:
raise ValueError("Missed value")
if self.validate_factory is not None:
return self.validate_factory(self, value) or self.default
if value is None:
return self.default
if inspect.isclass(self.type) and issubclass(self.type, DataModel):
return self.type(**self.ensure_value_is_dict(value))
if inspect.isclass(self.type) and issubclass(self.type, (str, bool)):
return getattr(self, "_validate_%s_value" % self.type.__name__)(value)
except ValueError as e:
raise DataFieldException(self, str(e))
return value
@staticmethod
def ensure_value_is_dict(value):
if not isinstance(value, dict):
raise ValueError("Value should be type of dict, not `%s`" % type(value))
return value
def _validate_str_value(self, value):
if not isinstance(value, string_types):
value = str(value)
if self.min_length and len(value) < self.min_length:
raise ValueError(
"Minimum allowed length is %d characters" % self.min_length
)
if self.max_length and len(value) > self.max_length:
raise ValueError(
"Maximum allowed length is %d characters" % self.max_length
)
if self.regex and not re.match(self.regex, value):
raise ValueError(
"Value `%s` does not match RegExp `%s` pattern" % (value, self.regex)
)
return value
@staticmethod
def _validate_bool_value(value):
if isinstance(value, bool):
return value
return str(value).lower() in ("true", "yes", "1")
class DataModel(object):
_field_names = None
_exceptions = None
def __init__(self, **kwargs):
self._field_names = []
self._exceptions = set()
for name, field in get_class_attributes(self).items():
if not isinstance(field, DataField):
continue
field.parent = self
field.name = name
self._field_names.append(name)
raw_value = kwargs.get(name)
value = None
try:
if isinstance(field.type, ListOfType):
value = self._validate_list_of_type(field, name, raw_value)
elif isinstance(field.type, DictOfType):
value = self._validate_dict_of_type(field, name, raw_value)
else:
value = field.validate(raw_value)
except DataFieldException as e:
self._exceptions.add(e)
if isinstance(self, StrictDataModel):
raise e
finally:
setattr(self, name, value)
def _validate_list_of_type(self, field, name, value):
data_type = field.type.type
# check if ListOfType is not required
value = field.validate(value)
if not value:
return None
if not isinstance(value, list):
raise DataFieldException(field, "Value should be a list")
if isinstance(data_type, DataField):
result = []
data_type.parent = self
data_type.name = name
for v in value:
try:
result.append(data_type.validate(v))
except DataFieldException as e:
self._exceptions.add(e)
if isinstance(self, StrictDataModel):
raise e
return result
assert issubclass(data_type, DataModel)
result = []
for v in value:
try:
if not isinstance(v, dict):
raise DataFieldException(
field, "Value `%s` should be type of dictionary" % v
)
m = data_type(**v)
me = m.get_exceptions()
if not me:
result.append(m)
else:
self._exceptions |= set(me)
except DataFieldException as e:
self._exceptions.add(e)
if isinstance(self, StrictDataModel):
raise e
return result
def _validate_dict_of_type(self, field, _, value):
data_type = field.type.type
assert issubclass(data_type, DataModel)
# check if DictOfType is not required
value = field.validate(value)
if not value:
return None
if not isinstance(value, dict):
raise DataFieldException(
field, "Value `%s` should be type of dictionary" % value
)
result = {}
for k, v in value.items():
try:
if not isinstance(v, dict):
raise DataFieldException(
field, "Value `%s` should be type of dictionary" % v
)
m = data_type(**v)
me = m.get_exceptions()
if not me:
result[k] = m
else:
self._exceptions |= set(me)
except DataFieldException as e:
self._exceptions.add(e)
if isinstance(self, StrictDataModel):
raise e
return result
def __eq__(self, other):
assert isinstance(other, DataModel)
if self.get_field_names() != other.get_field_names():
return False
return self.as_dict() == other.as_dict()
def __repr__(self):
fields = []
for name in self._field_names:
fields.append('%s="%s"' % (name, getattr(self, name)))
return "<%s %s>" % (self.__class__.__name__, " ".join(fields))
def get_field_names(self):
return self._field_names
def get_exceptions(self):
result = list(self._exceptions)
for name in self._field_names:
value = getattr(self, name)
if isinstance(value, DataModel):
result.extend(value.get_exceptions())
continue
if not isinstance(value, (dict, list)):
continue
for v in value.values() if isinstance(value, dict) else value:
if not isinstance(v, DataModel):
continue
result.extend(v.get_exceptions())
return result
def as_dict(self):
result = {}
for name in self._field_names:
value = getattr(self, name)
if isinstance(value, DataModel):
value = getattr(self, name).as_dict()
if isinstance(value, dict):
for k, v in value.items():
if not isinstance(v, DataModel):
continue
value[k] = v.as_dict()
elif isinstance(value, list):
value = [v.as_dict() if isinstance(v, DataModel) else v for v in value]
result[name] = value
return result
class StrictDataModel(DataModel):
pass
+36
View File
@@ -0,0 +1,36 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from platformio.exception import PlatformioException
class ManifestException(PlatformioException):
pass
class ManifestParserError(ManifestException):
pass
class ManifestValidationError(ManifestException):
def __init__(self, error, data):
super(ManifestValidationError, self).__init__()
self.error = error
self.data = data
def __str__(self):
return (
"Invalid manifest fields: %s. \nPlease check specification -> "
"http://docs.platformio.org/page/librarymanager/config.html" % self.error
)
-92
View File
@@ -1,92 +0,0 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import semantic_version
from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel
def validate_semver_field(_, value):
value = str(value)
if "." not in value:
raise ValueError("Invalid semantic versioning format")
return value if semantic_version.Version.coerce(value) else None
class AuthorModel(DataModel):
name = DataField(max_length=50, required=True)
email = DataField(max_length=50)
maintainer = DataField(default=False, type=bool)
url = DataField(max_length=255)
class StrictAuthorModel(AuthorModel, StrictDataModel):
pass
class RepositoryModel(DataModel):
type = DataField(max_length=3, required=True)
url = DataField(max_length=255, required=True)
branch = DataField(max_length=50)
class ExportModel(DataModel):
include = DataField(type=ListOfType(DataField()))
exclude = DataField(type=ListOfType(DataField()))
class ExampleModel(DataModel):
name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/]+$", required=True)
base = DataField(required=True)
files = DataField(type=ListOfType(DataField()), required=True)
class ManifestModel(DataModel):
# Required fields
name = DataField(max_length=100, required=True)
version = DataField(
max_length=50, validate_factory=validate_semver_field, required=True
)
description = DataField(max_length=1000, required=True)
keywords = DataField(
type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\-\+\. ]+$")),
required=True,
)
authors = DataField(type=ListOfType(AuthorModel), required=True)
homepage = DataField(max_length=255)
license = DataField(max_length=255)
platforms = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$"))
)
frameworks = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$"))
)
repository = DataField(type=RepositoryModel)
export = DataField(type=ExportModel)
examples = DataField(type=ListOfType(ExampleModel))
# platform.json specific
title = DataField(max_length=100)
# package.json specific
system = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^[a-z\d\-_]+$"))
)
class StrictManifestModel(ManifestModel, StrictDataModel):
authors = DataField(type=ListOfType(StrictAuthorModel), required=True)
+24 -27
View File
@@ -19,8 +19,8 @@ import re
import requests
from platformio.compat import get_class_attributes, string_types
from platformio.exception import PlatformioException
from platformio.fs import get_file_contents
from platformio.package.exception import ManifestParserError
from platformio.project.helpers import is_platformio_project
try:
@@ -29,14 +29,6 @@ except ImportError:
from urlparse import urlparse
class ManifestException(PlatformioException):
pass
class ManifestParserException(ManifestException):
pass
class ManifestFileType(object):
PLATFORM_JSON = "platform.json"
LIBRARY_JSON = "library.json"
@@ -67,11 +59,11 @@ class ManifestParserFactory(object):
@staticmethod
def new_from_file(path, remote_url=False):
if not path or not os.path.isfile(path):
raise ManifestException("Manifest file does not exist %s" % path)
raise ManifestParserError("Manifest file does not exist %s" % path)
for t in get_class_attributes(ManifestFileType).values():
if path.endswith(t):
return ManifestParserFactory.new(get_file_contents(path), t, remote_url)
raise ManifestException("Unknown manifest file type %s" % path)
raise ManifestParserError("Unknown manifest file type %s" % path)
@staticmethod
def new_from_dir(path, remote_url=None):
@@ -102,7 +94,7 @@ class ManifestParserFactory(object):
remote_url=remote_url,
package_dir=path,
)
raise ManifestException("Unknown manifest file type in %s directory" % path)
raise ManifestParserError("Unknown manifest file type in %s directory" % path)
@staticmethod
def new_from_url(remote_url):
@@ -119,7 +111,7 @@ class ManifestParserFactory(object):
# pylint: disable=redefined-builtin
clsname = ManifestParserFactory.type_to_clsname(type)
if clsname not in globals():
raise ManifestException("Unknown manifest file type %s" % clsname)
raise ManifestParserError("Unknown manifest file type %s" % clsname)
return globals()[clsname](contents, remote_url, package_dir)
@@ -130,9 +122,14 @@ class BaseManifestParser(object):
try:
self._data = self.parse(contents)
except Exception as e:
raise ManifestParserException("Could not parse manifest -> %s" % e)
raise ManifestParserError("Could not parse manifest -> %s" % e)
self._data = self.parse_examples(self._data)
# remove None fields
for key in list(self._data.keys()):
if self._data[key] is None:
del self._data[key]
def parse(self, contents):
raise NotImplementedError
@@ -141,8 +138,12 @@ class BaseManifestParser(object):
@staticmethod
def cleanup_author(author):
assert isinstance(author, dict)
if author.get("email"):
author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"])
for key in list(author.keys()):
if author[key] is None:
del author[key]
return author
@staticmethod
@@ -351,9 +352,7 @@ class ModuleJsonManifestParser(BaseManifestParser):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
result.append(
self.cleanup_author(dict(name=name, email=email, maintainer=False))
)
result.append(self.cleanup_author(dict(name=name, email=email)))
return result
@staticmethod
@@ -431,7 +430,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
}
for arch in properties.get("architectures", "").split(","):
if "particle-" in arch:
raise ManifestParserException("Particle is not supported yet")
raise ManifestParserError("Particle is not supported yet")
arch = arch.strip()
if not arch:
continue
@@ -449,20 +448,18 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
authors.append(
self.cleanup_author(dict(name=name, email=email, maintainer=False))
)
authors.append(self.cleanup_author(dict(name=name, email=email)))
for author in properties.get("maintainer", "").split(","):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
found = False
for item in authors:
if item["name"].lower() != name.lower():
if item.get("name", "").lower() != name.lower():
continue
found = True
item["maintainer"] = True
if not item["email"]:
if not item.get("email"):
item["email"] = email
if not found:
authors.append(
@@ -495,6 +492,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
return None
def _parse_export(self):
result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]}
include = None
if self.remote_url:
repo_parse = urlparse(self.remote_url)
@@ -506,10 +504,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
"/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :])
or None
)
return {
"include": include,
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
}
if include:
result["include"] = include
return result
class PlatformJsonManifestParser(BaseManifestParser):
+181
View File
@@ -0,0 +1,181 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import requests
import semantic_version
from marshmallow import Schema, ValidationError, fields, validate, validates
from platformio.package.exception import ManifestValidationError
from platformio.util import memoized
class StrictSchema(Schema):
def handle_error(self, error, data):
# skip broken records
if self.many:
error.data = [
item for idx, item in enumerate(data) if idx not in error.messages
]
else:
error.data = None
raise error
class StrictListField(fields.List):
def _deserialize(self, value, attr, data):
try:
return super(StrictListField, self)._deserialize(value, attr, data)
except ValidationError as exc:
exc.data = [item for item in exc.data if item is not None]
raise exc
class AuthorSchema(StrictSchema):
name = fields.Str(required=True, validate=validate.Length(min=1, max=50))
email = fields.Email(validate=validate.Length(min=1, max=50))
maintainer = fields.Bool(default=False)
url = fields.Url(validate=validate.Length(min=1, max=255))
class RepositorySchema(StrictSchema):
type = fields.Str(
required=True,
validate=validate.OneOf(
["git", "hg", "svn"],
error="Invalid repository type, please use one of [git, hg, svn]",
),
)
url = fields.Str(required=True, validate=validate.Length(min=1, max=255))
branch = fields.Str(validate=validate.Length(min=1, max=50))
class ExportSchema(Schema):
include = StrictListField(fields.Str)
exclude = StrictListField(fields.Str)
class ExampleSchema(StrictSchema):
name = fields.Str(
required=True,
validate=[
validate.Length(min=1, max=100),
validate.Regexp(
r"^[a-zA-Z\d\-\_/]+$", error="Only [a-zA-Z0-9-_/] chars are allowed"
),
],
)
base = fields.Str(required=True)
files = StrictListField(fields.Str, required=True)
class ManifestSchema(Schema):
# Required fields
name = fields.Str(required=True, validate=validate.Length(min=1, max=100))
version = fields.Str(required=True, validate=validate.Length(min=1, max=50))
# Optional fields
authors = fields.Nested(AuthorSchema, many=True)
description = fields.Str(validate=validate.Length(min=1, max=1000))
homepage = fields.Url(validate=validate.Length(min=1, max=255))
license = fields.Str(validate=validate.Length(min=1, max=255))
repository = fields.Nested(RepositorySchema)
export = fields.Nested(ExportSchema)
examples = fields.Nested(ExampleSchema, many=True)
keywords = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed"
),
]
)
)
platforms = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
),
]
)
)
frameworks = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
),
]
)
)
# platform.json specific
title = fields.Str(validate=validate.Length(min=1, max=100))
# package.json specific
system = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed"
),
]
)
)
def handle_error(self, error, data):
if self.strict:
raise ManifestValidationError(error, data)
@validates("version")
def validate_version(self, value): # pylint: disable=no-self-use
try:
value = str(value)
assert "." in value
semantic_version.Version.coerce(value)
except (AssertionError, ValueError):
raise ValidationError(
"Invalid semantic versioning format, see https://semver.org/"
)
@validates("license")
def validate_license(self, value):
try:
spdx = self.load_spdx_licenses()
except requests.exceptions.RequestException:
raise ValidationError("Could not load SPDX licenses for validation")
for item in spdx.get("licenses", []):
if item.get("licenseId") == value:
return
raise ValidationError(
"Invalid SPDX license identifier. See valid identifiers at "
"https://spdx.org/licenses/"
)
@staticmethod
@memoized(expire="1h")
def load_spdx_licenses():
r = requests.get(
"https://raw.githubusercontent.com/spdx/license-list-data"
"/v3.6/json/licenses.json"
)
r.raise_for_status()
return r.json()