Switch to Marshmallow ODM framework

This commit is contained in:
Ivan Kravets committed 2019-10-17 00:17:16 +03:00
1 parent 9cfccc5cd4
commit 27fc19d6b3
10 files changed
+358 -585

No files matched your search

+36
View File
@@ -0,0 +1,36 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from platformio.exception import PlatformioException
class ManifestException(PlatformioException):
pass
class ManifestParserError(ManifestException):
pass
class ManifestValidationError(ManifestException):
def __init__(self, error, data):
super(ManifestValidationError, self).__init__()
self.error = error
self.data = data
def __str__(self):
return (
"Invalid manifest fields: %s. \nPlease check specification -> "
"http://docs.platformio.org/page/librarymanager/config.html" % self.error
)
-92
View File
@@ -1,92 +0,0 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import semantic_version
from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel
def validate_semver_field(_, value):
value = str(value)
if "." not in value:
raise ValueError("Invalid semantic versioning format")
return value if semantic_version.Version.coerce(value) else None
class AuthorModel(DataModel):
name = DataField(max_length=50, required=True)
email = DataField(max_length=50)
maintainer = DataField(default=False, type=bool)
url = DataField(max_length=255)
class StrictAuthorModel(AuthorModel, StrictDataModel):
pass
class RepositoryModel(DataModel):
type = DataField(max_length=3, required=True)
url = DataField(max_length=255, required=True)
branch = DataField(max_length=50)
class ExportModel(DataModel):
include = DataField(type=ListOfType(DataField()))
exclude = DataField(type=ListOfType(DataField()))
class ExampleModel(DataModel):
name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/]+$", required=True)
base = DataField(required=True)
files = DataField(type=ListOfType(DataField()), required=True)
class ManifestModel(DataModel):
# Required fields
name = DataField(max_length=100, required=True)
version = DataField(
max_length=50, validate_factory=validate_semver_field, required=True
)
description = DataField(max_length=1000, required=True)
keywords = DataField(
type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\-\+\. ]+$")),
required=True,
)
authors = DataField(type=ListOfType(AuthorModel), required=True)
homepage = DataField(max_length=255)
license = DataField(max_length=255)
platforms = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$"))
)
frameworks = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$"))
)
repository = DataField(type=RepositoryModel)
export = DataField(type=ExportModel)
examples = DataField(type=ListOfType(ExampleModel))
# platform.json specific
title = DataField(max_length=100)
# package.json specific
system = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^[a-z\d\-_]+$"))
)
class StrictManifestModel(ManifestModel, StrictDataModel):
authors = DataField(type=ListOfType(StrictAuthorModel), required=True)
+24 -27
View File
@@ -19,8 +19,8 @@ import re
import requests
from platformio.compat import get_class_attributes, string_types
from platformio.exception import PlatformioException
from platformio.fs import get_file_contents
from platformio.package.exception import ManifestParserError
from platformio.project.helpers import is_platformio_project
try:
@@ -29,14 +29,6 @@ except ImportError:
from urlparse import urlparse
class ManifestException(PlatformioException):
pass
class ManifestParserException(ManifestException):
pass
class ManifestFileType(object):
PLATFORM_JSON = "platform.json"
LIBRARY_JSON = "library.json"
@@ -67,11 +59,11 @@ class ManifestParserFactory(object):
@staticmethod
def new_from_file(path, remote_url=False):
if not path or not os.path.isfile(path):
raise ManifestException("Manifest file does not exist %s" % path)
raise ManifestParserError("Manifest file does not exist %s" % path)
for t in get_class_attributes(ManifestFileType).values():
if path.endswith(t):
return ManifestParserFactory.new(get_file_contents(path), t, remote_url)
raise ManifestException("Unknown manifest file type %s" % path)
raise ManifestParserError("Unknown manifest file type %s" % path)
@staticmethod
def new_from_dir(path, remote_url=None):
@@ -102,7 +94,7 @@ class ManifestParserFactory(object):
remote_url=remote_url,
package_dir=path,
)
raise ManifestException("Unknown manifest file type in %s directory" % path)
raise ManifestParserError("Unknown manifest file type in %s directory" % path)
@staticmethod
def new_from_url(remote_url):
@@ -119,7 +111,7 @@ class ManifestParserFactory(object):
# pylint: disable=redefined-builtin
clsname = ManifestParserFactory.type_to_clsname(type)
if clsname not in globals():
raise ManifestException("Unknown manifest file type %s" % clsname)
raise ManifestParserError("Unknown manifest file type %s" % clsname)
return globals()[clsname](contents, remote_url, package_dir)
@@ -130,9 +122,14 @@ class BaseManifestParser(object):
try:
self._data = self.parse(contents)
except Exception as e:
raise ManifestParserException("Could not parse manifest -> %s" % e)
raise ManifestParserError("Could not parse manifest -> %s" % e)
self._data = self.parse_examples(self._data)
# remove None fields
for key in list(self._data.keys()):
if self._data[key] is None:
del self._data[key]
def parse(self, contents):
raise NotImplementedError
@@ -141,8 +138,12 @@ class BaseManifestParser(object):
@staticmethod
def cleanup_author(author):
assert isinstance(author, dict)
if author.get("email"):
author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"])
for key in list(author.keys()):
if author[key] is None:
del author[key]
return author
@staticmethod
@@ -351,9 +352,7 @@ class ModuleJsonManifestParser(BaseManifestParser):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
result.append(
self.cleanup_author(dict(name=name, email=email, maintainer=False))
)
result.append(self.cleanup_author(dict(name=name, email=email)))
return result
@staticmethod
@@ -431,7 +430,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
}
for arch in properties.get("architectures", "").split(","):
if "particle-" in arch:
raise ManifestParserException("Particle is not supported yet")
raise ManifestParserError("Particle is not supported yet")
arch = arch.strip()
if not arch:
continue
@@ -449,20 +448,18 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
authors.append(
self.cleanup_author(dict(name=name, email=email, maintainer=False))
)
authors.append(self.cleanup_author(dict(name=name, email=email)))
for author in properties.get("maintainer", "").split(","):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
found = False
for item in authors:
if item["name"].lower() != name.lower():
if item.get("name", "").lower() != name.lower():
continue
found = True
item["maintainer"] = True
if not item["email"]:
if not item.get("email"):
item["email"] = email
if not found:
authors.append(
@@ -495,6 +492,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
return None
def _parse_export(self):
result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]}
include = None
if self.remote_url:
repo_parse = urlparse(self.remote_url)
@@ -506,10 +504,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
"/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :])
or None
)
return {
"include": include,
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
}
if include:
result["include"] = include
return result
class PlatformJsonManifestParser(BaseManifestParser):
+181
View File
@@ -0,0 +1,181 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import requests
import semantic_version
from marshmallow import Schema, ValidationError, fields, validate, validates
from platformio.package.exception import ManifestValidationError
from platformio.util import memoized
class StrictSchema(Schema):
def handle_error(self, error, data):
# skip broken records
if self.many:
error.data = [
item for idx, item in enumerate(data) if idx not in error.messages
]
else:
error.data = None
raise error
class StrictListField(fields.List):
def _deserialize(self, value, attr, data):
try:
return super(StrictListField, self)._deserialize(value, attr, data)
except ValidationError as exc:
exc.data = [item for item in exc.data if item is not None]
raise exc
class AuthorSchema(StrictSchema):
name = fields.Str(required=True, validate=validate.Length(min=1, max=50))
email = fields.Email(validate=validate.Length(min=1, max=50))
maintainer = fields.Bool(default=False)
url = fields.Url(validate=validate.Length(min=1, max=255))
class RepositorySchema(StrictSchema):
type = fields.Str(
required=True,
validate=validate.OneOf(
["git", "hg", "svn"],
error="Invalid repository type, please use one of [git, hg, svn]",
),
)
url = fields.Str(required=True, validate=validate.Length(min=1, max=255))
branch = fields.Str(validate=validate.Length(min=1, max=50))
class ExportSchema(Schema):
include = StrictListField(fields.Str)
exclude = StrictListField(fields.Str)
class ExampleSchema(StrictSchema):
name = fields.Str(
required=True,
validate=[
validate.Length(min=1, max=100),
validate.Regexp(
r"^[a-zA-Z\d\-\_/]+$", error="Only [a-zA-Z0-9-_/] chars are allowed"
),
],
)
base = fields.Str(required=True)
files = StrictListField(fields.Str, required=True)
class ManifestSchema(Schema):
# Required fields
name = fields.Str(required=True, validate=validate.Length(min=1, max=100))
version = fields.Str(required=True, validate=validate.Length(min=1, max=50))
# Optional fields
authors = fields.Nested(AuthorSchema, many=True)
description = fields.Str(validate=validate.Length(min=1, max=1000))
homepage = fields.Url(validate=validate.Length(min=1, max=255))
license = fields.Str(validate=validate.Length(min=1, max=255))
repository = fields.Nested(RepositorySchema)
export = fields.Nested(ExportSchema)
examples = fields.Nested(ExampleSchema, many=True)
keywords = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed"
),
]
)
)
platforms = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
),
]
)
)
frameworks = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
),
]
)
)
# platform.json specific
title = fields.Str(validate=validate.Length(min=1, max=100))
# package.json specific
system = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed"
),
]
)
)
def handle_error(self, error, data):
if self.strict:
raise ManifestValidationError(error, data)
@validates("version")
def validate_version(self, value): # pylint: disable=no-self-use
try:
value = str(value)
assert "." in value
semantic_version.Version.coerce(value)
except (AssertionError, ValueError):
raise ValidationError(
"Invalid semantic versioning format, see https://semver.org/"
)
@validates("license")
def validate_license(self, value):
try:
spdx = self.load_spdx_licenses()
except requests.exceptions.RequestException:
raise ValidationError("Could not load SPDX licenses for validation")
for item in spdx.get("licenses", []):
if item.get("licenseId") == value:
return
raise ValidationError(
"Invalid SPDX license identifier. See valid identifiers at "
"https://spdx.org/licenses/"
)
@staticmethod
@memoized(expire="1h")
def load_spdx_licenses():
r = requests.get(
"https://raw.githubusercontent.com/spdx/license-list-data"
"/v3.6/json/licenses.json"
)
r.raise_for_status()
return r.json()