Switch to Marshmallow ODM framework

This commit is contained in:
Ivan Kravets
2019-10-17 00:17:16 +03:00
parent 9cfccc5cd4
commit 27fc19d6b3
10 changed files with 358 additions and 585 deletions

2
docs

Submodule docs updated: 3c565d175d...e8b9361615

View File

@ -25,8 +25,8 @@ from platformio import exception, util
from platformio.commands import PlatformioCLI
from platformio.compat import dump_json_to_unicode
from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
from platformio.package.manifest.model import StrictManifestModel
from platformio.package.manifest.parser import ManifestParserFactory
from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError
from platformio.proc import is_ci
from platformio.project.config import ProjectConfig
from platformio.project.helpers import get_project_dir, is_platformio_project
@ -495,7 +495,11 @@ def lib_register(config_url):
raise exception.InvalidLibConfURL(config_url)
# Validate manifest
StrictManifestModel(**ManifestParserFactory.new_from_url(config_url).as_dict())
data, error = ManifestSchema(strict=False).load(
ManifestParserFactory.new_from_url(config_url).as_dict()
)
if error:
raise ManifestValidationError(error, data)
result = util.get_api_result("/lib/register", data=dict(config_url=config_url))
if "message" in result and result["message"]:

View File

@ -1,303 +0,0 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
import re
from platformio.compat import get_class_attributes, string_types
from platformio.exception import PlatformioException
# pylint: disable=too-many-instance-attributes
# pylint: disable=redefined-builtin, too-many-arguments
class DataModelException(PlatformioException):
pass
class DataFieldException(DataModelException):
def __init__(self, field, message):
self.field = field
self.message = message
super(DataFieldException, self).__init__()
def __str__(self):
return "%s for `%s.%s` field" % (
self.message,
self.field.parent.__class__.__name__,
self.field.name,
)
def __repr__(self):
return str(self)
class ListOfType(object):
def __init__(self, type):
self.type = type
class DictOfType(object):
def __init__(self, type):
self.type = type
class DataField(object):
def __init__(
self,
default=None,
type=str,
required=False,
min_length=None,
max_length=None,
regex=None,
validate_factory=None,
title=None,
):
self.default = default
self.type = type
self.required = required
self.min_length = min_length
self.max_length = max_length
self.regex = regex
self.validate_factory = validate_factory
self.title = title
self._parent = None
self._name = None
self.value = None
def __repr__(self):
return '<DataField %s="%s">' % (
self.title,
self.default if self.value is None else self.value,
)
@property
def parent(self):
return self._parent
@parent.setter
def parent(self, value):
self._parent = value
@property
def name(self):
return self._name
@name.setter
def name(self, value):
self._name = value
self.title = self.title or value.title()
def validate(self, value):
try:
if self.required and value is None:
raise ValueError("Missed value")
if self.validate_factory is not None:
return self.validate_factory(self, value) or self.default
if value is None:
return self.default
if inspect.isclass(self.type) and issubclass(self.type, DataModel):
return self.type(**self.ensure_value_is_dict(value))
if inspect.isclass(self.type) and issubclass(self.type, (str, bool)):
return getattr(self, "_validate_%s_value" % self.type.__name__)(value)
except ValueError as e:
raise DataFieldException(self, str(e))
return value
@staticmethod
def ensure_value_is_dict(value):
if not isinstance(value, dict):
raise ValueError("Value should be type of dict, not `%s`" % type(value))
return value
def _validate_str_value(self, value):
if not isinstance(value, string_types):
value = str(value)
if self.min_length and len(value) < self.min_length:
raise ValueError(
"Minimum allowed length is %d characters" % self.min_length
)
if self.max_length and len(value) > self.max_length:
raise ValueError(
"Maximum allowed length is %d characters" % self.max_length
)
if self.regex and not re.match(self.regex, value):
raise ValueError(
"Value `%s` does not match RegExp `%s` pattern" % (value, self.regex)
)
return value
@staticmethod
def _validate_bool_value(value):
if isinstance(value, bool):
return value
return str(value).lower() in ("true", "yes", "1")
class DataModel(object):
_field_names = None
_exceptions = None
def __init__(self, **kwargs):
self._field_names = []
self._exceptions = set()
for name, field in get_class_attributes(self).items():
if not isinstance(field, DataField):
continue
field.parent = self
field.name = name
self._field_names.append(name)
raw_value = kwargs.get(name)
value = None
try:
if isinstance(field.type, ListOfType):
value = self._validate_list_of_type(field, name, raw_value)
elif isinstance(field.type, DictOfType):
value = self._validate_dict_of_type(field, name, raw_value)
else:
value = field.validate(raw_value)
except DataFieldException as e:
self._exceptions.add(e)
if isinstance(self, StrictDataModel):
raise e
finally:
setattr(self, name, value)
def _validate_list_of_type(self, field, name, value):
data_type = field.type.type
# check if ListOfType is not required
value = field.validate(value)
if not value:
return None
if not isinstance(value, list):
raise DataFieldException(field, "Value should be a list")
if isinstance(data_type, DataField):
result = []
data_type.parent = self
data_type.name = name
for v in value:
try:
result.append(data_type.validate(v))
except DataFieldException as e:
self._exceptions.add(e)
if isinstance(self, StrictDataModel):
raise e
return result
assert issubclass(data_type, DataModel)
result = []
for v in value:
try:
if not isinstance(v, dict):
raise DataFieldException(
field, "Value `%s` should be type of dictionary" % v
)
m = data_type(**v)
me = m.get_exceptions()
if not me:
result.append(m)
else:
self._exceptions |= set(me)
except DataFieldException as e:
self._exceptions.add(e)
if isinstance(self, StrictDataModel):
raise e
return result
def _validate_dict_of_type(self, field, _, value):
data_type = field.type.type
assert issubclass(data_type, DataModel)
# check if DictOfType is not required
value = field.validate(value)
if not value:
return None
if not isinstance(value, dict):
raise DataFieldException(
field, "Value `%s` should be type of dictionary" % value
)
result = {}
for k, v in value.items():
try:
if not isinstance(v, dict):
raise DataFieldException(
field, "Value `%s` should be type of dictionary" % v
)
m = data_type(**v)
me = m.get_exceptions()
if not me:
result[k] = m
else:
self._exceptions |= set(me)
except DataFieldException as e:
self._exceptions.add(e)
if isinstance(self, StrictDataModel):
raise e
return result
def __eq__(self, other):
assert isinstance(other, DataModel)
if self.get_field_names() != other.get_field_names():
return False
return self.as_dict() == other.as_dict()
def __repr__(self):
fields = []
for name in self._field_names:
fields.append('%s="%s"' % (name, getattr(self, name)))
return "<%s %s>" % (self.__class__.__name__, " ".join(fields))
def get_field_names(self):
return self._field_names
def get_exceptions(self):
result = list(self._exceptions)
for name in self._field_names:
value = getattr(self, name)
if isinstance(value, DataModel):
result.extend(value.get_exceptions())
continue
if not isinstance(value, (dict, list)):
continue
for v in value.values() if isinstance(value, dict) else value:
if not isinstance(v, DataModel):
continue
result.extend(v.get_exceptions())
return result
def as_dict(self):
result = {}
for name in self._field_names:
value = getattr(self, name)
if isinstance(value, DataModel):
value = getattr(self, name).as_dict()
if isinstance(value, dict):
for k, v in value.items():
if not isinstance(v, DataModel):
continue
value[k] = v.as_dict()
elif isinstance(value, list):
value = [v.as_dict() if isinstance(v, DataModel) else v for v in value]
result[name] = value
return result
class StrictDataModel(DataModel):
pass

View File

@ -0,0 +1,36 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from platformio.exception import PlatformioException
class ManifestException(PlatformioException):
pass
class ManifestParserError(ManifestException):
pass
class ManifestValidationError(ManifestException):
def __init__(self, error, data):
super(ManifestValidationError, self).__init__()
self.error = error
self.data = data
def __str__(self):
return (
"Invalid manifest fields: %s. \nPlease check specification -> "
"http://docs.platformio.org/page/librarymanager/config.html" % self.error
)

View File

@ -1,92 +0,0 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import semantic_version
from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel
def validate_semver_field(_, value):
value = str(value)
if "." not in value:
raise ValueError("Invalid semantic versioning format")
return value if semantic_version.Version.coerce(value) else None
class AuthorModel(DataModel):
name = DataField(max_length=50, required=True)
email = DataField(max_length=50)
maintainer = DataField(default=False, type=bool)
url = DataField(max_length=255)
class StrictAuthorModel(AuthorModel, StrictDataModel):
pass
class RepositoryModel(DataModel):
type = DataField(max_length=3, required=True)
url = DataField(max_length=255, required=True)
branch = DataField(max_length=50)
class ExportModel(DataModel):
include = DataField(type=ListOfType(DataField()))
exclude = DataField(type=ListOfType(DataField()))
class ExampleModel(DataModel):
name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/]+$", required=True)
base = DataField(required=True)
files = DataField(type=ListOfType(DataField()), required=True)
class ManifestModel(DataModel):
# Required fields
name = DataField(max_length=100, required=True)
version = DataField(
max_length=50, validate_factory=validate_semver_field, required=True
)
description = DataField(max_length=1000, required=True)
keywords = DataField(
type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\-\+\. ]+$")),
required=True,
)
authors = DataField(type=ListOfType(AuthorModel), required=True)
homepage = DataField(max_length=255)
license = DataField(max_length=255)
platforms = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$"))
)
frameworks = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$"))
)
repository = DataField(type=RepositoryModel)
export = DataField(type=ExportModel)
examples = DataField(type=ListOfType(ExampleModel))
# platform.json specific
title = DataField(max_length=100)
# package.json specific
system = DataField(
type=ListOfType(DataField(max_length=50, regex=r"^[a-z\d\-_]+$"))
)
class StrictManifestModel(ManifestModel, StrictDataModel):
authors = DataField(type=ListOfType(StrictAuthorModel), required=True)

View File

@ -19,8 +19,8 @@ import re
import requests
from platformio.compat import get_class_attributes, string_types
from platformio.exception import PlatformioException
from platformio.fs import get_file_contents
from platformio.package.exception import ManifestParserError
from platformio.project.helpers import is_platformio_project
try:
@ -29,14 +29,6 @@ except ImportError:
from urlparse import urlparse
class ManifestException(PlatformioException):
pass
class ManifestParserException(ManifestException):
pass
class ManifestFileType(object):
PLATFORM_JSON = "platform.json"
LIBRARY_JSON = "library.json"
@ -67,11 +59,11 @@ class ManifestParserFactory(object):
@staticmethod
def new_from_file(path, remote_url=False):
if not path or not os.path.isfile(path):
raise ManifestException("Manifest file does not exist %s" % path)
raise ManifestParserError("Manifest file does not exist %s" % path)
for t in get_class_attributes(ManifestFileType).values():
if path.endswith(t):
return ManifestParserFactory.new(get_file_contents(path), t, remote_url)
raise ManifestException("Unknown manifest file type %s" % path)
raise ManifestParserError("Unknown manifest file type %s" % path)
@staticmethod
def new_from_dir(path, remote_url=None):
@ -102,7 +94,7 @@ class ManifestParserFactory(object):
remote_url=remote_url,
package_dir=path,
)
raise ManifestException("Unknown manifest file type in %s directory" % path)
raise ManifestParserError("Unknown manifest file type in %s directory" % path)
@staticmethod
def new_from_url(remote_url):
@ -119,7 +111,7 @@ class ManifestParserFactory(object):
# pylint: disable=redefined-builtin
clsname = ManifestParserFactory.type_to_clsname(type)
if clsname not in globals():
raise ManifestException("Unknown manifest file type %s" % clsname)
raise ManifestParserError("Unknown manifest file type %s" % clsname)
return globals()[clsname](contents, remote_url, package_dir)
@ -130,9 +122,14 @@ class BaseManifestParser(object):
try:
self._data = self.parse(contents)
except Exception as e:
raise ManifestParserException("Could not parse manifest -> %s" % e)
raise ManifestParserError("Could not parse manifest -> %s" % e)
self._data = self.parse_examples(self._data)
# remove None fields
for key in list(self._data.keys()):
if self._data[key] is None:
del self._data[key]
def parse(self, contents):
raise NotImplementedError
@ -141,8 +138,12 @@ class BaseManifestParser(object):
@staticmethod
def cleanup_author(author):
assert isinstance(author, dict)
if author.get("email"):
author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"])
for key in list(author.keys()):
if author[key] is None:
del author[key]
return author
@staticmethod
@ -351,9 +352,7 @@ class ModuleJsonManifestParser(BaseManifestParser):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
result.append(
self.cleanup_author(dict(name=name, email=email, maintainer=False))
)
result.append(self.cleanup_author(dict(name=name, email=email)))
return result
@staticmethod
@ -431,7 +430,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
}
for arch in properties.get("architectures", "").split(","):
if "particle-" in arch:
raise ManifestParserException("Particle is not supported yet")
raise ManifestParserError("Particle is not supported yet")
arch = arch.strip()
if not arch:
continue
@ -449,20 +448,18 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
authors.append(
self.cleanup_author(dict(name=name, email=email, maintainer=False))
)
authors.append(self.cleanup_author(dict(name=name, email=email)))
for author in properties.get("maintainer", "").split(","):
name, email = self.parse_author_name_and_email(author)
if not name:
continue
found = False
for item in authors:
if item["name"].lower() != name.lower():
if item.get("name", "").lower() != name.lower():
continue
found = True
item["maintainer"] = True
if not item["email"]:
if not item.get("email"):
item["email"] = email
if not found:
authors.append(
@ -495,6 +492,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
return None
def _parse_export(self):
result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]}
include = None
if self.remote_url:
repo_parse = urlparse(self.remote_url)
@ -506,10 +504,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
"/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :])
or None
)
return {
"include": include,
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
}
if include:
result["include"] = include
return result
class PlatformJsonManifestParser(BaseManifestParser):

View File

@ -0,0 +1,181 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import requests
import semantic_version
from marshmallow import Schema, ValidationError, fields, validate, validates
from platformio.package.exception import ManifestValidationError
from platformio.util import memoized
class StrictSchema(Schema):
def handle_error(self, error, data):
# skip broken records
if self.many:
error.data = [
item for idx, item in enumerate(data) if idx not in error.messages
]
else:
error.data = None
raise error
class StrictListField(fields.List):
def _deserialize(self, value, attr, data):
try:
return super(StrictListField, self)._deserialize(value, attr, data)
except ValidationError as exc:
exc.data = [item for item in exc.data if item is not None]
raise exc
class AuthorSchema(StrictSchema):
name = fields.Str(required=True, validate=validate.Length(min=1, max=50))
email = fields.Email(validate=validate.Length(min=1, max=50))
maintainer = fields.Bool(default=False)
url = fields.Url(validate=validate.Length(min=1, max=255))
class RepositorySchema(StrictSchema):
type = fields.Str(
required=True,
validate=validate.OneOf(
["git", "hg", "svn"],
error="Invalid repository type, please use one of [git, hg, svn]",
),
)
url = fields.Str(required=True, validate=validate.Length(min=1, max=255))
branch = fields.Str(validate=validate.Length(min=1, max=50))
class ExportSchema(Schema):
include = StrictListField(fields.Str)
exclude = StrictListField(fields.Str)
class ExampleSchema(StrictSchema):
name = fields.Str(
required=True,
validate=[
validate.Length(min=1, max=100),
validate.Regexp(
r"^[a-zA-Z\d\-\_/]+$", error="Only [a-zA-Z0-9-_/] chars are allowed"
),
],
)
base = fields.Str(required=True)
files = StrictListField(fields.Str, required=True)
class ManifestSchema(Schema):
# Required fields
name = fields.Str(required=True, validate=validate.Length(min=1, max=100))
version = fields.Str(required=True, validate=validate.Length(min=1, max=50))
# Optional fields
authors = fields.Nested(AuthorSchema, many=True)
description = fields.Str(validate=validate.Length(min=1, max=1000))
homepage = fields.Url(validate=validate.Length(min=1, max=255))
license = fields.Str(validate=validate.Length(min=1, max=255))
repository = fields.Nested(RepositorySchema)
export = fields.Nested(ExportSchema)
examples = fields.Nested(ExampleSchema, many=True)
keywords = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed"
),
]
)
)
platforms = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
),
]
)
)
frameworks = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
),
]
)
)
# platform.json specific
title = fields.Str(validate=validate.Length(min=1, max=100))
# package.json specific
system = StrictListField(
fields.Str(
validate=[
validate.Length(min=1, max=50),
validate.Regexp(
r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed"
),
]
)
)
def handle_error(self, error, data):
if self.strict:
raise ManifestValidationError(error, data)
@validates("version")
def validate_version(self, value): # pylint: disable=no-self-use
try:
value = str(value)
assert "." in value
semantic_version.Version.coerce(value)
except (AssertionError, ValueError):
raise ValidationError(
"Invalid semantic versioning format, see https://semver.org/"
)
@validates("license")
def validate_license(self, value):
try:
spdx = self.load_spdx_licenses()
except requests.exceptions.RequestException:
raise ValidationError("Could not load SPDX licenses for validation")
for item in spdx.get("licenses", []):
if item.get("licenseId") == value:
return
raise ValidationError(
"Invalid SPDX license identifier. See valid identifiers at "
"https://spdx.org/licenses/"
)
@staticmethod
@memoized(expire="1h")
def load_spdx_licenses():
r = requests.get(
"https://raw.githubusercontent.com/spdx/license-list-data"
"/v3.6/json/licenses.json"
)
r.raise_for_status()
return r.json()

View File

@ -33,6 +33,7 @@ install_requires = [
"semantic_version>=2.8.1,<3",
"tabulate>=0.8.3,<1",
"pyelftools>=0.25,<1",
"marshmallow>=2.20.5,<3"
]
setup(

View File

@ -12,11 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import jsondiff
import pytest
from platformio import datamodel
from platformio.compat import WINDOWS
from platformio.package.manifest import model, parser
from platformio.package.manifest import parser
from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError
def test_library_json_parser():
@ -31,14 +32,15 @@ def test_library_json_parser():
}
"""
mp = parser.LibraryJsonManifestParser(contents)
assert sorted(mp.as_dict().items()) == sorted(
assert not jsondiff.diff(
mp.as_dict(),
{
"name": "TestPackage",
"platforms": ["atmelavr", "espressif8266"],
"export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]},
"keywords": ["kw1", "kw2", "kw3"],
"homepage": "http://old.url.format",
}.items()
},
)
contents = """
@ -52,13 +54,14 @@ def test_library_json_parser():
}
"""
mp = parser.LibraryJsonManifestParser(contents)
assert sorted(mp.as_dict().items()) == sorted(
assert not jsondiff.diff(
mp.as_dict(),
{
"keywords": ["sound", "audio", "music", "sd", "card", "playback"],
"frameworks": ["arduino"],
"export": {"exclude": ["audio_samples"]},
"platforms": ["atmelavr"],
}.items()
},
)
@ -87,7 +90,8 @@ def test_module_json_parser():
}
"""
mp = parser.ModuleJsonManifestParser(contents)
assert sorted(mp.as_dict().items()) == sorted(
assert not jsondiff.diff(
mp.as_dict(),
{
"name": "YottaLibrary",
"description": "This is Yotta library",
@ -97,15 +101,9 @@ def test_module_json_parser():
"platforms": ["*"],
"frameworks": ["mbed"],
"export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]},
"authors": [
{
"maintainer": False,
"email": "name@surname.com",
"name": "Name Surname",
}
],
"authors": [{"email": "name@surname.com", "name": "Name Surname"}],
"version": "1.2.3",
}.items()
},
)
@ -118,24 +116,20 @@ author=SomeAuthor <info AT author.com>
sentence=This is Arduino library
"""
mp = parser.LibraryPropertiesManifestParser(contents)
assert sorted(mp.as_dict().items()) == sorted(
assert not jsondiff.diff(
mp.as_dict(),
{
"name": "TestPackage",
"version": "1.2.3",
"description": "This is Arduino library",
"repository": None,
"platforms": ["*"],
"frameworks": ["arduino"],
"export": {
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
"include": None,
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]
},
"authors": [
{"maintainer": False, "email": "info@author.com", "name": "SomeAuthor"}
],
"authors": [{"email": "info@author.com", "name": "SomeAuthor"}],
"keywords": ["uncategorized"],
"homepage": None,
}.items()
},
)
# Platforms ALL
@ -170,7 +164,6 @@ sentence=This is Arduino library
data = parser.LibraryPropertiesManifestParser(
"url=https://github.com/username/reponame.git\n" + contents
).as_dict()
assert data["homepage"] is None
assert data["repository"] == {
"type": "git",
"url": "https://github.com/username/reponame.git",
@ -216,15 +209,20 @@ def test_library_json_model():
]
}
"""
data = parser.ManifestParserFactory.new(
raw_data = parser.ManifestParserFactory.new(
contents, parser.ManifestFileType.LIBRARY_JSON
).as_dict()
m = model.StrictManifestModel(**data)
assert m.repository.url == "https://github.com/bblanchon/ArduinoJson.git"
assert m.examples[1].base == "examples/JsonHttpClient"
assert m.examples[1].files == ["JsonHttpClient.ino"]
assert m == model.StrictManifestModel(
**{
data, errors = ManifestSchema(strict=True).load(raw_data)
assert not errors
assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git"
assert data["examples"][1]["base"] == "examples/JsonHttpClient"
assert data["examples"][1]["files"] == ["JsonHttpClient.ino"]
assert not jsondiff.diff(
data,
{
"name": "ArduinoJson",
"keywords": ["json", "rest", "http", "web"],
"description": "An elegant and efficient JSON library for embedded systems",
@ -232,21 +230,12 @@ def test_library_json_model():
"repository": {
"url": "https://github.com/bblanchon/ArduinoJson.git",
"type": "git",
"branch": None,
},
"version": "6.12.0",
"authors": [
{
"name": "Benoit Blanchon",
"url": "https://blog.benoitblanchon.fr",
"maintainer": False,
"email": None,
}
{"name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr"}
],
"export": {
"exclude": ["fuzzing", "scripts", "test", "third-party"],
"include": None,
},
"export": {"exclude": ["fuzzing", "scripts", "test", "third-party"]},
"frameworks": ["arduino"],
"platforms": ["*"],
"license": "MIT",
@ -262,7 +251,7 @@ def test_library_json_model():
"files": ["JsonHttpClient.ino"],
},
],
}
},
)
@ -278,42 +267,33 @@ category=Display
url=https://github.com/olikraus/u8glib
architectures=avr,sam
"""
data = parser.ManifestParserFactory.new(
raw_data = parser.ManifestParserFactory.new(
contents, parser.ManifestFileType.LIBRARY_PROPERTIES
).as_dict()
m = model.StrictManifestModel(**data)
assert not m.get_exceptions()
assert m == model.StrictManifestModel(
**{
"license": None,
data, errors = ManifestSchema(strict=True).load(raw_data)
assert not errors
assert not jsondiff.diff(
data,
{
"description": (
"A library for monochrome TFTs and OLEDs. Supported display "
"controller: SSD1306, SSD1309, SSD1322, SSD1325"
),
"repository": {
"url": "https://github.com/olikraus/u8glib",
"type": "git",
"branch": None,
},
"repository": {"url": "https://github.com/olikraus/u8glib", "type": "git"},
"frameworks": ["arduino"],
"platforms": ["atmelavr", "atmelsam"],
"version": "1.19.1",
"export": {
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
"include": None,
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]
},
"authors": [
{
"url": None,
"maintainer": True,
"email": "olikraus@gmail.com",
"name": "oliver",
}
{"maintainer": True, "email": "olikraus@gmail.com", "name": "oliver"}
],
"keywords": ["display"],
"homepage": None,
"name": "U8glib",
}
},
)
# Broken fields
@ -330,7 +310,7 @@ architectures=*
dot_a_linkage=false
includes=MozziGuts.h
"""
data = parser.ManifestParserFactory.new(
raw_data = parser.ManifestParserFactory.new(
contents,
parser.ManifestFileType.LIBRARY_PROPERTIES,
remote_url=(
@ -338,10 +318,13 @@ includes=MozziGuts.h
"master/library.properties"
),
).as_dict()
m = model.ManifestModel(**data)
assert m.get_exceptions()
assert m == model.ManifestModel(
**{
data, errors = ManifestSchema(strict=False).load(raw_data)
assert errors["authors"]
assert not jsondiff.diff(
data,
{
"name": "Mozzi",
"version": "1.0.3",
"description": (
@ -353,8 +336,7 @@ includes=MozziGuts.h
"platforms": ["*"],
"frameworks": ["arduino"],
"export": {
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
"include": None,
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]
},
"authors": [
{
@ -365,7 +347,7 @@ includes=MozziGuts.h
],
"keywords": ["signal", "input", "output"],
"homepage": "https://sensorium.github.io/Mozzi/",
}
},
)
@ -419,14 +401,16 @@ def test_platform_json_model():
}
}
"""
data = parser.ManifestParserFactory.new(
raw_data = parser.ManifestParserFactory.new(
contents, parser.ManifestFileType.PLATFORM_JSON
).as_dict()
data["frameworks"] = sorted(data["frameworks"])
m = model.ManifestModel(**data)
assert m.frameworks == ["arduino", "simba"]
assert m == model.ManifestModel(
**{
data, errors = ManifestSchema(strict=False).load(raw_data)
assert not errors
assert not jsondiff.diff(
data,
{
"name": "atmelavr",
"title": "Atmel AVR",
"description": (
@ -441,11 +425,10 @@ def test_platform_json_model():
"repository": {
"url": "https://github.com/platformio/platform-atmelavr.git",
"type": "git",
"branch": None,
},
"frameworks": ["arduino", "simba"],
"version": "1.15.0",
}
},
)
@ -458,19 +441,21 @@ def test_package_json_model():
"version": "3.30101.0"
}
"""
data = parser.ManifestParserFactory.new(
raw_data = parser.ManifestParserFactory.new(
contents, parser.ManifestFileType.PACKAGE_JSON
).as_dict()
m = model.ManifestModel(**data)
assert m.system is None
assert m.homepage == "http://www.scons.org"
assert m == model.ManifestModel(
**{
data, errors = ManifestSchema(strict=False).load(raw_data)
assert not errors
assert not jsondiff.diff(
data,
{
"name": "tool-scons",
"description": "SCons software construction tool",
"homepage": "http://www.scons.org",
"version": "3.30101.0",
}
},
)
mp = parser.ManifestParserFactory.new(
@ -543,20 +528,23 @@ def test_examples_from_dir(tmpdir_factory):
# Do testing
data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict()
assert isinstance(data["examples"], list)
assert len(data["examples"]) == 6
raw_data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict()
assert isinstance(raw_data["examples"], list)
assert len(raw_data["examples"]) == 6
def _sort_examples(items):
for i, item in enumerate(items):
items[i]["files"] = sorted(item["files"])
return sorted(items, key=lambda item: item["name"])
data["examples"] = _sort_examples(data["examples"])
m = model.ManifestModel(**data)
assert m.examples[3].name == "PlatformIO/hello"
assert m == model.ManifestModel(
**{
raw_data["examples"] = _sort_examples(raw_data["examples"])
data, errors = ManifestSchema(strict=True).load(raw_data)
assert not errors
assert not jsondiff.diff(
data,
{
"version": "1.0.0",
"name": "pkg",
"examples": _sort_examples(
@ -599,84 +587,44 @@ def test_examples_from_dir(tmpdir_factory):
},
]
),
}
},
)
def test_dict_of_type():
class TestModel(datamodel.DataModel):
examples = datamodel.DataField(type=datamodel.DictOfType(model.ExampleModel))
class StrictTestModel(TestModel, datamodel.StrictDataModel):
pass
# valid
m = TestModel(
examples={
"valid": dict(name="Valid", base="valid", files=["valid.h"]),
"invalid": "test",
}
)
assert list(m.examples.keys()) == ["valid"]
# invalid
with pytest.raises(datamodel.DataFieldException):
StrictTestModel(examples=[dict(name="Valid", base="valid", files=["valid.h"])])
with pytest.raises(datamodel.DataFieldException):
StrictTestModel(
examples={
"valid": dict(name="Valid", base="valid", files=["valid.h"]),
"invalid": "test",
}
)
def test_broken_models():
# non-strict mode
assert len(model.ManifestModel(name="MyPackage").get_exceptions()) == 4
assert (
model.ManifestModel(name="MyPackage", version="broken_version").version is None
)
data, errors = ManifestSchema(strict=False).load(dict(name="MyPackage"))
assert set(errors.keys()) == set(["version"])
assert data.get("version") is None
# invalid keywords
m = model.ManifestModel(keywords=["kw1", "*^[]"])
assert any(
"Value `*^[]` does not match RegExp" in str(e) for e in m.get_exceptions()
)
assert m.keywords == ["kw1"]
data, errors = ManifestSchema(strict=False).load(dict(keywords=["kw1", "*^[]"]))
assert errors
assert data["keywords"] == ["kw1"]
# strict mode
with pytest.raises(datamodel.DataFieldException) as excinfo:
assert model.StrictManifestModel(name="MyPackage")
assert excinfo.match(r"Missed value for `StrictManifestModel.[a-z]+` field")
with pytest.raises(
ManifestValidationError, match="Missing data for required field"
):
ManifestSchema(strict=True).load(dict(name="MyPackage"))
# broken SemVer
with pytest.raises(
datamodel.DataFieldException,
match=(
"Invalid semantic versioning format for "
"`StrictManifestModel.version` field"
),
ManifestValidationError, match=("Invalid semantic versioning format")
):
assert model.StrictManifestModel(
name="MyPackage",
description="MyDescription",
keywords=["a", "b"],
authors=[{"name": "Author"}],
version="broken_version",
ManifestSchema(strict=True).load(
dict(name="MyPackage", version="broken_version")
)
# broken value for DataModel
with pytest.raises(
datamodel.DataFieldException,
match=("Value `should be dict here` should be type of dictionary"),
):
assert model.StrictManifestModel(
name="MyPackage",
description="MyDescription",
keywords=["a", "b"],
authors=["should be dict here"],
version="1.2.3",
# broken value for Nested
with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"):
ManifestSchema(strict=True).load(
dict(
name="MyPackage",
description="MyDescription",
keywords=["a", "b"],
authors=["should be dict here"],
version="1.2.3",
)
)

View File

@ -24,6 +24,7 @@ deps =
pylint
pytest
pytest-xdist
jsondiff
commands =
{envpython} --version
pylint --rcfile=./.pylintrc ./platformio