From 442a7e357636522e844d95375c246644b21a7802 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 29 Dec 2019 13:13:04 +0200 Subject: [PATCH] Made package ManifestSchema compatible with marshmallow >= 3 // Resolve #3296 --- HISTORY.rst | 1 + platformio/commands/lib.py | 6 +-- platformio/package/exception.py | 7 +-- platformio/package/manifest/schema.py | 61 +++++++++++++++++++++------ platformio/util.py | 3 +- setup.py | 2 +- tests/test_pkgmanifest.py | 58 ++++++++++++------------- 7 files changed, 85 insertions(+), 53 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a681588c..1ede584d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ PlatformIO Core 4.0 * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Updated SCons tool to 3.1.2 +* Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 58cb85ed..d7c42d4c 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -26,7 +26,7 @@ from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib from platformio.package.manifest.parser import ManifestParserFactory -from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError +from platformio.package.manifest.schema import ManifestSchema from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -495,11 +495,9 @@ def lib_register(config_url): raise exception.InvalidLibConfURL(config_url) # Validate manifest - data, error = ManifestSchema(strict=False).load( + ManifestSchema().load_manifest( ManifestParserFactory.new_from_url(config_url).as_dict() ) - if error: - raise ManifestValidationError(error, data) result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 81753ad1..f2597dd7 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -24,13 +24,14 @@ class ManifestParserError(ManifestException): class ManifestValidationError(ManifestException): - def __init__(self, error, data): + def __init__(self, messages, data, valid_data): super(ManifestValidationError, self).__init__() - self.error = error + self.messages = messages self.data = data + self.valid_data = valid_data def __str__(self): return ( "Invalid manifest fields: %s. \nPlease check specification -> " - "http://docs.platformio.org/page/librarymanager/config.html" % self.error + "http://docs.platformio.org/page/librarymanager/config.html" % self.messages ) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 790f3226..7c1b34c4 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import marshmallow import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates @@ -19,23 +20,61 @@ from marshmallow import Schema, ValidationError, fields, validate, validates from platformio.package.exception import ManifestValidationError from platformio.util import memoized +MARSHMALLOW_2 = marshmallow.__version_info__ < (3,) -class StrictSchema(Schema): - def handle_error(self, error, data): + +if MARSHMALLOW_2: + + class CompatSchema(Schema): + pass + + +else: + + class CompatSchema(Schema): + class Meta: + unknown = marshmallow.EXCLUDE + + def handle_error( # pylint: disable=arguments-differ + self, error, data, **kwargs + ): + raise ManifestValidationError( + error.messages, + data, + error.valid_data if hasattr(error, "valid_data") else error.data, + ) + + +class BaseSchema(CompatSchema): + def load_manifest(self, data): + if MARSHMALLOW_2: + data, errors = self.load(data) + if errors: + raise ManifestValidationError(errors, data, data) + return data + return self.load(data) + + +class StrictSchema(BaseSchema): + def handle_error(self, error, data, **kwargs): # pylint: disable=arguments-differ # skip broken records if self.many: - error.data = [ + error.valid_data = [ item for idx, item in enumerate(data) if idx not in error.messages ] else: - error.data = None + error.valid_data = None + if MARSHMALLOW_2: + error.data = error.valid_data raise error class StrictListField(fields.List): - def _deserialize(self, value, attr, data): + def _deserialize(self, value, attr, data, **kwargs): try: - return super(StrictListField, self)._deserialize(value, attr, data) + return super(StrictListField, self)._deserialize( + value, attr, data, **kwargs + ) except ValidationError as exc: if exc.data: exc.data = [item for item in exc.data if item is not None] @@ -61,7 +100,7 @@ class RepositorySchema(StrictSchema): branch = fields.Str(validate=validate.Length(min=1, max=50)) -class ExportSchema(Schema): +class ExportSchema(BaseSchema): include = StrictListField(fields.Str) exclude = StrictListField(fields.Str) @@ -80,7 +119,7 @@ class ExampleSchema(StrictSchema): files = StrictListField(fields.Str, required=True) -class ManifestSchema(Schema): +class ManifestSchema(BaseSchema): # Required fields name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) @@ -144,10 +183,6 @@ class ManifestSchema(Schema): ) ) - def handle_error(self, error, data): - if self.strict: - raise ManifestValidationError(error, data) - @validates("version") def validate_version(self, value): # pylint: disable=no-self-use try: @@ -178,7 +213,7 @@ class ManifestSchema(Schema): def load_spdx_licenses(): r = requests.get( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.6/json/licenses.json" + "/v3.7/json/licenses.json" ) r.raise_for_status() return r.json() diff --git a/platformio/util.py b/platformio/util.py index 26529dbb..04bb723e 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -368,8 +368,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): PING_INTERNET_IPS = [ "192.30.253.113", # github.com - "31.28.1.238", # dl.platformio.org - "193.222.52.25", # dl.platformio.org + "78.46.220.20", # dl.platformio.org ] diff --git a/setup.py b/setup.py index 64a83fbf..6413ffbd 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ "semantic_version>=2.8.1,<3", "tabulate>=0.8.3,<1", "pyelftools>=0.25,<1", - "marshmallow>=2.20.5,<3" + "marshmallow>=2.20.5", ] setup( diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 204ee07f..978ba1b4 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -238,8 +238,7 @@ def test_library_json_schema(): contents, parser.ManifestFileType.LIBRARY_JSON ).as_dict() - data, errors = ManifestSchema(strict=True).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git" assert data["examples"][1]["base"] == "examples/JsonHttpClient" @@ -297,8 +296,7 @@ architectures=avr,sam contents, parser.ManifestFileType.LIBRARY_PROPERTIES ).as_dict() - data, errors = ManifestSchema(strict=True).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, @@ -348,7 +346,12 @@ includes=MozziGuts.h ), ).as_dict() - data, errors = ManifestSchema(strict=False).load(raw_data) + try: + ManifestSchema().load_manifest(raw_data) + except ManifestValidationError as e: + data = e.valid_data + errors = e.messages + assert errors["authors"] assert not jsondiff.diff( @@ -437,8 +440,7 @@ def test_platform_json_schema(): contents, parser.ManifestFileType.PLATFORM_JSON ).as_dict() raw_data["frameworks"] = sorted(raw_data["frameworks"]) - data, errors = ManifestSchema(strict=False).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, @@ -477,8 +479,7 @@ def test_package_json_schema(): contents, parser.ManifestFileType.PACKAGE_JSON ).as_dict() - data, errors = ManifestSchema(strict=False).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, @@ -580,8 +581,7 @@ def test_examples_from_dir(tmpdir_factory): raw_data["examples"] = _sort_examples(raw_data["examples"]) - data, errors = ManifestSchema(strict=True).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, @@ -637,34 +637,32 @@ def test_examples_from_dir(tmpdir_factory): def test_broken_schemas(): - # non-strict mode - data, errors = ManifestSchema(strict=False).load(dict(name="MyPackage")) - assert set(errors.keys()) == set(["version"]) - assert data.get("version") is None - - # invalid keywords - data, errors = ManifestSchema(strict=False).load(dict(keywords=["kw1", "*^[]"])) - assert errors - assert data["keywords"] == ["kw1"] - - # strict mode - + # missing required field with pytest.raises( - ManifestValidationError, match="Missing data for required field" - ): - ManifestSchema(strict=True).load(dict(name="MyPackage")) + ManifestValidationError, match=("Invalid semantic versioning format") + ) as exc_info: + ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version")) + assert exc_info.value.valid_data == {"name": "MyPackage"} + + # invalid StrictList + with pytest.raises( + ManifestValidationError, match=("Invalid manifest fields.+keywords") + ) as exc_info: + ManifestSchema().load_manifest( + dict(name="MyPackage", version="1.0.0", keywords=["kw1", "*^[]"]) + ) + assert list(exc_info.value.messages.keys()) == ["keywords"] + assert exc_info.value.valid_data["keywords"] == ["kw1"] # broken SemVer with pytest.raises( ManifestValidationError, match=("Invalid semantic versioning format") ): - ManifestSchema(strict=True).load( - dict(name="MyPackage", version="broken_version") - ) + ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version")) # broken value for Nested with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"): - ManifestSchema(strict=True).load( + ManifestSchema().load_manifest( dict( name="MyPackage", description="MyDescription",