Made package ManifestSchema compatible with marshmallow >= 3 // Resolve #3296

HISTORY.rst

@@ -16,6 +16,7 @@ PlatformIO Core 4.0
 
 * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 <https://github.com/platformio/platformio-core/issues/2591>`_)
 * Updated SCons tool to 3.1.2
+* Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 <https://github.com/platformio/platformio-core/issues/3296>`_)
 * Warn about broken library manifest when scanning dependencies (`issue #3268 <https://github.com/platformio/platformio-core/issues/3268>`_)
 * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 <https://github.com/platformio/platformio-core/issues/3264>`_)
 * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 <https://github.com/platformio/platformio-core/issues/3282>`_)

platformio/commands/lib.py

@@ -26,7 +26,7 @@ from platformio.commands import PlatformioCLI
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
 from platformio.package.manifest.parser import ManifestParserFactory
-from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError
+from platformio.package.manifest.schema import ManifestSchema
 from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import get_project_dir, is_platformio_project
@@ -495,11 +495,9 @@ def lib_register(config_url):
         raise exception.InvalidLibConfURL(config_url)
 
     # Validate manifest
-    data, error = ManifestSchema(strict=False).load(
+    ManifestSchema().load_manifest(
         ManifestParserFactory.new_from_url(config_url).as_dict()
     )
-    if error:
-        raise ManifestValidationError(error, data)
 
     result = util.get_api_result("/lib/register", data=dict(config_url=config_url))
     if "message" in result and result["message"]:

platformio/package/exception.py

@@ -24,13 +24,14 @@ class ManifestParserError(ManifestException):
 
 
 class ManifestValidationError(ManifestException):
-    def __init__(self, error, data):
+    def __init__(self, messages, data, valid_data):
         super(ManifestValidationError, self).__init__()
-        self.error = error
+        self.messages = messages
         self.data = data
+        self.valid_data = valid_data
 
     def __str__(self):
         return (
             "Invalid manifest fields: %s. \nPlease check specification -> "
-            "http://docs.platformio.org/page/librarymanager/config.html" % self.error
+            "http://docs.platformio.org/page/librarymanager/config.html" % self.messages
         )

platformio/package/manifest/schema.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import marshmallow
 import requests
 import semantic_version
 from marshmallow import Schema, ValidationError, fields, validate, validates
@@ -19,23 +20,61 @@ from marshmallow import Schema, ValidationError, fields, validate, validates
 from platformio.package.exception import ManifestValidationError
 from platformio.util import memoized
 
+MARSHMALLOW_2 = marshmallow.__version_info__ < (3,)
 
-class StrictSchema(Schema):
-    def handle_error(self, error, data):
+
+if MARSHMALLOW_2:
+
+    class CompatSchema(Schema):
+        pass
+
+
+else:
+
+    class CompatSchema(Schema):
+        class Meta:
+            unknown = marshmallow.EXCLUDE
+
+        def handle_error(  # pylint: disable=arguments-differ
+            self, error, data, **kwargs
+        ):
+            raise ManifestValidationError(
+                error.messages,
+                data,
+                error.valid_data if hasattr(error, "valid_data") else error.data,
+            )
+
+
+class BaseSchema(CompatSchema):
+    def load_manifest(self, data):
+        if MARSHMALLOW_2:
+            data, errors = self.load(data)
+            if errors:
+                raise ManifestValidationError(errors, data, data)
+            return data
+        return self.load(data)
+
+
+class StrictSchema(BaseSchema):
+    def handle_error(self, error, data, **kwargs):  # pylint: disable=arguments-differ
         # skip broken records
         if self.many:
-            error.data = [
+            error.valid_data = [
                 item for idx, item in enumerate(data) if idx not in error.messages
             ]
         else:
-            error.data = None
+            error.valid_data = None
+        if MARSHMALLOW_2:
+            error.data = error.valid_data
         raise error
 
 
 class StrictListField(fields.List):
-    def _deserialize(self, value, attr, data):
+    def _deserialize(self, value, attr, data, **kwargs):
         try:
-            return super(StrictListField, self)._deserialize(value, attr, data)
+            return super(StrictListField, self)._deserialize(
+                value, attr, data, **kwargs
+            )
         except ValidationError as exc:
             if exc.data:
                 exc.data = [item for item in exc.data if item is not None]
@@ -61,7 +100,7 @@ class RepositorySchema(StrictSchema):
     branch = fields.Str(validate=validate.Length(min=1, max=50))
 
 
-class ExportSchema(Schema):
+class ExportSchema(BaseSchema):
     include = StrictListField(fields.Str)
     exclude = StrictListField(fields.Str)
 
@@ -80,7 +119,7 @@ class ExampleSchema(StrictSchema):
     files = StrictListField(fields.Str, required=True)
 
 
-class ManifestSchema(Schema):
+class ManifestSchema(BaseSchema):
     # Required fields
     name = fields.Str(required=True, validate=validate.Length(min=1, max=100))
     version = fields.Str(required=True, validate=validate.Length(min=1, max=50))
@@ -144,10 +183,6 @@ class ManifestSchema(Schema):
         )
     )
 
-    def handle_error(self, error, data):
-        if self.strict:
-            raise ManifestValidationError(error, data)
-
     @validates("version")
     def validate_version(self, value):  # pylint: disable=no-self-use
         try:
@@ -178,7 +213,7 @@ class ManifestSchema(Schema):
     def load_spdx_licenses():
         r = requests.get(
             "https://raw.githubusercontent.com/spdx/license-list-data"
-            "/v3.6/json/licenses.json"
+            "/v3.7/json/licenses.json"
         )
         r.raise_for_status()
         return r.json()

platformio/util.py

@@ -368,8 +368,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
 
 PING_INTERNET_IPS = [
     "192.30.253.113",  # github.com
-    "31.28.1.238",  # dl.platformio.org
-    "193.222.52.25",  # dl.platformio.org
+    "78.46.220.20",  # dl.platformio.org
 ]
 
 

setup.py

@@ -33,7 +33,7 @@ install_requires = [
     "semantic_version>=2.8.1,<3",
     "tabulate>=0.8.3,<1",
     "pyelftools>=0.25,<1",
-    "marshmallow>=2.20.5,<3"
+    "marshmallow>=2.20.5",
 ]
 
 setup(

tests/test_pkgmanifest.py

@@ -238,8 +238,7 @@ def test_library_json_schema():
         contents, parser.ManifestFileType.LIBRARY_JSON
     ).as_dict()
 
-    data, errors = ManifestSchema(strict=True).load(raw_data)
-    assert not errors
+    data = ManifestSchema().load_manifest(raw_data)
 
     assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git"
     assert data["examples"][1]["base"] == "examples/JsonHttpClient"
@@ -297,8 +296,7 @@ architectures=avr,sam
         contents, parser.ManifestFileType.LIBRARY_PROPERTIES
     ).as_dict()
 
-    data, errors = ManifestSchema(strict=True).load(raw_data)
-    assert not errors
+    data = ManifestSchema().load_manifest(raw_data)
 
     assert not jsondiff.diff(
         data,
@@ -348,7 +346,12 @@ includes=MozziGuts.h
         ),
     ).as_dict()
 
-    data, errors = ManifestSchema(strict=False).load(raw_data)
+    try:
+        ManifestSchema().load_manifest(raw_data)
+    except ManifestValidationError as e:
+        data = e.valid_data
+        errors = e.messages
+
     assert errors["authors"]
 
     assert not jsondiff.diff(
@@ -437,8 +440,7 @@ def test_platform_json_schema():
         contents, parser.ManifestFileType.PLATFORM_JSON
     ).as_dict()
     raw_data["frameworks"] = sorted(raw_data["frameworks"])
-    data, errors = ManifestSchema(strict=False).load(raw_data)
-    assert not errors
+    data = ManifestSchema().load_manifest(raw_data)
 
     assert not jsondiff.diff(
         data,
@@ -477,8 +479,7 @@ def test_package_json_schema():
         contents, parser.ManifestFileType.PACKAGE_JSON
     ).as_dict()
 
-    data, errors = ManifestSchema(strict=False).load(raw_data)
-    assert not errors
+    data = ManifestSchema().load_manifest(raw_data)
 
     assert not jsondiff.diff(
         data,
@@ -580,8 +581,7 @@ def test_examples_from_dir(tmpdir_factory):
 
     raw_data["examples"] = _sort_examples(raw_data["examples"])
 
-    data, errors = ManifestSchema(strict=True).load(raw_data)
-    assert not errors
+    data = ManifestSchema().load_manifest(raw_data)
 
     assert not jsondiff.diff(
         data,
@@ -637,34 +637,32 @@ def test_examples_from_dir(tmpdir_factory):
 
 
 def test_broken_schemas():
-    # non-strict mode
-    data, errors = ManifestSchema(strict=False).load(dict(name="MyPackage"))
-    assert set(errors.keys()) == set(["version"])
-    assert data.get("version") is None
-
-    # invalid keywords
-    data, errors = ManifestSchema(strict=False).load(dict(keywords=["kw1", "*^[]"]))
-    assert errors
-    assert data["keywords"] == ["kw1"]
-
-    # strict mode
-
+    # missing required field
     with pytest.raises(
-        ManifestValidationError, match="Missing data for required field"
-    ):
-        ManifestSchema(strict=True).load(dict(name="MyPackage"))
+        ManifestValidationError, match=("Invalid semantic versioning format")
+    ) as exc_info:
+        ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version"))
+    assert exc_info.value.valid_data == {"name": "MyPackage"}
+
+    # invalid StrictList
+    with pytest.raises(
+        ManifestValidationError, match=("Invalid manifest fields.+keywords")
+    ) as exc_info:
+        ManifestSchema().load_manifest(
+            dict(name="MyPackage", version="1.0.0", keywords=["kw1", "*^[]"])
+        )
+    assert list(exc_info.value.messages.keys()) == ["keywords"]
+    assert exc_info.value.valid_data["keywords"] == ["kw1"]
 
     # broken SemVer
     with pytest.raises(
         ManifestValidationError, match=("Invalid semantic versioning format")
     ):
-        ManifestSchema(strict=True).load(
-            dict(name="MyPackage", version="broken_version")
-        )
+        ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version"))
 
     # broken value for Nested
     with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"):
-        ManifestSchema(strict=True).load(
+        ManifestSchema().load_manifest(
             dict(
                 name="MyPackage",
                 description="MyDescription",