# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=too-many-ancestors import json import re import marshmallow import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates from platformio.clients.http import fetch_remote_content from platformio.package.exception import ManifestValidationError from platformio.util import memoized class BaseSchema(Schema): class Meta(object): # pylint: disable=no-init unknown = marshmallow.EXCLUDE # pylint: disable=no-member def load_manifest(self, data): return self.load(data) def handle_error(self, error, data, **_): # pylint: disable=arguments-differ raise ManifestValidationError( error.messages, data, error.valid_data if hasattr(error, "valid_data") else error.data, ) class StrictSchema(BaseSchema): def handle_error(self, error, data, **_): # pylint: disable=arguments-differ # skip broken records if self.many: error.valid_data = [ item for idx, item in enumerate(data) if idx not in error.messages ] else: error.valid_data = None raise error class StrictListField(fields.List): def _deserialize( # pylint: disable=arguments-differ self, value, attr, data, **kwargs ): try: return super()._deserialize(value, attr, data, **kwargs) except ValidationError as exc: if exc.data: exc.data = [item for item in exc.data if item is not None] raise exc class AuthorSchema(StrictSchema): name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(dump_default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) class RepositorySchema(StrictSchema): type = fields.Str( required=True, validate=validate.OneOf( ["git", "hg", "svn"], error="Invalid repository type, please use one of [git, hg, svn]", ), ) url = fields.Str(required=True, validate=validate.Length(min=1, max=255)) branch = fields.Str(validate=validate.Length(min=1, max=50)) class DependencySchema(StrictSchema): owner = fields.Str(validate=validate.Length(min=1, max=100)) name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) version = fields.Str(validate=validate.Length(min=1, max=100)) authors = StrictListField(fields.Str(validate=validate.Length(min=1, max=50))) platforms = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" ), ] ) ) frameworks = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" ), ] ) ) class ExportSchema(BaseSchema): include = StrictListField(fields.Str) exclude = StrictListField(fields.Str) class ExampleSchema(StrictSchema): name = fields.Str( required=True, validate=[ validate.Length(min=1, max=255), validate.Regexp( r"^[a-zA-Z\d\-\_/\. ]+$", error="Only [a-zA-Z0-9-_/. ] chars are allowed", ), ], ) base = fields.Str(required=True) files = StrictListField(fields.Str, required=True) # Fields class ScriptField(fields.Field): def _deserialize(self, value, attr, data, **kwargs): if isinstance(value, (str, list)): return value raise ValidationError( "Script value must be a command (string) or list of arguments" ) # Scheme class ManifestSchema(BaseSchema): # Required fields name = fields.Str( required=True, validate=[ validate.Length(min=1, max=100), validate.Regexp( r"^[^:;/,@\<\>]+$", error="The next chars [:;/,@<>] are not allowed" ), ], ) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) # Optional fields authors = fields.Nested(AuthorSchema, many=True) description = fields.Str(validate=validate.Length(min=1, max=1000)) homepage = fields.Url(validate=validate.Length(min=1, max=255)) license = fields.Str(validate=validate.Length(min=1, max=255)) repository = fields.Nested(RepositorySchema) dependencies = fields.Nested(DependencySchema, many=True) scripts = fields.Dict( keys=fields.Str(validate=validate.OneOf(["postinstall", "preuninstall"])), values=ScriptField(), ) # library.json export = fields.Nested(ExportSchema) examples = fields.Nested(ExampleSchema, many=True) downloadUrl = fields.Url(validate=validate.Length(min=1, max=255)) keywords = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed" ), ] ) ) platforms = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" ), ] ) ) frameworks = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" ), ] ) ) headers = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=255), ] ) ) # platform.json specific title = fields.Str(validate=validate.Length(min=1, max=100)) # package.json specific system = StrictListField( fields.Str( validate=[ validate.Length(min=1, max=50), validate.Regexp( r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed" ), ] ) ) @validates("version") def validate_version(self, value): # pylint: disable=no-self-use try: value = str(value) assert "." in value # check leading zeros try: semantic_version.Version(value) except ValueError as exc: if "Invalid leading zero" in str(exc): raise exc semantic_version.Version.coerce(value) except (AssertionError, ValueError): raise ValidationError( "Invalid semantic versioning format, see https://semver.org/" ) @validates("license") def validate_license(self, value): try: spdx = self.load_spdx_licenses() except requests.exceptions.RequestException: raise ValidationError("Could not load SPDX licenses for validation") known_ids = set(item.get("licenseId") for item in spdx.get("licenses", [])) if value in known_ids: return True # parse license expression # https://spdx.github.io/spdx-spec/SPDX-license-expressions/ package_ids = [ item.strip() for item in re.sub(r"(\s+(?:OR|AND|WITH)\s+|[\(\)])", " ", value).split(" ") if item.strip() ] if known_ids >= set(package_ids): return True raise ValidationError( "Invalid SPDX license identifier. See valid identifiers at " "https://spdx.org/licenses/" ) @staticmethod @memoized(expire="1h") def load_spdx_licenses(): version = "3.17" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version ) return json.loads(fetch_remote_content(spdx_data_url))