mirror of
https://github.com/platformio/platformio-core.git
synced 2025-07-29 17:47:14 +02:00
Introduce DataModel, package manifest parser and base manifest model
This commit is contained in:
Ivan Kravets
@ -322,7 +322,7 @@ def ConfigureDebugFlags(env):
|
||||
env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"])
|
||||
|
||||
debug_flags = ["-Og", "-g3", "-ggdb3"]
|
||||
for scope in ("ASFLAGS", "CCFLAGS",):
|
||||
for scope in ("ASFLAGS", "CCFLAGS"):
|
||||
_cleanup_debug_flags(scope)
|
||||
env.Append(**{scope: debug_flags})
|
||||
|
||||
|
||||
@ -14,8 +14,8 @@
|
||||
|
||||
# pylint: disable=too-many-branches, too-many-locals
|
||||
|
||||
import os
|
||||
import time
|
||||
from os.path import isdir, join
|
||||
|
||||
import click
|
||||
import semantic_version
|
||||
@ -25,6 +25,7 @@ from platformio import exception, util
|
||||
from platformio.commands import PlatformioCLI
|
||||
from platformio.compat import dump_json_to_unicode
|
||||
from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
|
||||
from platformio.package.manifest.parser import ManifestFactory
|
||||
from platformio.proc import is_ci
|
||||
from platformio.project.config import ProjectConfig
|
||||
from platformio.project.helpers import get_project_dir, is_platformio_project
|
||||
@ -104,13 +105,13 @@ def cli(ctx, **options):
|
||||
if not is_platformio_project(storage_dir):
|
||||
ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
|
||||
continue
|
||||
config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini"))
|
||||
config = ProjectConfig.get_instance(os.path.join(storage_dir, "platformio.ini"))
|
||||
config.validate(options["environment"], silent=in_silence)
|
||||
libdeps_dir = config.get_optional_dir("libdeps")
|
||||
for env in config.envs():
|
||||
if options["environment"] and env not in options["environment"]:
|
||||
continue
|
||||
storage_dir = join(libdeps_dir, env)
|
||||
storage_dir = os.path.join(libdeps_dir, env)
|
||||
ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
|
||||
ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get(
|
||||
"env:" + env, "lib_deps", []
|
||||
@ -169,7 +170,7 @@ def lib_install( # pylint: disable=too-many-arguments
|
||||
input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
|
||||
project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
|
||||
for input_dir in input_dirs:
|
||||
config = ProjectConfig.get_instance(join(input_dir, "platformio.ini"))
|
||||
config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini"))
|
||||
config.validate(project_environments)
|
||||
for env in config.envs():
|
||||
if project_environments and env not in project_environments:
|
||||
@ -231,7 +232,7 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output):
|
||||
if only_check and json_output:
|
||||
result = []
|
||||
for library in _libraries:
|
||||
pkg_dir = library if isdir(library) else None
|
||||
pkg_dir = library if os.path.isdir(library) else None
|
||||
requirements = None
|
||||
url = None
|
||||
if not pkg_dir:
|
||||
@ -492,6 +493,9 @@ def lib_register(config_url):
|
||||
if not config_url.startswith("http://") and not config_url.startswith("https://"):
|
||||
raise exception.InvalidLibConfURL(config_url)
|
||||
|
||||
manifest = ManifestFactory.new_from_url(config_url)
|
||||
assert set(["name", "version"]) & set(list(manifest.as_dict()))
|
||||
|
||||
result = util.get_api_result("/lib/register", data=dict(config_url=config_url))
|
||||
if "message" in result and result["message"]:
|
||||
click.secho(
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
# pylint: disable=unused-import, no-name-in-module, import-error,
|
||||
# pylint: disable=no-member, undefined-variable
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@ -29,6 +30,15 @@ def get_filesystem_encoding():
|
||||
return sys.getfilesystemencoding() or sys.getdefaultencoding()
|
||||
|
||||
|
||||
def get_class_attributes(cls):
|
||||
attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a)))
|
||||
return {
|
||||
a[0]: a[1]
|
||||
for a in attributes
|
||||
if not (a[0].startswith("__") and a[0].endswith("__"))
|
||||
}
|
||||
|
||||
|
||||
if PY2:
|
||||
import imp
|
||||
|
||||
|
||||
154
platformio/datamodel.py
Normal file
154
platformio/datamodel.py
Normal file
@ -0,0 +1,154 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import inspect
|
||||
import re
|
||||
|
||||
from platformio.compat import get_class_attributes, string_types
|
||||
from platformio.exception import PlatformioException
|
||||
|
||||
# pylint: disable=too-many-instance-attributes
|
||||
# pylint: disable=redefined-builtin, too-many-arguments
|
||||
|
||||
|
||||
class DataModelException(PlatformioException):
|
||||
pass
|
||||
|
||||
|
||||
class DataField(object):
|
||||
def __init__(
|
||||
self,
|
||||
default=None,
|
||||
type=str,
|
||||
required=False,
|
||||
min_length=None,
|
||||
max_length=None,
|
||||
regex=None,
|
||||
validate_factory=None,
|
||||
title=None,
|
||||
):
|
||||
self.default = default
|
||||
self.type = type
|
||||
self.required = required
|
||||
self.min_length = min_length
|
||||
self.max_length = max_length
|
||||
self.regex = regex
|
||||
self.validate_factory = validate_factory
|
||||
self.title = title
|
||||
|
||||
self._value = None
|
||||
|
||||
def __repr__(self):
|
||||
return '<DataField %s="%s">' % (
|
||||
self.title,
|
||||
self.default if self._value is None else self._value,
|
||||
)
|
||||
|
||||
def validate(self, value, parent, attr):
|
||||
if self.title is None:
|
||||
self.title = attr.title()
|
||||
try:
|
||||
if self.required and value is None:
|
||||
raise ValueError("Required field, value is None")
|
||||
if self.validate_factory is not None:
|
||||
value = self.validate_factory(value)
|
||||
if value is None:
|
||||
return self.default
|
||||
if issubclass(self.type, (str, list, bool)):
|
||||
return getattr(self, "_validate_%s_value" % self.type.__name__)(value)
|
||||
except (AssertionError, ValueError) as e:
|
||||
raise DataModelException(
|
||||
"%s for %s.%s" % (str(e), parent.__class__.__name__, attr)
|
||||
)
|
||||
return value
|
||||
|
||||
def _validate_str_value(self, value):
|
||||
if not isinstance(value, string_types):
|
||||
value = str(value)
|
||||
assert self.min_length is None or len(value) >= self.min_length, (
|
||||
"Minimum allowed length is %d characters" % self.min_length
|
||||
)
|
||||
assert self.max_length is None or len(value) <= self.max_length, (
|
||||
"Maximum allowed length is %d characters" % self.max_length
|
||||
)
|
||||
assert self.regex is None or re.match(
|
||||
self.regex, value
|
||||
), "Value `%s` does not match RegExp `%s` pattern" % (value, self.regex)
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _validate_bool_value(value):
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value).lower() in ("true", "yes", "1")
|
||||
|
||||
|
||||
class DataModel(object):
|
||||
__PRIVATE_ATTRIBUTES__ = ("__PRIVATE_ATTRIBUTES__", "_init_type", "as_dict")
|
||||
|
||||
def __init__(self, data=None):
|
||||
data = data or {}
|
||||
assert isinstance(data, dict)
|
||||
|
||||
for attr, scheme_or_model in get_class_attributes(self).items():
|
||||
if attr in self.__PRIVATE_ATTRIBUTES__:
|
||||
continue
|
||||
if isinstance(scheme_or_model, list):
|
||||
assert len(scheme_or_model) == 1
|
||||
if data.get(attr) is None:
|
||||
setattr(self, attr, None)
|
||||
continue
|
||||
|
||||
if not isinstance(data.get(attr), list):
|
||||
raise DataModelException("Value should be a list for %s" % (attr))
|
||||
setattr(
|
||||
self,
|
||||
attr,
|
||||
[
|
||||
self._init_type(scheme_or_model[0], v, attr)
|
||||
for v in data.get(attr)
|
||||
],
|
||||
)
|
||||
else:
|
||||
setattr(
|
||||
self, attr, self._init_type(scheme_or_model, data.get(attr), attr)
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
attrs = []
|
||||
for name, value in get_class_attributes(self).items():
|
||||
if name in self.__PRIVATE_ATTRIBUTES__:
|
||||
continue
|
||||
attrs.append('%s="%s"' % (name, value))
|
||||
return "<%s %s>" % (self.__class__.__name__, " ".join(attrs))
|
||||
|
||||
def _init_type(self, type_, value, attr):
|
||||
if inspect.isclass(type_) and issubclass(type_, DataModel):
|
||||
return type_(value)
|
||||
if isinstance(type_, DataField):
|
||||
return type_.validate(value, parent=self, attr=attr)
|
||||
raise DataModelException("Undeclared or unknown data type for %s" % attr)
|
||||
|
||||
def as_dict(self):
|
||||
result = {}
|
||||
for name, value in get_class_attributes(self).items():
|
||||
if name in self.__PRIVATE_ATTRIBUTES__:
|
||||
continue
|
||||
if isinstance(value, DataModel):
|
||||
result[name] = value.as_dict()
|
||||
elif value and isinstance(value, list) and isinstance(value[0], DataModel):
|
||||
result[name] = value[0].as_dict()
|
||||
else:
|
||||
result[name] = value
|
||||
return result
|
||||
13
platformio/package/__init__.py
Normal file
13
platformio/package/__init__.py
Normal file
@ -0,0 +1,13 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
13
platformio/package/manifest/__init__.py
Normal file
13
platformio/package/manifest/__init__.py
Normal file
@ -0,0 +1,13 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
57
platformio/package/manifest/model.py
Normal file
57
platformio/package/manifest/model.py
Normal file
@ -0,0 +1,57 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import semantic_version
|
||||
|
||||
from platformio.datamodel import DataField, DataModel
|
||||
|
||||
|
||||
class AuthorModel(DataModel):
|
||||
name = DataField(max_length=50, required=True)
|
||||
email = DataField(max_length=50)
|
||||
maintainer = DataField(default=False, type=bool)
|
||||
url = DataField(max_length=255)
|
||||
|
||||
|
||||
class RepositoryModel(DataModel):
|
||||
type = DataField(max_length=3, required=True)
|
||||
url = DataField(max_length=255, required=True)
|
||||
branch = DataField(max_length=50)
|
||||
|
||||
|
||||
class ExportModel(DataModel):
|
||||
include = [DataField()]
|
||||
exclude = [DataField()]
|
||||
|
||||
|
||||
class ManifestModel(DataModel):
|
||||
|
||||
name = DataField(max_length=100, required=True)
|
||||
version = DataField(
|
||||
required=True,
|
||||
max_length=50,
|
||||
validate_factory=lambda v: v if semantic_version.Version.coerce(v) else None,
|
||||
)
|
||||
|
||||
description = DataField(max_length=1000)
|
||||
keywords = [DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")]
|
||||
authors = [AuthorModel]
|
||||
|
||||
homepage = DataField(max_length=255)
|
||||
license = DataField(max_length=255)
|
||||
platforms = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")]
|
||||
frameworks = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")]
|
||||
|
||||
repository = RepositoryModel
|
||||
export = ExportModel
|
||||
369
platformio/package/manifest/parser.py
Normal file
369
platformio/package/manifest/parser.py
Normal file
@ -0,0 +1,369 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
import requests
|
||||
|
||||
from platformio.compat import get_class_attributes, string_types
|
||||
from platformio.exception import PlatformioException
|
||||
from platformio.fs import get_file_contents
|
||||
from platformio.package.manifest.model import ManifestModel
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
except ImportError:
|
||||
from urlparse import urlparse
|
||||
|
||||
|
||||
class ManifestException(PlatformioException):
|
||||
pass
|
||||
|
||||
|
||||
class ManifestParserException(ManifestException):
|
||||
pass
|
||||
|
||||
|
||||
class ManifestFileType(object):
|
||||
PLATFORM_JSON = "platform.json"
|
||||
LIBRARY_JSON = "library.json"
|
||||
LIBRARY_PROPERTIES = "library.properties"
|
||||
MODULE_JSON = "module.json"
|
||||
PACKAGE_JSON = "package.json"
|
||||
|
||||
@classmethod
|
||||
def from_uri(cls, uri):
|
||||
if uri.endswith(".properties"):
|
||||
return ManifestFileType.LIBRARY_PROPERTIES
|
||||
if uri.endswith("platform.json"):
|
||||
return ManifestFileType.PLATFORM_JSON
|
||||
if uri.endswith("module.json"):
|
||||
return ManifestFileType.MODULE_JSON
|
||||
if uri.endswith("package.json"):
|
||||
return ManifestFileType.PACKAGE_JSON
|
||||
return ManifestFileType.LIBRARY_JSON
|
||||
|
||||
|
||||
class ManifestFactory(object):
|
||||
@staticmethod
|
||||
def type_to_clsname(type_):
|
||||
type_ = type_.replace(".", " ")
|
||||
type_ = type_.title()
|
||||
return "%sManifestParser" % type_.replace(" ", "")
|
||||
|
||||
@staticmethod
|
||||
def new_from_file(path):
|
||||
if not path or not os.path.isfile(path):
|
||||
raise ManifestException("Manifest file does not exist %s" % path)
|
||||
for type_ in get_class_attributes(ManifestFileType).values():
|
||||
if path.endswith(type_):
|
||||
return ManifestFactory.new(get_file_contents(path), type_)
|
||||
raise ManifestException("Unknown manifest file type %s" % path)
|
||||
|
||||
@staticmethod
|
||||
def new_from_url(remote_url):
|
||||
r = requests.get(remote_url)
|
||||
r.raise_for_status()
|
||||
return ManifestFactory.new(
|
||||
r.text, ManifestFileType.from_uri(remote_url), remote_url
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def new(contents, type_, remote_url=None):
|
||||
clsname = ManifestFactory.type_to_clsname(type_)
|
||||
if clsname not in globals():
|
||||
raise ManifestException("Unknown manifest file type %s" % clsname)
|
||||
mp = globals()[clsname](contents, remote_url)
|
||||
return ManifestModel(mp.as_dict())
|
||||
|
||||
|
||||
class BaseManifestParser(object):
|
||||
def __init__(self, contents, remote_url=None):
|
||||
self.remote_url = remote_url
|
||||
self._data = self.parse(contents)
|
||||
|
||||
def parse(self, contents):
|
||||
raise NotImplementedError
|
||||
|
||||
def as_dict(self):
|
||||
return self._data
|
||||
|
||||
@staticmethod
|
||||
def _cleanup_author(author):
|
||||
if author.get("email"):
|
||||
author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"])
|
||||
return author
|
||||
|
||||
@staticmethod
|
||||
def parse_author_name_and_email(raw):
|
||||
if raw == "None" or "://" in raw:
|
||||
return (None, None)
|
||||
name = raw
|
||||
email = None
|
||||
for ldel, rdel in [("<", ">"), ("(", ")")]:
|
||||
if ldel in raw and rdel in raw:
|
||||
name = raw[: raw.index(ldel)]
|
||||
email = raw[raw.index(ldel) + 1 : raw.index(rdel)]
|
||||
return (name.strip(), email.strip() if email else None)
|
||||
|
||||
|
||||
class LibraryJsonManifestParser(BaseManifestParser):
|
||||
def parse(self, contents):
|
||||
data = json.loads(contents)
|
||||
data = self._process_renamed_fields(data)
|
||||
|
||||
# normalize Union[str, list] fields
|
||||
for k in ("keywords", "platforms", "frameworks"):
|
||||
if k in data:
|
||||
data[k] = self._str_to_list(data[k], sep=",")
|
||||
|
||||
if "authors" in data:
|
||||
data["authors"] = self._parse_authors(data["authors"])
|
||||
if "platforms" in data:
|
||||
data["platforms"] = self._parse_platforms(data["platforms"]) or None
|
||||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _str_to_list(value, sep=",", lowercase=True):
|
||||
if isinstance(value, string_types):
|
||||
value = value.split(sep)
|
||||
assert isinstance(value, list)
|
||||
result = []
|
||||
for item in value:
|
||||
item = item.strip()
|
||||
if not item:
|
||||
continue
|
||||
if lowercase:
|
||||
item = item.lower()
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _process_renamed_fields(data):
|
||||
if "url" in data:
|
||||
data["homepage"] = data["url"]
|
||||
del data["url"]
|
||||
|
||||
for key in ("include", "exclude"):
|
||||
if key not in data:
|
||||
continue
|
||||
if "export" not in data:
|
||||
data["export"] = {}
|
||||
data["export"][key] = (
|
||||
data[key] if isinstance(data[key], list) else [data[key]]
|
||||
)
|
||||
del data[key]
|
||||
|
||||
return data
|
||||
|
||||
def _parse_authors(self, raw):
|
||||
if not raw:
|
||||
return None
|
||||
# normalize Union[dict, list] fields
|
||||
if not isinstance(raw, list):
|
||||
raw = [raw]
|
||||
return [self._cleanup_author(author) for author in raw]
|
||||
|
||||
@staticmethod
|
||||
def _parse_platforms(raw):
|
||||
assert isinstance(raw, list)
|
||||
result = []
|
||||
# renamed platforms
|
||||
for item in raw:
|
||||
if item == "espressif":
|
||||
item = "espressif8266"
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
class ModuleJsonManifestParser(BaseManifestParser):
|
||||
def parse(self, contents):
|
||||
data = json.loads(contents)
|
||||
return dict(
|
||||
name=data["name"],
|
||||
version=data["version"],
|
||||
keywords=data.get("keywords"),
|
||||
description=data["description"],
|
||||
frameworks=["mbed"],
|
||||
platforms=["*"],
|
||||
homepage=data.get("homepage"),
|
||||
export={"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]},
|
||||
authors=self._parse_authors(data.get("author")),
|
||||
license=self._parse_license(data.get("licenses")),
|
||||
)
|
||||
|
||||
def _parse_authors(self, raw):
|
||||
if not raw:
|
||||
return None
|
||||
result = []
|
||||
for author in raw.split(","):
|
||||
name, email = self.parse_author_name_and_email(author)
|
||||
if not name:
|
||||
continue
|
||||
result.append(
|
||||
self._cleanup_author(dict(name=name, email=email, maintainer=False))
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _parse_license(raw):
|
||||
if not raw or not isinstance(raw, list):
|
||||
return None
|
||||
return raw[0].get("type")
|
||||
|
||||
|
||||
class LibraryPropertiesManifestParser(BaseManifestParser):
|
||||
def parse(self, contents):
|
||||
properties = self._parse_properties(contents)
|
||||
repository = self._parse_repository(properties)
|
||||
homepage = properties.get("url")
|
||||
if repository and repository["url"] == homepage:
|
||||
homepage = None
|
||||
return dict(
|
||||
name=properties["name"],
|
||||
version=properties["version"],
|
||||
description=properties["sentence"],
|
||||
frameworks=["arduino"],
|
||||
platforms=self._process_platforms(properties) or ["*"],
|
||||
keywords=self._parse_keywords(properties),
|
||||
authors=self._parse_authors(properties) or None,
|
||||
homepage=homepage,
|
||||
repository=repository or None,
|
||||
export=self._parse_export(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_properties(contents):
|
||||
data = {}
|
||||
for line in contents.splitlines():
|
||||
line = line.strip()
|
||||
if not line or "=" not in line:
|
||||
continue
|
||||
# skip comments
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
data[key.strip()] = value.strip()
|
||||
|
||||
required_fields = set(["name", "version", "author", "sentence"])
|
||||
if not set(data.keys()) >= required_fields:
|
||||
raise ManifestParserException(
|
||||
"Missing fields: " + ",".join(required_fields - set(data.keys()))
|
||||
)
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _parse_keywords(properties):
|
||||
result = []
|
||||
for item in re.split(r"[\s/]+", properties.get("category", "uncategorized")):
|
||||
item = item.strip()
|
||||
if not item:
|
||||
continue
|
||||
result.append(item.lower())
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _process_platforms(properties):
|
||||
result = []
|
||||
platforms_map = {
|
||||
"avr": "atmelavr",
|
||||
"sam": "atmelsam",
|
||||
"samd": "atmelsam",
|
||||
"esp8266": "espressif8266",
|
||||
"esp32": "espressif32",
|
||||
"arc32": "intel_arc32",
|
||||
"stm32": "ststm32",
|
||||
}
|
||||
for arch in properties.get("architectures", "").split(","):
|
||||
if "particle-" in arch:
|
||||
raise ManifestParserException("Particle is not supported yet")
|
||||
arch = arch.strip()
|
||||
if not arch:
|
||||
continue
|
||||
if arch == "*":
|
||||
return ["*"]
|
||||
if arch in platforms_map:
|
||||
result.append(platforms_map[arch])
|
||||
return result
|
||||
|
||||
def _parse_authors(self, properties):
|
||||
authors = []
|
||||
for author in properties["author"].split(","):
|
||||
name, email = self.parse_author_name_and_email(author)
|
||||
if not name:
|
||||
continue
|
||||
authors.append(
|
||||
self._cleanup_author(dict(name=name, email=email, maintainer=False))
|
||||
)
|
||||
for author in properties.get("maintainer", "").split(","):
|
||||
name, email = self.parse_author_name_and_email(author)
|
||||
if not name:
|
||||
continue
|
||||
found = False
|
||||
for item in authors:
|
||||
if item["name"].lower() != name.lower():
|
||||
continue
|
||||
found = True
|
||||
item["maintainer"] = True
|
||||
if not item["email"]:
|
||||
item["email"] = email
|
||||
if not found:
|
||||
authors.append(
|
||||
self._cleanup_author(dict(name=name, email=email, maintainer=True))
|
||||
)
|
||||
return authors
|
||||
|
||||
def _parse_repository(self, properties):
|
||||
if self.remote_url:
|
||||
repo_parse = urlparse(self.remote_url)
|
||||
repo_path_tokens = repo_parse.path[1:].split("/")[:-1]
|
||||
if "github" in repo_parse.netloc:
|
||||
return dict(
|
||||
type="git",
|
||||
url="%s://github.com/%s"
|
||||
% (repo_parse.scheme, "/".join(repo_path_tokens[:2])),
|
||||
)
|
||||
if "raw" in repo_path_tokens:
|
||||
return dict(
|
||||
type="git",
|
||||
url="%s://%s/%s"
|
||||
% (
|
||||
repo_parse.scheme,
|
||||
repo_parse.netloc,
|
||||
"/".join(repo_path_tokens[: repo_path_tokens.index("raw")]),
|
||||
),
|
||||
)
|
||||
if properties.get("url", "").startswith("https://github.com"):
|
||||
return dict(type="git", url=properties["url"])
|
||||
return None
|
||||
|
||||
def _parse_export(self):
|
||||
include = None
|
||||
if self.remote_url:
|
||||
repo_parse = urlparse(self.remote_url)
|
||||
repo_path_tokens = repo_parse.path[1:].split("/")[:-1]
|
||||
if "github" in repo_parse.netloc:
|
||||
include = "/".join(repo_path_tokens[3:]) or None
|
||||
elif "raw" in repo_path_tokens:
|
||||
include = (
|
||||
"/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :])
|
||||
or None
|
||||
)
|
||||
return {
|
||||
"include": include,
|
||||
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
|
||||
}
|
||||
@ -13,32 +13,196 @@
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from platformio.package.manifest import parser
|
||||
|
||||
|
||||
def validate_response(r):
|
||||
assert r.status_code == 200, r.url
|
||||
assert int(r.headers["Content-Length"]) > 0, r.url
|
||||
assert r.headers["Content-Type"] in ("application/gzip", "application/octet-stream")
|
||||
def test_library_json_parser():
|
||||
contents = """
|
||||
{
|
||||
"name": "TestPackage",
|
||||
"keywords": "kw1, KW2, kw3",
|
||||
"platforms": ["atmelavr", "espressif"],
|
||||
"url": "http://old.url.format",
|
||||
"exclude": [".gitignore", "tests"],
|
||||
"include": "mylib"
|
||||
}
|
||||
"""
|
||||
mp = parser.LibraryJsonManifestParser(contents)
|
||||
assert sorted(mp.as_dict().items()) == sorted(
|
||||
{
|
||||
"name": "TestPackage",
|
||||
"platforms": ["atmelavr", "espressif8266"],
|
||||
"export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]},
|
||||
"keywords": ["kw1", "kw2", "kw3"],
|
||||
"homepage": "http://old.url.format",
|
||||
}.items()
|
||||
)
|
||||
|
||||
|
||||
def test_packages():
|
||||
pkgs_manifest = requests.get(
|
||||
"https://dl.bintray.com/platformio/dl-packages/manifest.json"
|
||||
).json()
|
||||
assert isinstance(pkgs_manifest, dict)
|
||||
items = []
|
||||
for _, variants in pkgs_manifest.items():
|
||||
for item in variants:
|
||||
items.append(item)
|
||||
def test_module_json_parser():
|
||||
contents = """
|
||||
{
|
||||
"author": "Name Surname <name@surname.com>",
|
||||
"description": "This is Yotta library",
|
||||
"homepage": "https://yottabuild.org",
|
||||
"keywords": [
|
||||
"mbed",
|
||||
"Yotta"
|
||||
],
|
||||
"licenses": [
|
||||
{
|
||||
"type": "Apache-2.0",
|
||||
"url": "https://spdx.org/licenses/Apache-2.0"
|
||||
}
|
||||
],
|
||||
"name": "YottaLibrary",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git@github.com:username/repo.git"
|
||||
},
|
||||
"version": "1.2.3"
|
||||
}
|
||||
"""
|
||||
mp = parser.ModuleJsonManifestParser(contents)
|
||||
assert sorted(mp.as_dict().items()) == sorted(
|
||||
{
|
||||
"name": "YottaLibrary",
|
||||
"description": "This is Yotta library",
|
||||
"homepage": "https://yottabuild.org",
|
||||
"keywords": ["mbed", "Yotta"],
|
||||
"license": "Apache-2.0",
|
||||
"platforms": ["*"],
|
||||
"frameworks": ["mbed"],
|
||||
"export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]},
|
||||
"authors": [
|
||||
{
|
||||
"maintainer": False,
|
||||
"email": "name@surname.com",
|
||||
"name": "Name Surname",
|
||||
}
|
||||
],
|
||||
"version": "1.2.3",
|
||||
}.items()
|
||||
)
|
||||
|
||||
for item in items:
|
||||
assert item["url"].endswith(".tar.gz"), item
|
||||
|
||||
r = requests.head(item["url"], allow_redirects=True)
|
||||
validate_response(r)
|
||||
def test_library_properties_parser():
|
||||
# test missed fields
|
||||
with pytest.raises(parser.ManifestParserException):
|
||||
parser.LibraryPropertiesManifestParser("name=TestPackage")
|
||||
|
||||
if "X-Checksum-Sha1" not in r.headers:
|
||||
return pytest.skip("X-Checksum-Sha1 is not provided")
|
||||
# Base
|
||||
contents = """
|
||||
name=TestPackage
|
||||
version=1.2.3
|
||||
author=SomeAuthor <info AT author.com>
|
||||
sentence=This is Arduino library
|
||||
"""
|
||||
mp = parser.LibraryPropertiesManifestParser(contents)
|
||||
assert sorted(mp.as_dict().items()) == sorted(
|
||||
{
|
||||
"name": "TestPackage",
|
||||
"version": "1.2.3",
|
||||
"description": "This is Arduino library",
|
||||
"repository": None,
|
||||
"platforms": ["*"],
|
||||
"frameworks": ["arduino"],
|
||||
"export": {
|
||||
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
|
||||
"include": None,
|
||||
},
|
||||
"authors": [
|
||||
{"maintainer": False, "email": "info@author.com", "name": "SomeAuthor"}
|
||||
],
|
||||
"keywords": ["uncategorized"],
|
||||
"homepage": None,
|
||||
}.items()
|
||||
)
|
||||
|
||||
assert item["sha1"] == r.headers.get("X-Checksum-Sha1")[0:40], item
|
||||
# Platforms ALL
|
||||
mp = parser.LibraryPropertiesManifestParser("architectures=*\n" + contents)
|
||||
assert mp.as_dict()["platforms"] == ["*"]
|
||||
# Platforms specific
|
||||
mp = parser.LibraryPropertiesManifestParser("architectures=avr, esp32\n" + contents)
|
||||
assert mp.as_dict()["platforms"] == ["atmelavr", "espressif32"]
|
||||
|
||||
# Remote URL
|
||||
mp = parser.LibraryPropertiesManifestParser(
|
||||
contents,
|
||||
remote_url=(
|
||||
"https://raw.githubusercontent.com/username/reponame/master/"
|
||||
"libraries/TestPackage/library.properties"
|
||||
),
|
||||
)
|
||||
assert mp.as_dict()["export"] == {
|
||||
"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
|
||||
"include": "libraries/TestPackage",
|
||||
}
|
||||
|
||||
# Hope page
|
||||
mp = parser.LibraryPropertiesManifestParser(
|
||||
"url=https://github.com/username/reponame.git\n" + contents
|
||||
)
|
||||
assert mp.as_dict()["homepage"] is None
|
||||
assert mp.as_dict()["repository"] == {
|
||||
"type": "git",
|
||||
"url": "https://github.com/username/reponame.git",
|
||||
}
|
||||
|
||||
|
||||
def test_library_json_model():
|
||||
contents = """
|
||||
{
|
||||
"name": "ArduinoJson",
|
||||
"keywords": "JSON, rest, http, web",
|
||||
"description": "An elegant and efficient JSON library for embedded systems",
|
||||
"homepage": "https://arduinojson.org",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/bblanchon/ArduinoJson.git"
|
||||
},
|
||||
"version": "6.12.0",
|
||||
"authors": {
|
||||
"name": "Benoit Blanchon",
|
||||
"url": "https://blog.benoitblanchon.fr"
|
||||
},
|
||||
"exclude": [
|
||||
"fuzzing",
|
||||
"scripts",
|
||||
"test",
|
||||
"third-party"
|
||||
],
|
||||
"frameworks": "arduino",
|
||||
"platforms": "*",
|
||||
"license": "MIT"
|
||||
}
|
||||
"""
|
||||
model = parser.ManifestFactory.new(contents, parser.ManifestFileType.LIBRARY_JSON)
|
||||
assert sorted(model.as_dict().items()) == sorted(
|
||||
{
|
||||
"name": "ArduinoJson",
|
||||
"keywords": ["json", "rest", "http", "web"],
|
||||
"description": "An elegant and efficient JSON library for embedded systems",
|
||||
"homepage": "https://arduinojson.org",
|
||||
"repository": {
|
||||
"url": "https://github.com/bblanchon/ArduinoJson.git",
|
||||
"type": "git",
|
||||
"branch": None,
|
||||
},
|
||||
"version": "6.12.0",
|
||||
"authors": {
|
||||
"url": "https://blog.benoitblanchon.fr",
|
||||
"maintainer": False,
|
||||
"email": None,
|
||||
"name": "Benoit Blanchon",
|
||||
},
|
||||
"export": {
|
||||
"include": None,
|
||||
"exclude": ["fuzzing", "scripts", "test", "third-party"],
|
||||
},
|
||||
"frameworks": ["arduino"],
|
||||
"platforms": ["*"],
|
||||
"license": "MIT",
|
||||
}.items()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user