From efe8e599fdaa020f451df5ba96c3068893cbc3f1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Feb 2020 00:04:16 +0200 Subject: [PATCH] Added support for Arduino's library.properties ``depends`` field // Resolve #2781 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piolib.py | 4 +- platformio/managers/lib.py | 27 +-------- platformio/package/manifest/parser.py | 53 ++++++++++++++++ platformio/package/manifest/schema.py | 27 +++++++++ platformio/util.py | 7 +-- tests/package/test_manifest.py | 87 +++++++++++++++++++++++++-- 8 files changed, 169 insertions(+), 39 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9cdbec47..dcf46aa0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,7 @@ PlatformIO Core 4.0 * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) * Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) +* Added support for Arduino's library.properties ``depends`` field (`issue #2781 `_) * Updated SCons tool to 3.1.2 * Updated Unity tool to 2.5.0 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) diff --git a/docs b/docs index 1712ba74..a5c3fb32 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1712ba74a29534eed900e7570015cd22c24e0628 +Subproject commit a5c3fb32b7d89ef87320476769f206572b4b95ef diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 58d965e0..3c8746be 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -154,9 +154,7 @@ class LibBuilderBase(object): @property def dependencies(self): - return LibraryManager.normalize_dependencies( - self._manifest.get("dependencies", []) - ) + return self._manifest.get("dependencies") @property def src_filter(self): diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index e85a1225..b006d8af 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -23,7 +23,7 @@ import click import semantic_version from platformio import app, exception, util -from platformio.compat import glob_escape, string_types +from platformio.compat import glob_escape from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.project.config import ProjectConfig @@ -61,29 +61,6 @@ class LibraryManager(BasePkgManager): return None - @staticmethod - def normalize_dependencies(dependencies): - if not dependencies: - return [] - items = [] - if isinstance(dependencies, dict): - if "name" in dependencies: - items.append(dependencies) - else: - for name, version in dependencies.items(): - items.append({"name": name, "version": version}) - elif isinstance(dependencies, list): - items = [d for d in dependencies if "name" in d] - for item in items: - for k in ("frameworks", "platforms"): - if k not in item or isinstance(k, list): - continue - if item[k] == "*": - del item[k] - elif isinstance(item[k], string_types): - item[k] = [i.strip() for i in item[k].split(",") if i.strip()] - return items - def max_satisfying_repo_version(self, versions, requirements=None): def _cmp_dates(datestr1, datestr2): date1 = util.parse_date(datestr1) @@ -312,7 +289,7 @@ class LibraryManager(BasePkgManager): click.secho("Installing dependencies", fg="yellow") builtin_lib_storages = None - for filters in self.normalize_dependencies(manifest["dependencies"]): + for filters in manifest["dependencies"]: assert "name" in filters # avoid circle dependencies diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 64d52cbe..2391a84d 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -19,6 +19,7 @@ import re import requests +from platformio import util from platformio.compat import get_class_attributes, string_types from platformio.fs import get_file_contents from platformio.package.exception import ManifestParserError, UnknownManifestError @@ -286,6 +287,8 @@ class LibraryJsonManifestParser(BaseManifestParser): data["platforms"] = self._parse_platforms(data["platforms"]) or None if "export" in data: data["export"] = self._parse_export(data["export"]) + if "dependencies" in data: + data["dependencies"] = self._parse_dependencies(data["dependencies"]) return data @@ -350,6 +353,26 @@ class LibraryJsonManifestParser(BaseManifestParser): result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]] return result + @staticmethod + def _parse_dependencies(raw): + if isinstance(raw, dict): + if "name" in raw: # compatibility with dep as dict + return [raw] + return [dict(name=name, version=version) for name, version in raw.items()] + if isinstance(raw, list): + for i, dependency in enumerate(raw): + assert isinstance(dependency, dict) + for k, v in dependency.items(): + if k not in ("platforms", "frameworks", "authors"): + continue + if "*" in v: + del raw[i][k] + raw[i][k] = util.items_to_list(v) + return raw + raise ManifestParserError( + "Invalid dependencies format, should be list or dictionary" + ) + class ModuleJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.MODULE_JSON @@ -408,6 +431,8 @@ class LibraryPropertiesManifestParser(BaseManifestParser): if "author" in data: data["authors"] = self._parse_authors(data) del data["author"] + if "depends" in data: + data["dependencies"] = self._parse_dependencies(data["depends"]) return data @staticmethod @@ -534,6 +559,26 @@ class LibraryPropertiesManifestParser(BaseManifestParser): result["include"] = [include] return result + @staticmethod + def _parse_dependencies(raw): + result = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + if item.endswith(")") and "(" in item: + name, version = item.split("(") + result.append( + dict( + name=name.strip(), + version=version[:-1].strip(), + frameworks=["arduino"], + ) + ) + else: + result.append(dict(name=item, frameworks=["arduino"])) + return result + class PlatformJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.PLATFORM_JSON @@ -542,6 +587,8 @@ class PlatformJsonManifestParser(BaseManifestParser): data = json.loads(contents) if "frameworks" in data: data["frameworks"] = self._parse_frameworks(data["frameworks"]) + if "packages" in data: + data["dependencies"] = self._parse_dependencies(data["packages"]) return data @staticmethod @@ -550,6 +597,12 @@ class PlatformJsonManifestParser(BaseManifestParser): return None return [name.lower() for name in raw.keys()] + @staticmethod + def _parse_dependencies(raw): + return [ + dict(name=name, version=opts.get("version")) for name, opts in raw.items() + ] + class PackageJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.PACKAGE_JSON diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index fe16413b..11885f34 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -102,6 +102,32 @@ class RepositorySchema(StrictSchema): branch = fields.Str(validate=validate.Length(min=1, max=50)) +class DependencySchema(StrictSchema): + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + version = fields.Str(validate=validate.Length(min=1, max=100)) + authors = StrictListField(fields.Str(validate=validate.Length(min=1, max=50))) + platforms = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + frameworks = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + + class ExportSchema(BaseSchema): include = StrictListField(fields.Str) exclude = StrictListField(fields.Str) @@ -133,6 +159,7 @@ class ManifestSchema(BaseSchema): homepage = fields.Url(validate=validate.Length(min=1, max=255)) license = fields.Str(validate=validate.Length(min=1, max=255)) repository = fields.Nested(RepositorySchema) + dependencies = fields.Nested(DependencySchema, many=True) # library.json export = fields.Nested(ExportSchema) diff --git a/platformio/util.py b/platformio/util.py index f9c304f9..d35ce6a2 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -383,7 +383,6 @@ def _internet_on(): if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) return True except: # pylint: disable=bare-except @@ -403,9 +402,9 @@ def pepver_to_semver(pepver): def items_to_list(items): - if not isinstance(items, list): - items = [i.strip() for i in items.split(",")] - return [i.lower() for i in items if i] + if isinstance(items, list): + return items + return [i.strip() for i in items.split(",") if i.strip()] def items_in_list(needle, haystack): diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 90e2f6c7..8ca16e2c 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -40,6 +40,11 @@ def test_library_json_parser(): "flags": ["-DHELLO"] }, "examples": ["examples/*/*.pde"], + "dependencies": { + "deps1": "1.2.0", + "deps2": "https://github.com/username/package.git", + "@owner/deps3": "^2.1.3" + }, "customField": "Custom Value" } """ @@ -57,6 +62,11 @@ def test_library_json_parser(): "keywords": ["kw1", "kw2", "kw3"], "homepage": "http://old.url.format", "build": {"flags": ["-DHELLO"]}, + "dependencies": [ + {"name": "deps1", "version": "1.2.0"}, + {"name": "deps2", "version": "https://github.com/username/package.git"}, + {"name": "@owner/deps3", "version": "^2.1.3"}, + ], "customField": "Custom Value", }, ) @@ -68,7 +78,12 @@ def test_library_json_parser(): "platforms": "atmelavr", "export": { "exclude": "audio_samples" - } + }, + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino, espidf"}, + {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} + ] } """ mp = parser.LibraryJsonManifestParser(contents) @@ -79,9 +94,26 @@ def test_library_json_parser(): "frameworks": ["arduino"], "export": {"exclude": ["audio_samples"]}, "platforms": ["atmelavr"], + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + { + "name": "@owner/deps2", + "version": "1.0.0", + "frameworks": ["arduino", "espidf"], + }, + { + "name": "deps3", + "version": "1.0.0", + "platforms": ["ststm32", "sifive"], + }, + ], }, ) + # broken dependencies + with pytest.raises(parser.ManifestParserError): + mp = parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]}) + def test_module_json_parser(): contents = """ @@ -137,6 +169,7 @@ version=1.2.3 author=SomeAuthor sentence=This is Arduino library customField=Custom Value +depends=First Library (=2.0.0), Second Library (>=1.2.0), Third """ mp = parser.LibraryPropertiesManifestParser(contents) assert not jsondiff.diff( @@ -154,6 +187,20 @@ customField=Custom Value "authors": [{"email": "info@author.com", "name": "SomeAuthor"}], "keywords": ["uncategorized"], "customField": "Custom Value", + "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", + "dependencies": [ + { + "name": "First Library", + "version": "=2.0.0", + "frameworks": ["arduino"], + }, + { + "name": "Second Library", + "version": ">=1.2.0", + "frameworks": ["arduino"], + }, + {"name": "Third", "frameworks": ["arduino"]}, + ], }, ) @@ -244,6 +291,11 @@ def test_library_json_schema(): "base": "examples/JsonHttpClient", "files": ["JsonHttpClient.ino"] } + ], + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino"}, + {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } """ @@ -289,6 +341,15 @@ def test_library_json_schema(): "files": ["JsonHttpClient.ino"], }, ], + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + {"name": "@owner/deps2", "version": "1.0.0", "frameworks": ["arduino"]}, + { + "name": "deps3", + "version": "1.0.0", + "platforms": ["ststm32", "sifive"], + }, + ], }, ) @@ -304,6 +365,7 @@ paragraph=Supported display controller: SSD1306, SSD1309, SSD1322, SSD1325 category=Display url=https://github.com/olikraus/u8glib architectures=avr,sam +depends=First Library (=2.0.0), Second Library (>=1.2.0), Third """ raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES @@ -333,6 +395,19 @@ architectures=avr,sam ], "keywords": ["display"], "name": "U8glib", + "dependencies": [ + { + "name": "First Library", + "version": "=2.0.0", + "frameworks": ["arduino"], + }, + { + "name": "Second Library", + "version": ">=1.2.0", + "frameworks": ["arduino"], + }, + {"name": "Third", "frameworks": ["arduino"]}, + ], }, ) @@ -436,11 +511,6 @@ def test_platform_json_schema(): "optional": true, "version": "~4.2.0" }, - "framework-simba": { - "type": "framework", - "optional": true, - "version": ">=7.0.0" - }, "tool-avrdude": { "type": "uploader", "optional": true, @@ -475,6 +545,11 @@ def test_platform_json_schema(): }, "frameworks": sorted(["arduino", "simba"]), "version": "1.15.0", + "dependencies": [ + {"name": "toolchain-atmelavr", "version": "~1.50400.0"}, + {"name": "framework-arduinoavr", "version": "~4.2.0"}, + {"name": "tool-avrdude", "version": "~1.60300.0"}, + ], }, )