Added support for Arduino's library.properties `depends` field // Resolve #2781

HISTORY.rst

@@ -22,6 +22,7 @@ PlatformIO Core 4.0
 * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install <https://docs.platformio.org/page/userguide/platforms/cmd_install.html>`__ command  (`issue #3345 <https://github.com/platformio/platformio-core/issues/3345>`_)
 * Added support for "pythonPackages" in `platform.json <https://docs.platformio.org/page/platforms/creating_platform.html#manifest-file-platform-json>`__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed)
 * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 <https://github.com/platformio/platformio-core/issues/2591>`_)
+* Added support for Arduino's library.properties ``depends`` field (`issue #2781 <https://github.com/platformio/platformio-core/issues/2781>`_)
 * Updated SCons tool to 3.1.2
 * Updated Unity tool to 2.5.0
 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 <https://github.com/platformio/platformio-core/issues/3296>`_)

platformio/builder/tools/piolib.py

@@ -154,9 +154,7 @@ class LibBuilderBase(object):
 
     @property
     def dependencies(self):
-        return LibraryManager.normalize_dependencies(
+        return self._manifest.get("dependencies")
-            self._manifest.get("dependencies", [])
-        )
 
     @property
     def src_filter(self):

platformio/managers/lib.py

@@ -23,7 +23,7 @@ import click
 import semantic_version
 
 from platformio import app, exception, util
-from platformio.compat import glob_escape, string_types
+from platformio.compat import glob_escape
 from platformio.managers.package import BasePkgManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
 from platformio.project.config import ProjectConfig
@@ -61,29 +61,6 @@ class LibraryManager(BasePkgManager):
 
         return None
 
-    @staticmethod
-    def normalize_dependencies(dependencies):
-        if not dependencies:
-            return []
-        items = []
-        if isinstance(dependencies, dict):
-            if "name" in dependencies:
-                items.append(dependencies)
-            else:
-                for name, version in dependencies.items():
-                    items.append({"name": name, "version": version})
-        elif isinstance(dependencies, list):
-            items = [d for d in dependencies if "name" in d]
-        for item in items:
-            for k in ("frameworks", "platforms"):
-                if k not in item or isinstance(k, list):
-                    continue
-                if item[k] == "*":
-                    del item[k]
-                elif isinstance(item[k], string_types):
-                    item[k] = [i.strip() for i in item[k].split(",") if i.strip()]
-        return items
-
     def max_satisfying_repo_version(self, versions, requirements=None):
         def _cmp_dates(datestr1, datestr2):
             date1 = util.parse_date(datestr1)
@@ -312,7 +289,7 @@ class LibraryManager(BasePkgManager):
             click.secho("Installing dependencies", fg="yellow")
 
         builtin_lib_storages = None
-        for filters in self.normalize_dependencies(manifest["dependencies"]):
+        for filters in manifest["dependencies"]:
             assert "name" in filters
 
             # avoid circle dependencies

platformio/package/manifest/parser.py

@@ -19,6 +19,7 @@ import re
 
 import requests
 
+from platformio import util
 from platformio.compat import get_class_attributes, string_types
 from platformio.fs import get_file_contents
 from platformio.package.exception import ManifestParserError, UnknownManifestError
@@ -286,6 +287,8 @@ class LibraryJsonManifestParser(BaseManifestParser):
             data["platforms"] = self._parse_platforms(data["platforms"]) or None
         if "export" in data:
             data["export"] = self._parse_export(data["export"])
+        if "dependencies" in data:
+            data["dependencies"] = self._parse_dependencies(data["dependencies"])
 
         return data
 
@@ -350,6 +353,26 @@ class LibraryJsonManifestParser(BaseManifestParser):
             result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]]
         return result
 
+    @staticmethod
+    def _parse_dependencies(raw):
+        if isinstance(raw, dict):
+            if "name" in raw:  # compatibility with dep as dict
+                return [raw]
+            return [dict(name=name, version=version) for name, version in raw.items()]
+        if isinstance(raw, list):
+            for i, dependency in enumerate(raw):
+                assert isinstance(dependency, dict)
+                for k, v in dependency.items():
+                    if k not in ("platforms", "frameworks", "authors"):
+                        continue
+                    if "*" in v:
+                        del raw[i][k]
+                    raw[i][k] = util.items_to_list(v)
+            return raw
+        raise ManifestParserError(
+            "Invalid dependencies format, should be list or dictionary"
+        )
+
 
 class ModuleJsonManifestParser(BaseManifestParser):
     manifest_type = ManifestFileType.MODULE_JSON
@@ -408,6 +431,8 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
         if "author" in data:
             data["authors"] = self._parse_authors(data)
             del data["author"]
+        if "depends" in data:
+            data["dependencies"] = self._parse_dependencies(data["depends"])
         return data
 
     @staticmethod
@@ -534,6 +559,26 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
             result["include"] = [include]
         return result
 
+    @staticmethod
+    def _parse_dependencies(raw):
+        result = []
+        for item in raw.split(","):
+            item = item.strip()
+            if not item:
+                continue
+            if item.endswith(")") and "(" in item:
+                name, version = item.split("(")
+                result.append(
+                    dict(
+                        name=name.strip(),
+                        version=version[:-1].strip(),
+                        frameworks=["arduino"],
+                    )
+                )
+            else:
+                result.append(dict(name=item, frameworks=["arduino"]))
+        return result
+
 
 class PlatformJsonManifestParser(BaseManifestParser):
     manifest_type = ManifestFileType.PLATFORM_JSON
@@ -542,6 +587,8 @@ class PlatformJsonManifestParser(BaseManifestParser):
         data = json.loads(contents)
         if "frameworks" in data:
             data["frameworks"] = self._parse_frameworks(data["frameworks"])
+        if "packages" in data:
+            data["dependencies"] = self._parse_dependencies(data["packages"])
         return data
 
     @staticmethod
@@ -550,6 +597,12 @@ class PlatformJsonManifestParser(BaseManifestParser):
             return None
         return [name.lower() for name in raw.keys()]
 
+    @staticmethod
+    def _parse_dependencies(raw):
+        return [
+            dict(name=name, version=opts.get("version")) for name, opts in raw.items()
+        ]
+
 
 class PackageJsonManifestParser(BaseManifestParser):
     manifest_type = ManifestFileType.PACKAGE_JSON

platformio/package/manifest/schema.py

@@ -102,6 +102,32 @@ class RepositorySchema(StrictSchema):
     branch = fields.Str(validate=validate.Length(min=1, max=50))
 
 
+class DependencySchema(StrictSchema):
+    name = fields.Str(required=True, validate=validate.Length(min=1, max=100))
+    version = fields.Str(validate=validate.Length(min=1, max=100))
+    authors = StrictListField(fields.Str(validate=validate.Length(min=1, max=50)))
+    platforms = StrictListField(
+        fields.Str(
+            validate=[
+                validate.Length(min=1, max=50),
+                validate.Regexp(
+                    r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
+                ),
+            ]
+        )
+    )
+    frameworks = StrictListField(
+        fields.Str(
+            validate=[
+                validate.Length(min=1, max=50),
+                validate.Regexp(
+                    r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
+                ),
+            ]
+        )
+    )
+
+
 class ExportSchema(BaseSchema):
     include = StrictListField(fields.Str)
     exclude = StrictListField(fields.Str)
@@ -133,6 +159,7 @@ class ManifestSchema(BaseSchema):
     homepage = fields.Url(validate=validate.Length(min=1, max=255))
     license = fields.Str(validate=validate.Length(min=1, max=255))
     repository = fields.Nested(RepositorySchema)
+    dependencies = fields.Nested(DependencySchema, many=True)
 
     # library.json
     export = fields.Nested(ExportSchema)

platformio/util.py

@@ -383,7 +383,6 @@ def _internet_on():
             if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
                 requests.get("http://%s" % host, allow_redirects=False, timeout=timeout)
             else:
-
                 socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80))
             return True
         except:  # pylint: disable=bare-except
@@ -403,9 +402,9 @@ def pepver_to_semver(pepver):
 
 
 def items_to_list(items):
-    if not isinstance(items, list):
+    if isinstance(items, list):
-        items = [i.strip() for i in items.split(",")]
+        return items
-    return [i.lower() for i in items if i]
+    return [i.strip() for i in items.split(",") if i.strip()]
 
 
 def items_in_list(needle, haystack):

tests/package/test_manifest.py

@@ -40,6 +40,11 @@ def test_library_json_parser():
         "flags": ["-DHELLO"]
     },
     "examples": ["examples/*/*.pde"],
+    "dependencies": {
+        "deps1": "1.2.0",
+        "deps2": "https://github.com/username/package.git",
+        "@owner/deps3": "^2.1.3"
+    },
     "customField": "Custom Value"
 }
 """
@@ -57,6 +62,11 @@ def test_library_json_parser():
             "keywords": ["kw1", "kw2", "kw3"],
             "homepage": "http://old.url.format",
             "build": {"flags": ["-DHELLO"]},
+            "dependencies": [
+                {"name": "deps1", "version": "1.2.0"},
+                {"name": "deps2", "version": "https://github.com/username/package.git"},
+                {"name": "@owner/deps3", "version": "^2.1.3"},
+            ],
             "customField": "Custom Value",
         },
     )
@@ -68,7 +78,12 @@ def test_library_json_parser():
     "platforms": "atmelavr",
     "export": {
         "exclude": "audio_samples"
-    }
+    },
+    "dependencies": [
+        {"name": "deps1", "version": "1.0.0"},
+        {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino, espidf"},
+        {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]}
+    ]
 }
 """
     mp = parser.LibraryJsonManifestParser(contents)
@@ -79,9 +94,26 @@ def test_library_json_parser():
             "frameworks": ["arduino"],
             "export": {"exclude": ["audio_samples"]},
             "platforms": ["atmelavr"],
+            "dependencies": [
+                {"name": "deps1", "version": "1.0.0"},
+                {
+                    "name": "@owner/deps2",
+                    "version": "1.0.0",
+                    "frameworks": ["arduino", "espidf"],
+                },
+                {
+                    "name": "deps3",
+                    "version": "1.0.0",
+                    "platforms": ["ststm32", "sifive"],
+                },
+            ],
         },
     )
 
+    # broken dependencies
+    with pytest.raises(parser.ManifestParserError):
+        mp = parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]})
+
 
 def test_module_json_parser():
     contents = """
@@ -137,6 +169,7 @@ version=1.2.3
 author=SomeAuthor <info AT author.com>
 sentence=This is Arduino library
 customField=Custom Value
+depends=First Library (=2.0.0), Second Library (>=1.2.0), Third
 """
     mp = parser.LibraryPropertiesManifestParser(contents)
     assert not jsondiff.diff(
@@ -154,6 +187,20 @@ customField=Custom Value
             "authors": [{"email": "info@author.com", "name": "SomeAuthor"}],
             "keywords": ["uncategorized"],
             "customField": "Custom Value",
+            "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third",
+            "dependencies": [
+                {
+                    "name": "First Library",
+                    "version": "=2.0.0",
+                    "frameworks": ["arduino"],
+                },
+                {
+                    "name": "Second Library",
+                    "version": ">=1.2.0",
+                    "frameworks": ["arduino"],
+                },
+                {"name": "Third", "frameworks": ["arduino"]},
+            ],
         },
     )
 
@@ -244,6 +291,11 @@ def test_library_json_schema():
         "base": "examples/JsonHttpClient",
         "files": ["JsonHttpClient.ino"]
     }
+  ],
+  "dependencies": [
+    {"name": "deps1", "version": "1.0.0"},
+    {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino"},
+    {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]}
   ]
 }
 """
@@ -289,6 +341,15 @@ def test_library_json_schema():
                     "files": ["JsonHttpClient.ino"],
                 },
             ],
+            "dependencies": [
+                {"name": "deps1", "version": "1.0.0"},
+                {"name": "@owner/deps2", "version": "1.0.0", "frameworks": ["arduino"]},
+                {
+                    "name": "deps3",
+                    "version": "1.0.0",
+                    "platforms": ["ststm32", "sifive"],
+                },
+            ],
         },
     )
 
@@ -304,6 +365,7 @@ paragraph=Supported display controller: SSD1306, SSD1309, SSD1322, SSD1325
 category=Display
 url=https://github.com/olikraus/u8glib
 architectures=avr,sam
+depends=First Library (=2.0.0), Second Library (>=1.2.0), Third
 """
     raw_data = parser.ManifestParserFactory.new(
         contents, parser.ManifestFileType.LIBRARY_PROPERTIES
@@ -333,6 +395,19 @@ architectures=avr,sam
             ],
             "keywords": ["display"],
             "name": "U8glib",
+            "dependencies": [
+                {
+                    "name": "First Library",
+                    "version": "=2.0.0",
+                    "frameworks": ["arduino"],
+                },
+                {
+                    "name": "Second Library",
+                    "version": ">=1.2.0",
+                    "frameworks": ["arduino"],
+                },
+                {"name": "Third", "frameworks": ["arduino"]},
+            ],
         },
     )
 
@@ -436,11 +511,6 @@ def test_platform_json_schema():
       "optional": true,
       "version": "~4.2.0"
     },
-    "framework-simba": {
-      "type": "framework",
-      "optional": true,
-      "version": ">=7.0.0"
-    },
     "tool-avrdude": {
       "type": "uploader",
       "optional": true,
@@ -475,6 +545,11 @@ def test_platform_json_schema():
             },
             "frameworks": sorted(["arduino", "simba"]),
             "version": "1.15.0",
+            "dependencies": [
+                {"name": "toolchain-atmelavr", "version": "~1.50400.0"},
+                {"name": "framework-arduinoavr", "version": "~4.2.0"},
+                {"name": "tool-avrdude", "version": "~1.60300.0"},
+            ],
         },
     )