Added support for symbolic links allowing pointing the local source folder to the Package Manager // Resolve #3348

HISTORY.rst

@@ -24,6 +24,7 @@ PlatformIO Core 5
     * `pio pkg uninstall <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_uninstall.html>`_ - uninstall the project dependencies or custom packages
     * `pio pkg update <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_update.html>`__ - update the project dependencies or custom packages
 
+  - Added support for `symbolic links <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html#local-folder>`__ allowing pointing the local source folder to the Package Manager (`issue #3348 <https://github.com/platformio/platformio-core/issues/3348>`_)
   - Added support for `"scripts" <https://docs.platformio.org/en/latest/librarymanager/config.html#scripts>`__ in package manifest (`issue #485 <https://github.com/platformio/platformio-core/issues/485>`_)
   - Added support for `multi-licensed <https://docs.platformio.org/en/latest/librarymanager/config.html#license>`__ packages using SPDX Expressions (`issue #4037 <https://github.com/platformio/platformio-core/issues/4037>`_)
   - Added support for `"dependencies" <https://docs.platformio.org/en/latest/librarymanager/config.html#dependencies>`__ declared in a "tool" package manifest

platformio/builder/tools/piolib.py

@@ -1032,7 +1032,11 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             continue
         for item in sorted(os.listdir(storage_dir)):
             lib_dir = os.path.join(storage_dir, item)
-            if item == "__cores__" or not os.path.isdir(lib_dir):
+            if item == "__cores__":
+                continue
+            if LibraryPackageManager.is_symlink(lib_dir):
+                lib_dir, _ = LibraryPackageManager.resolve_symlink(lib_dir)
+            if not lib_dir or not os.path.isdir(lib_dir):
                 continue
             try:
                 lb = LibBuilderFactory.new(env, lib_dir)

platformio/package/manager/_install.py

@@ -148,6 +148,10 @@ class PackageManagerInstallMixin(object):
 
     def install_from_uri(self, uri, spec, checksum=None):
         spec = self.ensure_spec(spec)
+
+        if spec.symlink:
+            return self.install_symlink(spec)
+
         tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir())
         vcs = None
         try:

platformio/package/manager/_symlink.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from platformio import fs
+from platformio.package.exception import PackageException
+from platformio.package.meta import PackageItem, PackageSpec
+
+
+class PackageManagerSymlinkMixin(object):
+    @staticmethod
+    def is_symlink(path):
+        return path and path.endswith(".pio-link") and os.path.isfile(path)
+
+    @classmethod
+    def resolve_symlink(cls, path):
+        assert cls.is_symlink(path)
+        data = None
+        with open(path, "r", encoding="utf-8") as fp:
+            data = json.load(fp)
+        spec = PackageSpec(**data["spec"])
+        assert spec.symlink
+        pkg_dir = os.path.realpath(spec.uri[10:])
+        if not os.path.isdir(pkg_dir):
+            with fs.cd(data["cwd"]):
+                pkg_dir = os.path.realpath(pkg_dir)
+        return (pkg_dir if os.path.isdir(pkg_dir) else None, spec)
+
+    def get_symlinked_package(self, path):
+        pkg_dir, spec = self.resolve_symlink(path)
+        if not pkg_dir:
+            return None
+        pkg = PackageItem(os.path.realpath(pkg_dir))
+        if not pkg.metadata:
+            pkg.metadata = self.build_metadata(pkg.path, spec)
+        return pkg
+
+    def install_symlink(self, spec):
+        assert spec.symlink
+        pkg_dir = spec.uri[10:]
+        if not os.path.isdir(pkg_dir):
+            raise PackageException(
+                f"Can not create a symbolic link for `{pkg_dir}`, not a directory"
+            )
+        link_path = os.path.join(
+            self.package_dir,
+            "%s.pio-link" % (spec.name or os.path.basename(os.path.abspath(pkg_dir))),
+        )
+        with open(link_path, mode="w", encoding="utf-8") as fp:
+            json.dump(dict(cwd=os.getcwd(), spec=spec.as_dict()), fp)
+        return self.get_symlinked_package(link_path)
+
+    def uninstall_symlink(self, spec):
+        assert spec.symlink
+        for name in os.listdir(self.package_dir):
+            path = os.path.join(self.package_dir, name)
+            if not self.is_symlink(path):
+                continue
+            pkg = self.get_symlinked_package(path)
+            if pkg.metadata.spec.uri == spec.uri:
+                os.remove(path)

platformio/package/manager/_uninstall.py

@@ -46,7 +46,9 @@ class PackageManagerUninstallMixin(object):
         if not skip_dependencies:
             self.uninstall_dependencies(pkg)
 
-        if os.path.islink(pkg.path):
+        if pkg.metadata.spec.symlink:
+            self.uninstall_symlink(pkg.metadata.spec)
+        elif os.path.islink(pkg.path):
             os.unlink(pkg.path)
         else:
             fs.rmtree(pkg.path)

platformio/package/manager/base.py

@@ -29,6 +29,7 @@ from platformio.package.manager._download import PackageManagerDownloadMixin
 from platformio.package.manager._install import PackageManagerInstallMixin
 from platformio.package.manager._legacy import PackageManagerLegacyMixin
 from platformio.package.manager._registry import PackageManagerRegistryMixin
+from platformio.package.manager._symlink import PackageManagerSymlinkMixin
 from platformio.package.manager._uninstall import PackageManagerUninstallMixin
 from platformio.package.manager._update import PackageManagerUpdateMixin
 from platformio.package.manifest.parser import ManifestParserFactory
@@ -50,6 +51,7 @@ class ClickLoggingHandler(logging.Handler):
 class BasePackageManager(  # pylint: disable=too-many-public-methods,too-many-instance-attributes
     PackageManagerDownloadMixin,
     PackageManagerRegistryMixin,
+    PackageManagerSymlinkMixin,
     PackageManagerInstallMixin,
     PackageManagerUninstallMixin,
     PackageManagerUpdateMixin,
@@ -213,7 +215,7 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods,too-many-in
             metadata.version = self.generate_rand_version()
         return metadata
 
-    def get_installed(self):
+    def get_installed(self):  # pylint: disable=too-many-branches
         if not os.path.isdir(self.package_dir):
             return []
 
@@ -225,14 +227,18 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods,too-many-in
         for name in sorted(os.listdir(self.package_dir)):
             if name.startswith("_tmp_installing"):  # legacy tmp folder
                 continue
-            pkg_dir = os.path.join(self.package_dir, name)
-            if not os.path.isdir(pkg_dir):
+            pkg = None
+            path = os.path.join(self.package_dir, name)
+            if os.path.isdir(path):
+                pkg = PackageItem(path)
+            elif self.is_symlink(path):
+                pkg = self.get_symlinked_package(path)
+            if not pkg:
                 continue
-            pkg = PackageItem(pkg_dir)
             if not pkg.metadata:
                 try:
-                    spec = self.build_legacy_spec(pkg_dir)
-                    pkg.metadata = self.build_metadata(pkg_dir, spec)
+                    spec = self.build_legacy_spec(pkg.path)
+                    pkg.metadata = self.build_metadata(pkg.path, spec)
                 except MissingPackageManifestError:
                     pass
             if not pkg.metadata:

platformio/package/meta.py

@@ -170,6 +170,10 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
     def external(self):
         return bool(self.uri)
 
+    @property
+    def symlink(self):
+        return self.uri and self.uri.startswith("symlink://")
+
     @property
     def requirements(self):
         return self._requirements
@@ -253,14 +257,16 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
 
     @staticmethod
     def _parse_local_file(raw):
-        if raw.startswith("file://") or not any(c in raw for c in ("/", "\\")):
+        if raw.startswith(("file://", "symlink://")) or not any(
+            c in raw for c in ("/", "\\")
+        ):
             return raw
         if os.path.exists(raw):
             return "file://%s" % raw
         return raw
 
     def _parse_requirements(self, raw):
-        if "@" not in raw or raw.startswith("file://"):
+        if "@" not in raw or raw.startswith(("file://", "symlink://")):
             return raw
         tokens = raw.rsplit("@", 1)
         if any(s in tokens[1] for s in (":", "/")):
@@ -302,7 +308,7 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
 
         # if local file or valid URI with scheme vcs+protocol://
         if (
-            parts.scheme in ("file", )
+            parts.scheme in ("file", "symlink://")
             or "+" in parts.scheme
             or self.uri.startswith("git+")
         ):
@@ -334,7 +340,7 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
         if uri.endswith("/"):
             uri = uri[:-1]
         stop_chars = ["#", "?"]
-        if uri.startswith(("file://", )):
+        if uri.startswith(("file://", "symlink://")):
             stop_chars.append("@")  # detached path
         for c in stop_chars:
             if c in uri:

tests/package/test_manager.py

@@ -41,11 +41,11 @@ def test_download(isolated_pio_core):
     lm.set_log_level(logging.ERROR)
     archive_path = lm.download(url, checksum)
     assert fs.calculate_file_hashsum("sha256", archive_path) == checksum
-    lm.cleanup_expired_downloads()
+    lm.cleanup_expired_downloads(time.time())
     assert os.path.isfile(archive_path)
     # test outdated downloads
     lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1)
-    lm.cleanup_expired_downloads()
+    lm.cleanup_expired_downloads(time.time())
     assert not os.path.isfile(archive_path)
     # check that key is deleted from DB
     with open(lm.get_download_usagedb_path(), encoding="utf8") as fp:
@@ -289,6 +289,63 @@ def test_install_force(isolated_pio_core, tmpdir_factory):
     assert pkg.metadata.version.major > 5
 
 
+def test_symlink(tmp_path: Path):
+    external_pkg_dir = tmp_path / "External"
+    external_pkg_dir.mkdir()
+    (external_pkg_dir / "library.json").write_text(
+        """
+{
+    "name": "External",
+    "version": "1.0.0"
+}
+"""
+    )
+
+    storage_dir = tmp_path / "storage"
+    installed_pkg_dir = storage_dir / "installed"
+    installed_pkg_dir.mkdir(parents=True)
+    (installed_pkg_dir / "library.json").write_text(
+        """
+{
+    "name": "Installed",
+    "version": "1.0.0"
+}
+"""
+    )
+
+    spec = "CustomExternal=symlink://%s" % str(external_pkg_dir)
+    lm = LibraryPackageManager(str(storage_dir))
+    lm.set_log_level(logging.ERROR)
+    pkg = lm.install(spec)
+    assert os.path.isfile(str(storage_dir / "CustomExternal.pio-link"))
+    assert pkg.metadata.name == "External"
+    assert pkg.metadata.version.major == 1
+    assert ["External", "Installed"] == [
+        pkg.metadata.name for pkg in lm.get_installed()
+    ]
+    assert lm.get_package("External").metadata.spec.uri.startswith("symlink://")
+    assert lm.get_package(spec).metadata.spec.uri.startswith("symlink://")
+
+    # try to update
+    lm.update(pkg)
+
+    # uninstall
+    lm.uninstall("External")
+    assert ["Installed"] == [pkg.metadata.name for pkg in lm.get_installed()]
+    # ensure original package was not rmeoved
+    assert external_pkg_dir.is_dir()
+
+    # install again, remove from a disk
+    assert lm.install("symlink://%s" % str(external_pkg_dir))
+    assert os.path.isfile(str(storage_dir / "External.pio-link"))
+    assert ["External", "Installed"] == [
+        pkg.metadata.name for pkg in lm.get_installed()
+    ]
+    fs.rmtree(str(external_pkg_dir))
+    lm.memcache_reset()
+    assert ["Installed"] == [pkg.metadata.name for pkg in lm.get_installed()]
+
+
 def test_scripts(isolated_pio_core, tmp_path: Path):
     pkg_dir = tmp_path / "foo"
     scripts_dir = pkg_dir / "scripts"

tests/package/test_meta.py

@@ -90,6 +90,9 @@ def test_spec_local_urls(tmpdir_factory):
     assert PackageSpec("file:///tmp/some-lib/") == PackageSpec(
         uri="file:///tmp/some-lib/", name="some-lib"
     )
+    assert PackageSpec("symlink:///tmp/soft-link/") == PackageSpec(
+        uri="symlink:///tmp/soft-link/", name="soft-link"
+    )
     # detached package
     assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec(
         uri="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib"

tests/test_builder.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from pathlib import Path
+
 from platformio.commands.run.command import cli as cmd_run
 
 
@@ -176,3 +178,47 @@ int main() {
                 for level in (0, 1, 2)
             )
             assert all("-O%s" % optimization not in line for optimization in ("g", "s"))
+
+
+def test_symlinked_libs(clirunner, validate_cliresult, tmp_path: Path):
+    external_pkg_dir = tmp_path / "External"
+    external_pkg_dir.mkdir()
+    (external_pkg_dir / "External.h").write_text(
+        """
+#define EXTERNAL 1
+"""
+    )
+    (external_pkg_dir / "library.json").write_text(
+        """
+{
+    "name": "External",
+    "version": "1.0.0"
+}
+"""
+    )
+
+    project_dir = tmp_path / "project"
+    src_dir = project_dir / "src"
+    src_dir.mkdir(parents=True)
+    (src_dir / "main.c").write_text(
+        """
+#include <External.h>
+#
+#if !defined(EXTERNAL)
+#error "EXTERNAL is not defined"
+#endif
+
+int main() {
+}
+"""
+    )
+    (project_dir / "platformio.ini").write_text(
+        """
+[env:native]
+platform = native
+lib_deps = symlink://%s
+    """
+        % str(external_pkg_dir)
+    )
+    result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir), "--verbose"])
+    validate_cliresult(result)