From e2f21212b77145e4db066033ac493c09c373b7d1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Apr 2022 23:14:19 +0300 Subject: [PATCH] Added support for symbolic links allowing pointing the local source folder to the Package Manager // Resolve #3348 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piolib.py | 6 +- platformio/package/manager/_install.py | 4 ++ platformio/package/manager/_symlink.py | 74 ++++++++++++++++++++++++ platformio/package/manager/_uninstall.py | 4 +- platformio/package/manager/base.py | 18 ++++-- platformio/package/meta.py | 14 +++-- tests/package/test_manager.py | 61 ++++++++++++++++++- tests/package/test_meta.py | 3 + tests/test_builder.py | 46 +++++++++++++++ 11 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 platformio/package/manager/_symlink.py diff --git a/HISTORY.rst b/HISTORY.rst index a862278a..511ccd3a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,6 +24,7 @@ PlatformIO Core 5 * `pio pkg uninstall `_ - uninstall the project dependencies or custom packages * `pio pkg update `__ - update the project dependencies or custom packages + - Added support for `symbolic links `__ allowing pointing the local source folder to the Package Manager (`issue #3348 `_) - Added support for `"scripts" `__ in package manifest (`issue #485 `_) - Added support for `multi-licensed `__ packages using SPDX Expressions (`issue #4037 `_) - Added support for `"dependencies" `__ declared in a "tool" package manifest diff --git a/docs b/docs index e24bd4f1..f834c85d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e24bd4f12de6da6646bdcbf45fafced11032aa3e +Subproject commit f834c85d5956851172ae1daa4e4aa9474839472e diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index cf6f4239..f293d9c4 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1032,7 +1032,11 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches continue for item in sorted(os.listdir(storage_dir)): lib_dir = os.path.join(storage_dir, item) - if item == "__cores__" or not os.path.isdir(lib_dir): + if item == "__cores__": + continue + if LibraryPackageManager.is_symlink(lib_dir): + lib_dir, _ = LibraryPackageManager.resolve_symlink(lib_dir) + if not lib_dir or not os.path.isdir(lib_dir): continue try: lb = LibBuilderFactory.new(env, lib_dir) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index f3631844..cc2d1713 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -148,6 +148,10 @@ class PackageManagerInstallMixin(object): def install_from_uri(self, uri, spec, checksum=None): spec = self.ensure_spec(spec) + + if spec.symlink: + return self.install_symlink(spec) + tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir()) vcs = None try: diff --git a/platformio/package/manager/_symlink.py b/platformio/package/manager/_symlink.py new file mode 100644 index 00000000..98a03174 --- /dev/null +++ b/platformio/package/manager/_symlink.py @@ -0,0 +1,74 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from platformio import fs +from platformio.package.exception import PackageException +from platformio.package.meta import PackageItem, PackageSpec + + +class PackageManagerSymlinkMixin(object): + @staticmethod + def is_symlink(path): + return path and path.endswith(".pio-link") and os.path.isfile(path) + + @classmethod + def resolve_symlink(cls, path): + assert cls.is_symlink(path) + data = None + with open(path, "r", encoding="utf-8") as fp: + data = json.load(fp) + spec = PackageSpec(**data["spec"]) + assert spec.symlink + pkg_dir = os.path.realpath(spec.uri[10:]) + if not os.path.isdir(pkg_dir): + with fs.cd(data["cwd"]): + pkg_dir = os.path.realpath(pkg_dir) + return (pkg_dir if os.path.isdir(pkg_dir) else None, spec) + + def get_symlinked_package(self, path): + pkg_dir, spec = self.resolve_symlink(path) + if not pkg_dir: + return None + pkg = PackageItem(os.path.realpath(pkg_dir)) + if not pkg.metadata: + pkg.metadata = self.build_metadata(pkg.path, spec) + return pkg + + def install_symlink(self, spec): + assert spec.symlink + pkg_dir = spec.uri[10:] + if not os.path.isdir(pkg_dir): + raise PackageException( + f"Can not create a symbolic link for `{pkg_dir}`, not a directory" + ) + link_path = os.path.join( + self.package_dir, + "%s.pio-link" % (spec.name or os.path.basename(os.path.abspath(pkg_dir))), + ) + with open(link_path, mode="w", encoding="utf-8") as fp: + json.dump(dict(cwd=os.getcwd(), spec=spec.as_dict()), fp) + return self.get_symlinked_package(link_path) + + def uninstall_symlink(self, spec): + assert spec.symlink + for name in os.listdir(self.package_dir): + path = os.path.join(self.package_dir, name) + if not self.is_symlink(path): + continue + pkg = self.get_symlinked_package(path) + if pkg.metadata.spec.uri == spec.uri: + os.remove(path) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 529e2432..9c6b5772 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -46,7 +46,9 @@ class PackageManagerUninstallMixin(object): if not skip_dependencies: self.uninstall_dependencies(pkg) - if os.path.islink(pkg.path): + if pkg.metadata.spec.symlink: + self.uninstall_symlink(pkg.metadata.spec) + elif os.path.islink(pkg.path): os.unlink(pkg.path) else: fs.rmtree(pkg.path) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 9188fb63..07ca2f5b 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -29,6 +29,7 @@ from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager._legacy import PackageManagerLegacyMixin from platformio.package.manager._registry import PackageManagerRegistryMixin +from platformio.package.manager._symlink import PackageManagerSymlinkMixin from platformio.package.manager._uninstall import PackageManagerUninstallMixin from platformio.package.manager._update import PackageManagerUpdateMixin from platformio.package.manifest.parser import ManifestParserFactory @@ -50,6 +51,7 @@ class ClickLoggingHandler(logging.Handler): class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-instance-attributes PackageManagerDownloadMixin, PackageManagerRegistryMixin, + PackageManagerSymlinkMixin, PackageManagerInstallMixin, PackageManagerUninstallMixin, PackageManagerUpdateMixin, @@ -213,7 +215,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in metadata.version = self.generate_rand_version() return metadata - def get_installed(self): + def get_installed(self): # pylint: disable=too-many-branches if not os.path.isdir(self.package_dir): return [] @@ -225,14 +227,18 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in for name in sorted(os.listdir(self.package_dir)): if name.startswith("_tmp_installing"): # legacy tmp folder continue - pkg_dir = os.path.join(self.package_dir, name) - if not os.path.isdir(pkg_dir): + pkg = None + path = os.path.join(self.package_dir, name) + if os.path.isdir(path): + pkg = PackageItem(path) + elif self.is_symlink(path): + pkg = self.get_symlinked_package(path) + if not pkg: continue - pkg = PackageItem(pkg_dir) if not pkg.metadata: try: - spec = self.build_legacy_spec(pkg_dir) - pkg.metadata = self.build_metadata(pkg_dir, spec) + spec = self.build_legacy_spec(pkg.path) + pkg.metadata = self.build_metadata(pkg.path, spec) except MissingPackageManifestError: pass if not pkg.metadata: diff --git a/platformio/package/meta.py b/platformio/package/meta.py index be494608..d712ac7d 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -170,6 +170,10 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes def external(self): return bool(self.uri) + @property + def symlink(self): + return self.uri and self.uri.startswith("symlink://") + @property def requirements(self): return self._requirements @@ -253,14 +257,16 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes @staticmethod def _parse_local_file(raw): - if raw.startswith("file://") or not any(c in raw for c in ("/", "\\")): + if raw.startswith(("file://", "symlink://")) or not any( + c in raw for c in ("/", "\\") + ): return raw if os.path.exists(raw): return "file://%s" % raw return raw def _parse_requirements(self, raw): - if "@" not in raw or raw.startswith("file://"): + if "@" not in raw or raw.startswith(("file://", "symlink://")): return raw tokens = raw.rsplit("@", 1) if any(s in tokens[1] for s in (":", "/")): @@ -302,7 +308,7 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes # if local file or valid URI with scheme vcs+protocol:// if ( - parts.scheme in ("file", ) + parts.scheme in ("file", "symlink://") or "+" in parts.scheme or self.uri.startswith("git+") ): @@ -334,7 +340,7 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes if uri.endswith("/"): uri = uri[:-1] stop_chars = ["#", "?"] - if uri.startswith(("file://", )): + if uri.startswith(("file://", "symlink://")): stop_chars.append("@") # detached path for c in stop_chars: if c in uri: diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 12cedd40..d1c0fc5b 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -41,11 +41,11 @@ def test_download(isolated_pio_core): lm.set_log_level(logging.ERROR) archive_path = lm.download(url, checksum) assert fs.calculate_file_hashsum("sha256", archive_path) == checksum - lm.cleanup_expired_downloads() + lm.cleanup_expired_downloads(time.time()) assert os.path.isfile(archive_path) # test outdated downloads lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1) - lm.cleanup_expired_downloads() + lm.cleanup_expired_downloads(time.time()) assert not os.path.isfile(archive_path) # check that key is deleted from DB with open(lm.get_download_usagedb_path(), encoding="utf8") as fp: @@ -289,6 +289,63 @@ def test_install_force(isolated_pio_core, tmpdir_factory): assert pkg.metadata.version.major > 5 +def test_symlink(tmp_path: Path): + external_pkg_dir = tmp_path / "External" + external_pkg_dir.mkdir() + (external_pkg_dir / "library.json").write_text( + """ +{ + "name": "External", + "version": "1.0.0" +} +""" + ) + + storage_dir = tmp_path / "storage" + installed_pkg_dir = storage_dir / "installed" + installed_pkg_dir.mkdir(parents=True) + (installed_pkg_dir / "library.json").write_text( + """ +{ + "name": "Installed", + "version": "1.0.0" +} +""" + ) + + spec = "CustomExternal=symlink://%s" % str(external_pkg_dir) + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + pkg = lm.install(spec) + assert os.path.isfile(str(storage_dir / "CustomExternal.pio-link")) + assert pkg.metadata.name == "External" + assert pkg.metadata.version.major == 1 + assert ["External", "Installed"] == [ + pkg.metadata.name for pkg in lm.get_installed() + ] + assert lm.get_package("External").metadata.spec.uri.startswith("symlink://") + assert lm.get_package(spec).metadata.spec.uri.startswith("symlink://") + + # try to update + lm.update(pkg) + + # uninstall + lm.uninstall("External") + assert ["Installed"] == [pkg.metadata.name for pkg in lm.get_installed()] + # ensure original package was not rmeoved + assert external_pkg_dir.is_dir() + + # install again, remove from a disk + assert lm.install("symlink://%s" % str(external_pkg_dir)) + assert os.path.isfile(str(storage_dir / "External.pio-link")) + assert ["External", "Installed"] == [ + pkg.metadata.name for pkg in lm.get_installed() + ] + fs.rmtree(str(external_pkg_dir)) + lm.memcache_reset() + assert ["Installed"] == [pkg.metadata.name for pkg in lm.get_installed()] + + def test_scripts(isolated_pio_core, tmp_path: Path): pkg_dir = tmp_path / "foo" scripts_dir = pkg_dir / "scripts" diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index 869f0a89..4faabeba 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -90,6 +90,9 @@ def test_spec_local_urls(tmpdir_factory): assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( uri="file:///tmp/some-lib/", name="some-lib" ) + assert PackageSpec("symlink:///tmp/soft-link/") == PackageSpec( + uri="symlink:///tmp/soft-link/", name="soft-link" + ) # detached package assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec( uri="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib" diff --git a/tests/test_builder.py b/tests/test_builder.py index f220e50c..80ccd241 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path + from platformio.commands.run.command import cli as cmd_run @@ -176,3 +178,47 @@ int main() { for level in (0, 1, 2) ) assert all("-O%s" % optimization not in line for optimization in ("g", "s")) + + +def test_symlinked_libs(clirunner, validate_cliresult, tmp_path: Path): + external_pkg_dir = tmp_path / "External" + external_pkg_dir.mkdir() + (external_pkg_dir / "External.h").write_text( + """ +#define EXTERNAL 1 +""" + ) + (external_pkg_dir / "library.json").write_text( + """ +{ + "name": "External", + "version": "1.0.0" +} +""" + ) + + project_dir = tmp_path / "project" + src_dir = project_dir / "src" + src_dir.mkdir(parents=True) + (src_dir / "main.c").write_text( + """ +#include +# +#if !defined(EXTERNAL) +#error "EXTERNAL is not defined" +#endif + +int main() { +} +""" + ) + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +lib_deps = symlink://%s + """ + % str(external_pkg_dir) + ) + result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir), "--verbose"]) + validate_cliresult(result)