diff --git a/HISTORY.rst b/HISTORY.rst index ff55d09d..17987fe0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -53,6 +53,10 @@ PlatformIO Core 6 - Fixed an issue with the |LDF| when recursively scanning dependencies in the ``chain`` mode - Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) +* **Package Management** + + - Fixed an issue when library dependencies were installed for the incompatible project environment (`issue #4338 `_) + * **Miscellaneous** - Warn about incompatible Bash version for the `Shell Completion `__ (`issue #4326 `_) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 9059a25e..c830c881 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -28,7 +28,7 @@ import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error -from platformio import exception, fs, util +from platformio import exception, fs from platformio.builder.tools import platformio as piotool from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types from platformio.http import HTTPClientError, InternetIsOffline @@ -41,7 +41,7 @@ from platformio.package.manifest.parser import ( ManifestParserError, ManifestParserFactory, ) -from platformio.package.meta import PackageItem +from platformio.package.meta import PackageCompatibility, PackageItem from platformio.project.options import ProjectOptions @@ -582,10 +582,14 @@ class ArduinoLibBuilder(LibBuilderBase): return "chain+" def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, ["arduino", "energia"]) + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=["arduino", "energia"]) + ) def is_platforms_compatible(self, platforms): - return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) + return PackageCompatibility(platforms=platforms).is_compatible( + PackageCompatibility(platforms=self._manifest.get("platforms")) + ) @property def build_flags(self): @@ -640,7 +644,9 @@ class MbedLibBuilder(LibBuilderBase): return include_dirs def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, ["mbed"]) + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=["mbed"]) + ) def process_extra_options(self): self._process_mbed_lib_confs() @@ -853,10 +859,14 @@ class PlatformIOLibBuilder(LibBuilderBase): ) def is_platforms_compatible(self, platforms): - return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) + return PackageCompatibility(platforms=platforms).is_compatible( + PackageCompatibility(platforms=self._manifest.get("platforms")) + ) def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"]) + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=self._manifest.get("frameworks")) + ) class ProjectAsLibBuilder(LibBuilderBase): diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py index 4aa6ec5d..ecfbe7bb 100644 --- a/platformio/package/commands/install.py +++ b/platformio/package/commands/install.py @@ -23,7 +23,9 @@ from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageSpec +from platformio.package.meta import PackageCompatibility, PackageSpec +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies from platformio.test.result import TestSuite @@ -202,8 +204,24 @@ def _install_project_env_libraries(project_env, options): _uninstall_project_unused_libdeps(project_env, options) already_up_to_date = not options.get("force") config = ProjectConfig.get_instance() + + compatibility_qualifiers = {} + if config.get(f"env:{project_env}", "platform"): + try: + p = PlatformFactory.new(config.get(f"env:{project_env}", "platform")) + compatibility_qualifiers["platforms"] = [p.name] + except UnknownPlatform: + pass + if config.get(f"env:{project_env}", "framework"): + compatibility_qualifiers["frameworks"] = config.get( + f"env:{project_env}", "framework" + ) + env_lm = LibraryPackageManager( - os.path.join(config.get("platformio", "libdeps_dir"), project_env) + os.path.join(config.get("platformio", "libdeps_dir"), project_env), + compatibility=PackageCompatibility(**compatibility_qualifiers) + if compatibility_qualifiers + else None, ) private_lm = LibraryPackageManager( os.path.join(config.get("platformio", "lib_dir")) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index ae442bd0..c89d7b86 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,7 @@ import click from platformio import app, compat, fs, util from platformio.package.exception import PackageException, UnknownPackageError -from platformio.package.meta import PackageItem +from platformio.package.meta import PackageCompatibility, PackageItem from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -55,9 +55,9 @@ class PackageManagerInstallMixin: def _install( self, spec, - search_qualifiers=None, skip_dependencies=False, force=False, + compatibility: PackageCompatibility = None, ): spec = self.ensure_spec(spec) @@ -97,7 +97,12 @@ class PackageManagerInstallMixin: if spec.external: pkg = self.install_from_uri(spec.uri, spec) else: - pkg = self.install_from_registry(spec, search_qualifiers) + pkg = self.install_from_registry( + spec, + search_qualifiers=compatibility.to_search_qualifiers() + if compatibility + else None, + ) if not pkg or not pkg.metadata: raise PackageException( @@ -137,20 +142,29 @@ class PackageManagerInstallMixin: if dependency.get("owner"): self.log.warning( click.style( - "Warning! Could not install dependency %s for package '%s'" - % (dependency, pkg.metadata.name), + "Warning! Could not install `%s` dependency " + "for the`%s` package" % (dependency, pkg.metadata.name), fg="yellow", ) ) def install_dependency(self, dependency): - spec = self.dependency_to_spec(dependency) - search_qualifiers = { - key: value - for key, value in dependency.items() - if key in ("authors", "platforms", "frameworks") - } - return self._install(spec, search_qualifiers=search_qualifiers or None) + dependency_compatibility = PackageCompatibility.from_dependency(dependency) + if self.compatibility and not dependency_compatibility.is_compatible( + self.compatibility + ): + self.log.debug( + click.style( + "Skip incompatible `%s` dependency with `%s`" + % (dependency, self.compatibility), + fg="yellow", + ) + ) + return None + return self._install( + spec=self.dependency_to_spec(dependency), + compatibility=dependency_compatibility, + ) def install_from_uri(self, uri, spec, checksum=None): spec = self.ensure_spec(spec) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index c90b940f..8369f736 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -59,9 +59,10 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in ): _MEMORY_CACHE = {} - def __init__(self, pkg_type, package_dir): + def __init__(self, pkg_type, package_dir, compatibility=None): self.pkg_type = pkg_type self.package_dir = package_dir + self.compatibility = compatibility self.log = self._setup_logger() self._MEMORY_CACHE = {} diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index f17e5eb8..6babfc9c 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -24,11 +24,12 @@ from platformio.project.config import ProjectConfig class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors - def __init__(self, package_dir=None): + def __init__(self, package_dir=None, **kwargs): super().__init__( PackageType.LIBRARY, package_dir or ProjectConfig.get_instance().get("platformio", "globallib_dir"), + **kwargs ) @property diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 34475a9c..fbd2b734 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -25,6 +25,7 @@ from platformio import fs from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType from platformio.package.version import cast_version_to_semver +from platformio.util import items_in_list class PackageType: @@ -63,6 +64,46 @@ class PackageType: return None +class PackageCompatibility: + + KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors") + + @classmethod + def from_dependency(cls, dependency): + assert isinstance(dependency, dict) + qualifiers = { + key: value + for key, value in dependency.items() + if key in cls.KNOWN_QUALIFIERS + } + return PackageCompatibility(**qualifiers) + + def __init__(self, **kwargs): + self.qualifiers = {} + for key, value in kwargs.items(): + if key not in self.KNOWN_QUALIFIERS: + raise ValueError( + "Unknown package compatibility qualifier -> `%s`" % key + ) + self.qualifiers[key] = value + + def __repr__(self): + return "PackageCompatibility <%s>" % self.qualifiers + + def to_search_qualifiers(self): + return self.qualifiers + + def is_compatible(self, other): + assert isinstance(other, PackageCompatibility) + for key, value in self.qualifiers.items(): + other_value = other.qualifiers.get(key) + if not value or not other_value: + continue + if not items_in_list(value, other_value): + return False + return True + + class PackageOutdatedResult: UPDATE_INCREMENT_MAJOR = "major" UPDATE_INCREMENT_MINOR = "minor" diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py index 449783d1..415baea8 100644 --- a/tests/commands/pkg/test_install.py +++ b/tests/commands/pkg/test_install.py @@ -29,7 +29,9 @@ from platformio.project.config import ProjectConfig PROJECT_CONFIG_TPL = """ [env] platform = platformio/atmelavr@^3.4.0 -lib_deps = milesburton/DallasTemperature@^3.9.1 +lib_deps = + milesburton/DallasTemperature@^3.9.1 + https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip [env:baremetal] board = uno @@ -134,7 +136,8 @@ def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ - PackageSpec("DallasTemperature@3.10.0") + PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), ] assert len(ToolPackageManager().get_installed()) == 0 @@ -154,6 +157,7 @@ def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec("OneWire@2.3.7"), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ @@ -177,6 +181,7 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec("OneWire@2.3.7"), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ @@ -184,7 +189,8 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): PackageSpec("toolchain-atmelavr@1.70300.191015"), ] assert config.get("env:devkit", "lib_deps") == [ - "milesburton/DallasTemperature@^3.9.1" + "milesburton/DallasTemperature@^3.9.1", + "https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip", ] # test "Already up-to-date" @@ -270,6 +276,7 @@ def test_remove_project_unused_libdeps( lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec("OneWire@2.3.7"), ] @@ -286,6 +293,7 @@ def test_remove_project_unused_libdeps( assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec("OneWire@2.3.7"), ] diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index 4faabeba..f7eae748 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -18,6 +18,7 @@ import jsondiff import semantic_version from platformio.package.meta import ( + PackageCompatibility, PackageMetaData, PackageOutdatedResult, PackageSpec, @@ -312,3 +313,25 @@ def test_metadata_load(tmpdir_factory): metadata.dump(str(piopm_path)) restored_metadata = PackageMetaData.load(str(piopm_path)) assert metadata == restored_metadata + + +def test_compatibility(): + assert PackageCompatibility().is_compatible(PackageCompatibility()) + assert PackageCompatibility().is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility(frameworks=["arduino"]).is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility(platforms="espressif32").is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=None)) + assert PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=["*"])) + assert not PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=["atmelavr"]))