diff --git a/HISTORY.rst b/HISTORY.rst index e886594a..e57e3fe6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,8 +15,9 @@ PlatformIO Core 5 - New unified Package Management CLI (``pio pkg``): - * `pio pkg outdated `__ - check for project outdated packages * `pio pkg exec `_ - run command from package tool (`issue #4163 `_) + * `pio pkg install `_ - install the project dependencies or custom packages + * `pio pkg outdated `__ - check for project outdated packages - Added support for dependencies declared in a "tool" type package - Ignore files according to the patterns declared in ".gitignore" when using `pio package pack `__ command (`issue #4188 `_) diff --git a/docs b/docs index 0e834b92..0c41108f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0e834b92f74f88ba7974bc1caa0d652b4b9a055b +Subproject commit 0c41108fe3d8e577bf6b4cf5074a08ad104571f0 diff --git a/platformio/commands/pkg.py b/platformio/commands/pkg.py index caac35ac..c2454993 100644 --- a/platformio/commands/pkg.py +++ b/platformio/commands/pkg.py @@ -15,6 +15,7 @@ import click from platformio.package.commands.exec import package_exec_cmd +from platformio.package.commands.install import package_install_cmd from platformio.package.commands.outdated import package_outdated_cmd from platformio.package.commands.pack import package_pack_cmd from platformio.package.commands.publish import package_publish_cmd @@ -25,6 +26,7 @@ from platformio.package.commands.unpublish import package_unpublish_cmd "pkg", commands=[ package_exec_cmd, + package_install_cmd, package_outdated_cmd, package_pack_cmd, package_publish_cmd, diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 191a071f..f6b184d6 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -14,6 +14,7 @@ from platformio.commands.platform import init_platform from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME +from platformio.package.commands.install import install_project_env_dependencies from platformio.project.exception import UndefinedEnvPlatformError # pylint: disable=too-many-instance-attributes @@ -64,6 +65,11 @@ class EnvironmentProcessor(object): if "monitor" in build_targets: build_targets.remove("monitor") + install_project_env_dependencies( + self.name, + {"project_targets": build_targets}, + ) + result = init_platform(self.options["platform"]).run( build_vars, build_targets, self.silent, self.verbose, self.jobs ) diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py new file mode 100644 index 00000000..cc5ced44 --- /dev/null +++ b/platformio/package/commands/install.py @@ -0,0 +1,194 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import click + +from platformio import fs +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.project.config import ProjectConfig + + +@click.command( + "install", short_help="Install the project dependencies or custom packages" +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), +) +@click.option("-e", "--environment", "environments", multiple=True) +@click.option("-p", "--platform", "platforms", multiple=True) +@click.option("-t", "--tool", "tools", multiple=True) +@click.option("-l", "--library", "libraries", multiple=True) +@click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies") +@click.option("-g", "--global", is_flag=True, help="Install package globally") +@click.option( + "--storage-dir", + default=None, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + help="Custom package storage directory", +) +@click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists") +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +def package_install_cmd(**options): + if options.get("global"): + install_global_dependencies(options) + else: + install_project_dependencies(options) + + +def install_global_dependencies(options): + pm = PlatformPackageManager(options.get("storage_dir")) + tm = ToolPackageManager(options.get("storage_dir")) + lm = LibraryPackageManager(options.get("storage_dir")) + for obj in (pm, tm, lm): + obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG) + for spec in options.get("platforms"): + pm.install( + spec, + skip_default_package=options.get("skip_dependencies"), + force=options.get("force"), + ) + for spec in options.get("tools"): + tm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + for spec in options.get("libraries", []): + lm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + + +def install_project_dependencies(options): + environments = options["environments"] + with fs.cd(options["project_dir"]): + config = ProjectConfig.get_instance() + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + if not options["silent"]: + click.echo( + "Resolving %s environment packages..." % click.style(env, fg="cyan") + ) + already_up_to_date = install_project_env_dependencies(env, options) + if not options["silent"] and already_up_to_date: + click.secho("Already up-to-date.", fg="green") + + +def install_project_env_dependencies(project_env, options=None): + """Used in `pio run` -> Processor""" + options = options or {} + return any( + [ + _install_project_env_platform(project_env, options), + _install_project_env_libraries(project_env, options), + ] + ) + + +def _install_project_env_platform(project_env, options): + already_up_to_date = not options.get("force") + config = ProjectConfig.get_instance() + pm = PlatformPackageManager() + if options.get("silent"): + pm.set_log_level(logging.WARN) + + if options.get("platforms") or options.get("tools"): + already_up_to_date = False + tm = ToolPackageManager() + if not options.get("silent"): + pm.set_log_level(logging.DEBUG) + tm.set_log_level(logging.DEBUG) + for platform in options.get("platforms"): + pm.install( + platform, + project_env=project_env, + project_targets=options.get("project_targets"), + skip_default_package=options.get("skip_dependencies"), + force=options.get("force"), + ) + for spec in options.get("tools"): + tm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + return not already_up_to_date + + if options.get("libraries"): + return False + + # if not custom libraries, install declared platform + platform = config.get(f"env:{project_env}", "platform") + if platform: + if not pm.get_package(platform): + already_up_to_date = False + PlatformPackageManager().install( + platform, + project_env=project_env, + project_targets=options.get("project_targets"), + skip_default_package=options.get("skip_dependencies"), + force=options.get("force"), + ) + return not already_up_to_date + + +def _install_project_env_libraries(project_env, options): + already_up_to_date = not options.get("force") + config = ProjectConfig.get_instance() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), project_env) + ) + if options.get("silent"): + lm.set_log_level(logging.WARN) + + # custom libraries + if options.get("libraries"): + if not options.get("silent"): + lm.set_log_level(logging.DEBUG) + for spec in options.get("libraries", []): + lm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + return not already_up_to_date + + if options.get("platforms") or options.get("tools"): + return False + + # if not custom platforms/tools, install declared libraries + for spec in config.get(f"env:{project_env}", "lib_deps"): + # skip built-in dependencies + if "/" not in spec: + continue + if not lm.get_package(spec): + already_up_to_date = False + lm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + return not already_up_to_date diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index c24c7038..165f7ff2 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -47,6 +47,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an with_all_packages=False, force=False, project_env=None, + project_targets=None, ): already_installed = self.get_package(spec) pkg = super(PlatformPackageManager, self).install( @@ -63,7 +64,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an p.pm.set_log_level(self.log.getEffectiveLevel()) if project_env: - p.configure_project_packages(project_env) + p.configure_project_packages(project_env, project_targets) if with_all_packages: with_packages = list(p.packages) diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index 66412bc0..b0feebb2 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -65,15 +65,6 @@ class PlatformPackagesMixin(object): result.append(item) return result - def autoinstall_required_packages(self): - for name, options in self.packages.items(): - if options.get("optional", False): - continue - if self.get_package(name): - continue - self.pm.install(self.get_package_spec(name)) - return True - def install_packages( # pylint: disable=too-many-arguments self, with_packages=None, diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index e436dcfe..37dee069 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -51,10 +51,7 @@ class PlatformRunMixin(object): assert isinstance(targets, list) self.ensure_engine_compatible() - self.configure_project_packages(variables["pioenv"], targets) - self.autoinstall_required_packages() - self._report_non_sensitive_data(variables["pioenv"], targets) self.silent = silent diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py new file mode 100644 index 00000000..e2ea7094 --- /dev/null +++ b/tests/commands/pkg/test_install.py @@ -0,0 +1,364 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import os + +import pytest + +from platformio import fs +from platformio.package.commands.install import package_install_cmd +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.project.config import ProjectConfig + +PROJECT_CONFIG_TPL = """ +[env] +platform = platformio/atmelavr@^3.4.0 +lib_deps = milesburton/DallasTemperature@^3.9.1 + +[env:baremetal] +board = uno + +[env:devkit] +framework = arduino +board = attiny88 +""" + + +def pkgs_to_names(pkgs): + return [pkg.metadata.name for pkg in pkgs] + + +def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp_path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "--skip-dependencies"], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + installed_lib_pkgs = LibraryPackageManager( + os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") + ).get_installed() + assert pkgs_to_names(installed_lib_pkgs) == ["DallasTemperature"] + assert len(ToolPackageManager().get_installed()) == 0 + + +def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "baremetal"], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + installed_lib_pkgs = LibraryPackageManager( + os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "baremetal") + ).get_installed() + assert pkgs_to_names(installed_lib_pkgs) == ["DallasTemperature", "OneWire"] + assert pkgs_to_names(ToolPackageManager().get_installed()) == [ + "toolchain-atmelavr" + ] + + +def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + lm = LibraryPackageManager( + os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_names(lm.get_installed()) == ["DallasTemperature", "OneWire"] + assert pkgs_to_names(ToolPackageManager().get_installed()) == [ + "framework-arduino-avr-attiny", + "toolchain-atmelavr", + ] + + +def test_unknown_project_dependencies( + clirunner, validate_cliresult, isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text( + """ +[env:unknown_platform] +platform = unknown_platform + +[env:unknown_lib_deps] +lib_deps = SPI, platformio/unknown_library +""" + ) + with fs.cd(str(project_dir)): + result = clirunner.invoke( + package_install_cmd, + ["-e", "unknown_platform"], + ) + with pytest.raises( + AssertionError, + match=("Could not find the package with 'unknown_platform' requirements"), + ): + validate_cliresult(result) + + # unknown libraries + result = clirunner.invoke( + package_install_cmd, + ["-e", "unknown_lib_deps"], + ) + with pytest.raises( + AssertionError, + match=( + "Could not find the package with 'platformio/unknown_library' requirements" + ), + ): + validate_cliresult(result) + + +def test_custom_project_libraries( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "bblanchon/ArduinoJson@^6.19.2" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-l", spec], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + # try again + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-l", spec], + ) + validate_cliresult(result) + assert "already installed" in result.output + # try again in the silent mode + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-l", spec, "--silent"], + ) + validate_cliresult(result) + assert not result.output.strip() + + # check folders + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_names(lm.get_installed()) == ["ArduinoJson"] + # do not expect any platforms/tools + assert not os.path.exists(config.get("platformio", "platforms_dir")) + assert not os.path.exists(config.get("platformio", "packages_dir")) + + # unknown libraries + result = clirunner.invoke( + package_install_cmd, ["-l", "platformio/unknown_library"] + ) + with pytest.raises( + AssertionError, + match=( + "Could not find the package with " + "'platformio/unknown_library' requirements" + ), + ): + validate_cliresult(result) + + +def test_custom_project_tools( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "platformio/tool-openocd" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-t", spec], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + # try again + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-t", spec], + ) + validate_cliresult(result) + assert "already installed" in result.output + # try again in the silent mode + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-t", spec, "--silent"], + ) + validate_cliresult(result) + assert not result.output.strip() + + config = ProjectConfig() + assert pkgs_to_names(ToolPackageManager().get_installed()) == ["tool-openocd"] + assert not LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ).get_installed() + # do not expect any platforms + assert not os.path.exists(config.get("platformio", "platforms_dir")) + + # unknown tool + result = clirunner.invoke( + package_install_cmd, ["-t", "platformio/unknown_tool"] + ) + with pytest.raises( + AssertionError, + match=( + "Could not find the package with " + "'platformio/unknown_tool' requirements" + ), + ): + validate_cliresult(result) + + +def test_custom_project_platforms( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "atmelavr" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-p", spec, "--skip-dependencies"], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + # try again + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-p", spec, "--skip-dependencies"], + ) + validate_cliresult(result) + assert "already installed" in result.output + # try again in the silent mode + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-p", spec, "--silent", "--skip-dependencies"], + ) + validate_cliresult(result) + assert not result.output.strip() + + config = ProjectConfig() + assert pkgs_to_names(PlatformPackageManager().get_installed()) == ["atmelavr"] + assert not LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ).get_installed() + # do not expect any packages + assert not os.path.exists(config.get("platformio", "packages_dir")) + + # unknown platform + result = clirunner.invoke(package_install_cmd, ["-p", "unknown_platform"]) + with pytest.raises( + AssertionError, + match="Could not find the package with 'unknown_platform' requirements", + ): + validate_cliresult(result) + + # incompatible board + result = clirunner.invoke(package_install_cmd, ["-e", "devkit", "-p", "sifive"]) + with pytest.raises( + AssertionError, + match="Unknown board ID", + ): + validate_cliresult(result) + + +def test_global_packages( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + # libraries + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "-l", + "milesburton/DallasTemperature@^3.9.1", + "--skip-dependencies", + ], + ) + validate_cliresult(result) + assert pkgs_to_names(LibraryPackageManager().get_installed()) == [ + "DallasTemperature" + ] + # with dependencies + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "-l", + "milesburton/DallasTemperature@^3.9.1", + "-l", + "bblanchon/ArduinoJson@^6.19.2", + ], + ) + validate_cliresult(result) + assert pkgs_to_names(LibraryPackageManager().get_installed()) == [ + "ArduinoJson", + "DallasTemperature", + "OneWire", + ] + # custom storage + storage_dir = tmp_path / "custom_lib_storage" + storage_dir.mkdir() + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "--storage-dir", + str(storage_dir), + "-l", + "bblanchon/ArduinoJson@^6.19.2", + ], + ) + validate_cliresult(result) + assert pkgs_to_names(LibraryPackageManager(storage_dir).get_installed()) == [ + "ArduinoJson" + ] + + # tools + result = clirunner.invoke( + package_install_cmd, + ["--global", "-t", "platformio/framework-arduino-avr-attiny@^1.5.2"], + ) + validate_cliresult(result) + assert pkgs_to_names(ToolPackageManager().get_installed()) == [ + "framework-arduino-avr-attiny" + ] + + # platforms + result = clirunner.invoke( + package_install_cmd, + ["--global", "-p", "platformio/atmelavr@^3.4.0", "--skip-dependencies"], + ) + validate_cliresult(result) + assert pkgs_to_names(PlatformPackageManager().get_installed()) == ["atmelavr"] diff --git a/tests/conftest.py b/tests/conftest.py index 368e2e11..1a9fd670 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -72,18 +72,28 @@ def strip_ansi(): return decorator -@pytest.fixture(scope="module") -def isolated_pio_core(request, tmpdir_factory): +def _isolated_pio_core(request, tmpdir_factory): core_dir = tmpdir_factory.mktemp(".platformio") os.environ["PLATFORMIO_CORE_DIR"] = str(core_dir) def fin(): - del os.environ["PLATFORMIO_CORE_DIR"] + if "PLATFORMIO_CORE_DIR" in os.environ: + del os.environ["PLATFORMIO_CORE_DIR"] request.addfinalizer(fin) return core_dir +@pytest.fixture(scope="module") +def isolated_pio_core(request, tmpdir_factory): + return _isolated_pio_core(request, tmpdir_factory) + + +@pytest.fixture(scope="function") +def func_isolated_pio_core(request, tmpdir_factory): + return _isolated_pio_core(request, tmpdir_factory) + + @pytest.fixture(scope="function") def without_internet(monkeypatch): monkeypatch.setattr(http, "_internet_on", lambda: False)