Implement unified "pio pkg install" CLI // Issue #3373

HISTORY.rst

@@ -15,8 +15,9 @@ PlatformIO Core 5
 
   - New unified Package Management CLI (``pio pkg``):
 
-    * `pio pkg outdated <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_pack.html>`__ - check for project outdated packages
     * `pio pkg exec <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_exec.html>`_ - run command from package tool (`issue #4163 <https://github.com/platformio/platformio-core/issues/4163>`_)
+    * `pio pkg install <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html>`_ - install the project dependencies or custom packages
+    * `pio pkg outdated <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_pack.html>`__ - check for project outdated packages
 
   - Added support for dependencies declared in a "tool" type package
   - Ignore files according to the patterns declared in ".gitignore" when using `pio package pack <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_pack.html>`__ command (`issue #4188 <https://github.com/platformio/platformio-core/issues/4188>`_)

platformio/commands/pkg.py

@@ -15,6 +15,7 @@
 import click
 
 from platformio.package.commands.exec import package_exec_cmd
+from platformio.package.commands.install import package_install_cmd
 from platformio.package.commands.outdated import package_outdated_cmd
 from platformio.package.commands.pack import package_pack_cmd
 from platformio.package.commands.publish import package_publish_cmd
@@ -25,6 +26,7 @@ from platformio.package.commands.unpublish import package_unpublish_cmd
     "pkg",
     commands=[
         package_exec_cmd,
+        package_install_cmd,
         package_outdated_cmd,
         package_pack_cmd,
         package_publish_cmd,

platformio/commands/run/processor.py

@@ -14,6 +14,7 @@
 
 from platformio.commands.platform import init_platform
 from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME
+from platformio.package.commands.install import install_project_env_dependencies
 from platformio.project.exception import UndefinedEnvPlatformError
 
 # pylint: disable=too-many-instance-attributes
@@ -64,6 +65,11 @@ class EnvironmentProcessor(object):
         if "monitor" in build_targets:
             build_targets.remove("monitor")
 
+        install_project_env_dependencies(
+            self.name,
+            {"project_targets": build_targets},
+        )
+
         result = init_platform(self.options["platform"]).run(
             build_vars, build_targets, self.silent, self.verbose, self.jobs
         )

platformio/package/commands/install.py

@@ -0,0 +1,194 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+import click
+
+from platformio import fs
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.project.config import ProjectConfig
+
+
+@click.command(
+    "install", short_help="Install the project dependencies or custom packages"
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option("-e", "--environment", "environments", multiple=True)
+@click.option("-p", "--platform", "platforms", multiple=True)
+@click.option("-t", "--tool", "tools", multiple=True)
+@click.option("-l", "--library", "libraries", multiple=True)
+@click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies")
+@click.option("-g", "--global", is_flag=True, help="Install package globally")
+@click.option(
+    "--storage-dir",
+    default=None,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    help="Custom package storage directory",
+)
+@click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists")
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
+def package_install_cmd(**options):
+    if options.get("global"):
+        install_global_dependencies(options)
+    else:
+        install_project_dependencies(options)
+
+
+def install_global_dependencies(options):
+    pm = PlatformPackageManager(options.get("storage_dir"))
+    tm = ToolPackageManager(options.get("storage_dir"))
+    lm = LibraryPackageManager(options.get("storage_dir"))
+    for obj in (pm, tm, lm):
+        obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG)
+    for spec in options.get("platforms"):
+        pm.install(
+            spec,
+            skip_default_package=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+    for spec in options.get("tools"):
+        tm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+    for spec in options.get("libraries", []):
+        lm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+
+
+def install_project_dependencies(options):
+    environments = options["environments"]
+    with fs.cd(options["project_dir"]):
+        config = ProjectConfig.get_instance()
+        config.validate(environments)
+        for env in config.envs():
+            if environments and env not in environments:
+                continue
+            if not options["silent"]:
+                click.echo(
+                    "Resolving %s environment packages..." % click.style(env, fg="cyan")
+                )
+            already_up_to_date = install_project_env_dependencies(env, options)
+            if not options["silent"] and already_up_to_date:
+                click.secho("Already up-to-date.", fg="green")
+
+
+def install_project_env_dependencies(project_env, options=None):
+    """Used in `pio run` -> Processor"""
+    options = options or {}
+    return any(
+        [
+            _install_project_env_platform(project_env, options),
+            _install_project_env_libraries(project_env, options),
+        ]
+    )
+
+
+def _install_project_env_platform(project_env, options):
+    already_up_to_date = not options.get("force")
+    config = ProjectConfig.get_instance()
+    pm = PlatformPackageManager()
+    if options.get("silent"):
+        pm.set_log_level(logging.WARN)
+
+    if options.get("platforms") or options.get("tools"):
+        already_up_to_date = False
+        tm = ToolPackageManager()
+        if not options.get("silent"):
+            pm.set_log_level(logging.DEBUG)
+            tm.set_log_level(logging.DEBUG)
+        for platform in options.get("platforms"):
+            pm.install(
+                platform,
+                project_env=project_env,
+                project_targets=options.get("project_targets"),
+                skip_default_package=options.get("skip_dependencies"),
+                force=options.get("force"),
+            )
+        for spec in options.get("tools"):
+            tm.install(
+                spec,
+                skip_dependencies=options.get("skip_dependencies"),
+                force=options.get("force"),
+            )
+        return not already_up_to_date
+
+    if options.get("libraries"):
+        return False
+
+    # if not custom libraries, install declared platform
+    platform = config.get(f"env:{project_env}", "platform")
+    if platform:
+        if not pm.get_package(platform):
+            already_up_to_date = False
+        PlatformPackageManager().install(
+            platform,
+            project_env=project_env,
+            project_targets=options.get("project_targets"),
+            skip_default_package=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+    return not already_up_to_date
+
+
+def _install_project_env_libraries(project_env, options):
+    already_up_to_date = not options.get("force")
+    config = ProjectConfig.get_instance()
+    lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "libdeps_dir"), project_env)
+    )
+    if options.get("silent"):
+        lm.set_log_level(logging.WARN)
+
+    # custom libraries
+    if options.get("libraries"):
+        if not options.get("silent"):
+            lm.set_log_level(logging.DEBUG)
+        for spec in options.get("libraries", []):
+            lm.install(
+                spec,
+                skip_dependencies=options.get("skip_dependencies"),
+                force=options.get("force"),
+            )
+        return not already_up_to_date
+
+    if options.get("platforms") or options.get("tools"):
+        return False
+
+    # if not custom platforms/tools, install declared libraries
+    for spec in config.get(f"env:{project_env}", "lib_deps"):
+        # skip built-in dependencies
+        if "/" not in spec:
+            continue
+        if not lm.get_package(spec):
+            already_up_to_date = False
+        lm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+    return not already_up_to_date

platformio/package/manager/platform.py

@@ -47,6 +47,7 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
         with_all_packages=False,
         force=False,
         project_env=None,
+        project_targets=None,
     ):
         already_installed = self.get_package(spec)
         pkg = super(PlatformPackageManager, self).install(
@@ -63,7 +64,7 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
         p.pm.set_log_level(self.log.getEffectiveLevel())
 
         if project_env:
-            p.configure_project_packages(project_env)
+            p.configure_project_packages(project_env, project_targets)
 
         if with_all_packages:
             with_packages = list(p.packages)

platformio/platform/_packages.py

@@ -65,15 +65,6 @@ class PlatformPackagesMixin(object):
             result.append(item)
         return result
 
-    def autoinstall_required_packages(self):
-        for name, options in self.packages.items():
-            if options.get("optional", False):
-                continue
-            if self.get_package(name):
-                continue
-            self.pm.install(self.get_package_spec(name))
-        return True
-
     def install_packages(  # pylint: disable=too-many-arguments
         self,
         with_packages=None,

platformio/platform/_run.py

@@ -51,10 +51,7 @@ class PlatformRunMixin(object):
         assert isinstance(targets, list)
 
         self.ensure_engine_compatible()
-
         self.configure_project_packages(variables["pioenv"], targets)
-        self.autoinstall_required_packages()
-
         self._report_non_sensitive_data(variables["pioenv"], targets)
 
         self.silent = silent

tests/commands/pkg/test_install.py

@@ -0,0 +1,364 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-argument
+
+import os
+
+import pytest
+
+from platformio import fs
+from platformio.package.commands.install import package_install_cmd
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.project.config import ProjectConfig
+
+PROJECT_CONFIG_TPL = """
+[env]
+platform = platformio/atmelavr@^3.4.0
+lib_deps = milesburton/DallasTemperature@^3.9.1
+
+[env:baremetal]
+board = uno
+
+[env:devkit]
+framework = arduino
+board = attiny88
+"""
+
+
+def pkgs_to_names(pkgs):
+    return [pkg.metadata.name for pkg in pkgs]
+
+
+def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp_path):
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL)
+    result = clirunner.invoke(
+        package_install_cmd,
+        ["-d", str(project_dir), "-e", "devkit", "--skip-dependencies"],
+    )
+    validate_cliresult(result)
+    with fs.cd(str(project_dir)):
+        installed_lib_pkgs = LibraryPackageManager(
+            os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit")
+        ).get_installed()
+        assert pkgs_to_names(installed_lib_pkgs) == ["DallasTemperature"]
+        assert len(ToolPackageManager().get_installed()) == 0
+
+
+def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path):
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL)
+    result = clirunner.invoke(
+        package_install_cmd,
+        ["-d", str(project_dir), "-e", "baremetal"],
+    )
+    validate_cliresult(result)
+    with fs.cd(str(project_dir)):
+        installed_lib_pkgs = LibraryPackageManager(
+            os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "baremetal")
+        ).get_installed()
+        assert pkgs_to_names(installed_lib_pkgs) == ["DallasTemperature", "OneWire"]
+        assert pkgs_to_names(ToolPackageManager().get_installed()) == [
+            "toolchain-atmelavr"
+        ]
+
+
+def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path):
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL)
+    result = clirunner.invoke(
+        package_install_cmd,
+        ["-d", str(project_dir)],
+    )
+    validate_cliresult(result)
+    with fs.cd(str(project_dir)):
+        lm = LibraryPackageManager(
+            os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit")
+        )
+        assert pkgs_to_names(lm.get_installed()) == ["DallasTemperature", "OneWire"]
+        assert pkgs_to_names(ToolPackageManager().get_installed()) == [
+            "framework-arduino-avr-attiny",
+            "toolchain-atmelavr",
+        ]
+
+
+def test_unknown_project_dependencies(
+    clirunner, validate_cliresult, isolated_pio_core, tmp_path
+):
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "platformio.ini").write_text(
+        """
+[env:unknown_platform]
+platform = unknown_platform
+
+[env:unknown_lib_deps]
+lib_deps = SPI, platformio/unknown_library
+"""
+    )
+    with fs.cd(str(project_dir)):
+        result = clirunner.invoke(
+            package_install_cmd,
+            ["-e", "unknown_platform"],
+        )
+        with pytest.raises(
+            AssertionError,
+            match=("Could not find the package with 'unknown_platform' requirements"),
+        ):
+            validate_cliresult(result)
+
+        # unknown libraries
+        result = clirunner.invoke(
+            package_install_cmd,
+            ["-e", "unknown_lib_deps"],
+        )
+        with pytest.raises(
+            AssertionError,
+            match=(
+                "Could not find the package with 'platformio/unknown_library' requirements"
+            ),
+        ):
+            validate_cliresult(result)
+
+
+def test_custom_project_libraries(
+    clirunner, validate_cliresult, func_isolated_pio_core, tmp_path
+):
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL)
+    spec = "bblanchon/ArduinoJson@^6.19.2"
+    result = clirunner.invoke(
+        package_install_cmd,
+        ["-d", str(project_dir), "-e", "devkit", "-l", spec],
+    )
+    validate_cliresult(result)
+    with fs.cd(str(project_dir)):
+        # try again
+        result = clirunner.invoke(
+            package_install_cmd,
+            ["-e", "devkit", "-l", spec],
+        )
+        validate_cliresult(result)
+        assert "already installed" in result.output
+        # try again in the silent mode
+        result = clirunner.invoke(
+            package_install_cmd,
+            ["-e", "devkit", "-l", spec, "--silent"],
+        )
+        validate_cliresult(result)
+        assert not result.output.strip()
+
+        # check folders
+        config = ProjectConfig()
+        lm = LibraryPackageManager(
+            os.path.join(config.get("platformio", "libdeps_dir"), "devkit")
+        )
+        assert pkgs_to_names(lm.get_installed()) == ["ArduinoJson"]
+        # do not expect any platforms/tools
+        assert not os.path.exists(config.get("platformio", "platforms_dir"))
+        assert not os.path.exists(config.get("platformio", "packages_dir"))
+
+        # unknown libraries
+        result = clirunner.invoke(
+            package_install_cmd, ["-l", "platformio/unknown_library"]
+        )
+        with pytest.raises(
+            AssertionError,
+            match=(
+                "Could not find the package with "
+                "'platformio/unknown_library' requirements"
+            ),
+        ):
+            validate_cliresult(result)
+
+
+def test_custom_project_tools(
+    clirunner, validate_cliresult, func_isolated_pio_core, tmp_path
+):
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL)
+    spec = "platformio/tool-openocd"
+    result = clirunner.invoke(
+        package_install_cmd,
+        ["-d", str(project_dir), "-e", "devkit", "-t", spec],
+    )
+    validate_cliresult(result)
+    with fs.cd(str(project_dir)):
+        # try again
+        result = clirunner.invoke(
+            package_install_cmd,
+            ["-e", "devkit", "-t", spec],
+        )
+        validate_cliresult(result)
+        assert "already installed" in result.output
+        # try again in the silent mode
+        result = clirunner.invoke(
+            package_install_cmd,
+            ["-e", "devkit", "-t", spec, "--silent"],
+        )
+        validate_cliresult(result)
+        assert not result.output.strip()
+
+        config = ProjectConfig()
+        assert pkgs_to_names(ToolPackageManager().get_installed()) == ["tool-openocd"]
+        assert not LibraryPackageManager(
+            os.path.join(config.get("platformio", "libdeps_dir"), "devkit")
+        ).get_installed()
+        # do not expect any platforms
+        assert not os.path.exists(config.get("platformio", "platforms_dir"))
+
+        # unknown tool
+        result = clirunner.invoke(
+            package_install_cmd, ["-t", "platformio/unknown_tool"]
+        )
+        with pytest.raises(
+            AssertionError,
+            match=(
+                "Could not find the package with "
+                "'platformio/unknown_tool' requirements"
+            ),
+        ):
+            validate_cliresult(result)
+
+
+def test_custom_project_platforms(
+    clirunner, validate_cliresult, func_isolated_pio_core, tmp_path
+):
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL)
+    spec = "atmelavr"
+    result = clirunner.invoke(
+        package_install_cmd,
+        ["-d", str(project_dir), "-e", "devkit", "-p", spec, "--skip-dependencies"],
+    )
+    validate_cliresult(result)
+    with fs.cd(str(project_dir)):
+        # try again
+        result = clirunner.invoke(
+            package_install_cmd,
+            ["-e", "devkit", "-p", spec, "--skip-dependencies"],
+        )
+        validate_cliresult(result)
+        assert "already installed" in result.output
+        # try again in the silent mode
+        result = clirunner.invoke(
+            package_install_cmd,
+            ["-e", "devkit", "-p", spec, "--silent", "--skip-dependencies"],
+        )
+        validate_cliresult(result)
+        assert not result.output.strip()
+
+        config = ProjectConfig()
+        assert pkgs_to_names(PlatformPackageManager().get_installed()) == ["atmelavr"]
+        assert not LibraryPackageManager(
+            os.path.join(config.get("platformio", "libdeps_dir"), "devkit")
+        ).get_installed()
+        # do not expect any packages
+        assert not os.path.exists(config.get("platformio", "packages_dir"))
+
+        # unknown platform
+        result = clirunner.invoke(package_install_cmd, ["-p", "unknown_platform"])
+        with pytest.raises(
+            AssertionError,
+            match="Could not find the package with 'unknown_platform' requirements",
+        ):
+            validate_cliresult(result)
+
+        # incompatible board
+        result = clirunner.invoke(package_install_cmd, ["-e", "devkit", "-p", "sifive"])
+        with pytest.raises(
+            AssertionError,
+            match="Unknown board ID",
+        ):
+            validate_cliresult(result)
+
+
+def test_global_packages(
+    clirunner, validate_cliresult, func_isolated_pio_core, tmp_path
+):
+    # libraries
+    result = clirunner.invoke(
+        package_install_cmd,
+        [
+            "--global",
+            "-l",
+            "milesburton/DallasTemperature@^3.9.1",
+            "--skip-dependencies",
+        ],
+    )
+    validate_cliresult(result)
+    assert pkgs_to_names(LibraryPackageManager().get_installed()) == [
+        "DallasTemperature"
+    ]
+    # with dependencies
+    result = clirunner.invoke(
+        package_install_cmd,
+        [
+            "--global",
+            "-l",
+            "milesburton/DallasTemperature@^3.9.1",
+            "-l",
+            "bblanchon/ArduinoJson@^6.19.2",
+        ],
+    )
+    validate_cliresult(result)
+    assert pkgs_to_names(LibraryPackageManager().get_installed()) == [
+        "ArduinoJson",
+        "DallasTemperature",
+        "OneWire",
+    ]
+    # custom storage
+    storage_dir = tmp_path / "custom_lib_storage"
+    storage_dir.mkdir()
+    result = clirunner.invoke(
+        package_install_cmd,
+        [
+            "--global",
+            "--storage-dir",
+            str(storage_dir),
+            "-l",
+            "bblanchon/ArduinoJson@^6.19.2",
+        ],
+    )
+    validate_cliresult(result)
+    assert pkgs_to_names(LibraryPackageManager(storage_dir).get_installed()) == [
+        "ArduinoJson"
+    ]
+
+    # tools
+    result = clirunner.invoke(
+        package_install_cmd,
+        ["--global", "-t", "platformio/framework-arduino-avr-attiny@^1.5.2"],
+    )
+    validate_cliresult(result)
+    assert pkgs_to_names(ToolPackageManager().get_installed()) == [
+        "framework-arduino-avr-attiny"
+    ]
+
+    # platforms
+    result = clirunner.invoke(
+        package_install_cmd,
+        ["--global", "-p", "platformio/atmelavr@^3.4.0", "--skip-dependencies"],
+    )
+    validate_cliresult(result)
+    assert pkgs_to_names(PlatformPackageManager().get_installed()) == ["atmelavr"]

tests/conftest.py

@@ -72,18 +72,28 @@ def strip_ansi():
     return decorator
 
 
-@pytest.fixture(scope="module")
+def _isolated_pio_core(request, tmpdir_factory):
-def isolated_pio_core(request, tmpdir_factory):
     core_dir = tmpdir_factory.mktemp(".platformio")
     os.environ["PLATFORMIO_CORE_DIR"] = str(core_dir)
 
     def fin():
-        del os.environ["PLATFORMIO_CORE_DIR"]
+        if "PLATFORMIO_CORE_DIR" in os.environ:
+            del os.environ["PLATFORMIO_CORE_DIR"]
 
     request.addfinalizer(fin)
     return core_dir
 
 
+@pytest.fixture(scope="module")
+def isolated_pio_core(request, tmpdir_factory):
+    return _isolated_pio_core(request, tmpdir_factory)
+
+
+@pytest.fixture(scope="function")
+def func_isolated_pio_core(request, tmpdir_factory):
+    return _isolated_pio_core(request, tmpdir_factory)
+
+
 @pytest.fixture(scope="function")
 def without_internet(monkeypatch):
     monkeypatch.setattr(http, "_internet_on", lambda: False)