New options for system prune command: remove unnecessary core and development platform packages // Resolve #923

HISTORY.rst

@@ -11,16 +11,30 @@ PlatformIO Core 5
 5.0.5 (2021-??-??)
 ~~~~~~~~~~~~~~~~~~
 
-* Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O
+* **Build System**
-* Improved listing of `multicast DNS services <https://docs.platformio.org/page/core/userguide/device/cmd_list.html>`_
+
-* Check for debugging server's "ready_pattern" in "stderr"
+  - Upgraded build engine to the SCons 4.1 (`release notes <https://scons.org/scons-410-is-available.html>`_)
-* Upgraded build engine to the SCons 4.1 (`release notes <https://scons.org/scons-410-is-available.html>`_)
+  - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
-* Disabled automatic removal of unnecessary development platform packages (`issue #3708 <https://github.com/platformio/platformio-core/issues/3708>`_, `issue #3770 <https://github.com/platformio/platformio-core/issues/3770>`_)
+  - Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json <https://docs.platformio.org/page/librarymanager/config.html>`__ manifest (`issue #3806 <https://github.com/platformio/platformio-core/issues/3806>`_)
-* Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 <https://github.com/platformio/platformio-core/issues/3804>`_)
+
-* Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
+* **Package Management System**
-* Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json <https://docs.platformio.org/page/librarymanager/config.html>`__ manifest (`issue #3806 <https://github.com/platformio/platformio-core/issues/3806>`_)
+
-* Fixed an issue with compiler driver for ".ccls" language server (`issue #3808 <https://github.com/platformio/platformio-core/issues/3808>`_)
+  - New options for `system prune <https://docs.platformio.org/page/core/userguide/system/cmd_prune.html>`__ command:
-* Fixed an issue when unnecessary packages were removed in  ``update --dry-run`` mode (`issue #3809 <https://github.com/platformio/platformio-core/issues/3809>`_)
+
+    + ``--dry-run`` option to show data that will be removed
+    + ``--core-packages`` option to remove unnecessary core packages
+    + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 <https://github.com/platformio/platformio-core/issues/923>`_)
+
+  - Disabled automatic removal of unnecessary development platform packages (`issue #3708 <https://github.com/platformio/platformio-core/issues/3708>`_, `issue #3770 <https://github.com/platformio/platformio-core/issues/3770>`_)
+  - Fixed an issue when unnecessary packages were removed in  ``update --dry-run`` mode (`issue #3809 <https://github.com/platformio/platformio-core/issues/3809>`_)
+
+* **Miscellaneous**
+
+  - Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O
+  - Improved listing of `multicast DNS services <https://docs.platformio.org/page/core/userguide/device/cmd_list.html>`_
+  - Check for debugging server's "ready_pattern" in "stderr"
+  - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 <https://github.com/platformio/platformio-core/issues/3804>`_)
+  - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 <https://github.com/platformio/platformio-core/issues/3808>`_)
 
 5.0.4 (2020-12-30)
 ~~~~~~~~~~~~~~~~~~

platformio/commands/system/command.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import json
-import os
 import platform
 import subprocess
 import sys
@@ -27,11 +26,15 @@ from platformio.commands.system.completion import (
     install_completion_code,
     uninstall_completion_code,
 )
+from platformio.commands.system.prune import (
+    prune_cached_data,
+    prune_core_packages,
+    prune_platform_packages,
+)
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.manager.tool import ToolPackageManager
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import get_project_cache_dir
 
 
 @click.group("system", short_help="Miscellaneous system commands")
@@ -99,22 +102,49 @@ def system_info(json_output):
 
 @cli.command("prune", short_help="Remove unused data")
 @click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation")
-def system_prune(force):
+@click.option(
-    click.secho("WARNING! This will remove:", fg="yellow")
+    "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed"
-    click.echo(" - cached API requests")
+)
-    click.echo(" - cached package downloads")
+@click.option("--cache", is_flag=True, help="Prune only cached data")
-    click.echo(" - temporary data")
+@click.option(
-    if not force:
+    "--core-packages", is_flag=True, help="Prune only unnecessary core packages"
-        click.confirm("Do you want to continue?", abort=True)
+)
+@click.option(
+    "--platform-packages",
+    is_flag=True,
+    help="Prune only unnecessary development platform packages",
+)
+def system_prune(force, dry_run, cache, core_packages, platform_packages):
+    if dry_run:
+        click.secho(
+            "Dry run mode (do not prune, only show data that will be removed)",
+            fg="yellow",
+        )
+        click.echo()
 
-    reclaimed_total = 0
+    reclaimed_cache = 0
-    cache_dir = get_project_cache_dir()
+    reclaimed_core_packages = 0
-    if os.path.isdir(cache_dir):
+    reclaimed_platform_packages = 0
-        reclaimed_total += fs.calculate_folder_size(cache_dir)
+    prune_all = not any([cache, core_packages, platform_packages])
-        fs.rmtree(cache_dir)
+
+    if cache or prune_all:
+        reclaimed_cache = prune_cached_data(force, dry_run)
+        click.echo()
+
+    if core_packages or prune_all:
+        reclaimed_core_packages = prune_core_packages(force, dry_run)
+        click.echo()
+
+    if platform_packages or prune_all:
+        reclaimed_platform_packages = prune_platform_packages(force, dry_run)
+        click.echo()
 
     click.secho(
-        "Total reclaimed space: %s" % fs.humanize_file_size(reclaimed_total), fg="green"
+        "Total reclaimed space: %s"
+        % fs.humanize_file_size(
+            reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages
+        ),
+        fg="green",
     )
 
 

platformio/commands/system/prune.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from operator import itemgetter
+
+import click
+from tabulate import tabulate
+
+from platformio import fs
+from platformio.package.manager.core import remove_unnecessary_core_packages
+from platformio.package.manager.platform import remove_unnecessary_platform_packages
+from platformio.project.helpers import get_project_cache_dir
+
+
+def prune_cached_data(force, dry_run):
+    reclaimed_space = 0
+    click.secho("Prune cached data:", bold=True)
+    click.echo(" - cached API requests")
+    click.echo(" - cached package downloads")
+    click.echo(" - temporary data")
+    cache_dir = get_project_cache_dir()
+    if os.path.isdir(cache_dir):
+        reclaimed_space += fs.calculate_folder_size(cache_dir)
+        if not dry_run:
+            if not force:
+                click.confirm("Do you want to continue?", abort=True)
+            fs.rmtree(cache_dir)
+    click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space))
+    return reclaimed_space
+
+
+def prune_core_packages(force, dry_run):
+    click.secho("Prune unnecessary core packages:", bold=True)
+    return _prune_packages(force, dry_run, remove_unnecessary_core_packages)
+
+
+def prune_platform_packages(force, dry_run):
+    click.secho("Prune unnecessary development platform packages:", bold=True)
+    return _prune_packages(force, dry_run, remove_unnecessary_platform_packages)
+
+
+def _prune_packages(force, dry_run, handler):
+    click.echo("Calculating...")
+    items = [
+        (
+            pkg,
+            fs.calculate_folder_size(pkg.path),
+        )
+        for pkg in handler(dry_run=True)
+    ]
+    items = sorted(items, key=itemgetter(1), reverse=True)
+    reclaimed_space = sum([item[1] for item in items])
+    if items:
+        click.echo(
+            tabulate(
+                [
+                    (
+                        pkg.metadata.spec.humanize(),
+                        str(pkg.metadata.version),
+                        fs.humanize_file_size(size),
+                    )
+                    for (pkg, size) in items
+                ],
+                headers=["Package", "Version", "Size"],
+            )
+        )
+    if not dry_run:
+        if not force:
+            click.confirm("Do you want to continue?", abort=True)
+        handler(dry_run=False)
+    click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space))
+    return reclaimed_space

platformio/package/manager/core.py

@@ -27,6 +27,17 @@ from platformio.package.meta import PackageItem, PackageSpec
 from platformio.proc import get_pythonexe_path
 
 
+def get_installed_core_packages():
+    result = []
+    pm = ToolPackageManager()
+    for name, requirements in __core_packages__.items():
+        spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
+        pkg = pm.get_package(spec)
+        if pkg:
+            result.append(pkg)
+    return result
+
+
 def get_core_package_dir(name, auto_install=True):
     if name not in __core_packages__:
         raise exception.PlatformioException("Please upgrade PlatformIO Core")
@@ -40,7 +51,7 @@ def get_core_package_dir(name, auto_install=True):
     if not auto_install:
         return None
     assert pm.install(spec)
-    _remove_unnecessary_packages()
+    remove_unnecessary_core_packages()
     return pm.get_package(spec).path
 
 
@@ -54,24 +65,40 @@ def update_core_packages(only_check=False, silent=False):
         if not silent or pm.outdated(pkg, spec).is_outdated():
             pm.update(pkg, spec, only_check=only_check)
     if not only_check:
-        _remove_unnecessary_packages()
+        remove_unnecessary_core_packages()
     return True
 
 
-def _remove_unnecessary_packages():
+def remove_unnecessary_core_packages(dry_run=False):
+    candidates = []
     pm = ToolPackageManager()
     best_pkg_versions = {}
+
     for name, requirements in __core_packages__.items():
         spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
         pkg = pm.get_package(spec)
         if not pkg:
             continue
         best_pkg_versions[pkg.metadata.name] = pkg.metadata.version
+
     for pkg in pm.get_installed():
-        if pkg.metadata.name not in best_pkg_versions:
+        skip_conds = [
-            continue
+            os.path.isfile(os.path.join(pkg.path, ".piokeep")),
-        if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]:
+            pkg.metadata.spec.owner != "platformio",
-            pm.uninstall(pkg)
+            pkg.metadata.name not in best_pkg_versions,
+            pkg.metadata.name in best_pkg_versions
+            and pkg.metadata.version == best_pkg_versions[pkg.metadata.name],
+        ]
+        if not any(skip_conds):
+            candidates.append(pkg)
+
+    if dry_run:
+        return candidates
+
+    for pkg in candidates:
+        pm.uninstall(pkg)
+
+    return candidates
 
 
 def inject_contrib_pysite(verify_openssl=False):

platformio/package/manager/platform.py

@@ -12,10 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+
 from platformio import util
 from platformio.clients.http import HTTPClientError, InternetIsOffline
 from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.base import BasePackageManager
+from platformio.package.manager.core import get_installed_core_packages
 from platformio.package.manager.tool import ToolPackageManager
 from platformio.package.meta import PackageType
 from platformio.platform.exception import IncompatiblePlatform, UnknownBoard
@@ -164,3 +167,37 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
             ):
                 return manifest
         raise UnknownBoard(id_)
+
+
+#
+# Helpers
+#
+
+
+def remove_unnecessary_platform_packages(dry_run=False):
+    candidates = []
+    required = set()
+    core_packages = get_installed_core_packages()
+    for platform in PlatformPackageManager().get_installed():
+        p = PlatformFactory.new(platform)
+        for pkg in p.get_installed_packages(with_optional=True):
+            required.add(pkg)
+
+    pm = ToolPackageManager()
+    for pkg in pm.get_installed():
+        skip_conds = [
+            pkg.metadata.spec.url,
+            os.path.isfile(os.path.join(pkg.path, ".piokeep")),
+            pkg in required,
+            pkg in core_packages,
+        ]
+        if not any(skip_conds):
+            candidates.append(pkg)
+
+    if dry_run:
+        return candidates
+
+    for pkg in candidates:
+        pm.uninstall(pkg)
+
+    return candidates

platformio/package/meta.py

@@ -405,7 +405,12 @@ class PackageItem(object):
         )
 
     def __eq__(self, other):
-        return all([self.path == other.path, self.metadata == other.metadata])
+        if not self.path or not other.path:
+            return self.path == other.path
+        return os.path.realpath(self.path) == os.path.realpath(other.path)
+
+    def __hash__(self):
+        return hash(os.path.realpath(self.path))
 
     def exists(self):
         return os.path.isdir(self.path)

platformio/platform/_packages.py

@@ -17,18 +17,18 @@ from platformio.package.meta import PackageSpec
 
 
 class PlatformPackagesMixin(object):
-    def get_package_spec(self, name):
+    def get_package_spec(self, name, version=None):
-        version = self.packages[name].get("version", "")
+        version = version or self.packages[name].get("version")
-        if any(c in version for c in (":", "/", "@")):
+        if version and any(c in version for c in (":", "/", "@")):
             return PackageSpec("%s=%s" % (name, version))
         return PackageSpec(
             owner=self.packages[name].get("owner"), name=name, requirements=version
         )
 
-    def get_package(self, name):
+    def get_package(self, name, spec=None):
         if not name:
             return None
-        return self.pm.get_package(self.get_package_spec(name))
+        return self.pm.get_package(spec or self.get_package_spec(name))
 
     def get_package_dir(self, name):
         pkg = self.get_package(name)
@@ -38,12 +38,18 @@ class PlatformPackagesMixin(object):
         pkg = self.get_package(name)
         return str(pkg.metadata.version) if pkg else None
 
-    def get_installed_packages(self):
+    def get_installed_packages(self, with_optional=False):
         result = []
-        for name in self.packages:
+        for name, options in self.packages.items():
-            pkg = self.get_package(name)
+            versions = [options.get("version")]
-            if pkg:
+            if with_optional:
-                result.append(pkg)
+                versions.extend(options.get("optionalVersions", []))
+            for version in versions:
+                if not version:
+                    continue
+                pkg = self.get_package(name, self.get_package_spec(name, version))
+                if pkg:
+                    result.append(pkg)
         return result
 
     def dump_used_packages(self):