diff --git a/HISTORY.rst b/HISTORY.rst index b60d4a2e..c8949179 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,16 +11,30 @@ PlatformIO Core 5 5.0.5 (2021-??-??) ~~~~~~~~~~~~~~~~~~ -* Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O -* Improved listing of `multicast DNS services `_ -* Check for debugging server's "ready_pattern" in "stderr" -* Upgraded build engine to the SCons 4.1 (`release notes `_) -* Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) -* Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) -* Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) -* Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) -* Fixed an issue with compiler driver for ".ccls" language server (`issue #3808 `_) -* Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) +* **Build System** + + - Upgraded build engine to the SCons 4.1 (`release notes `_) + - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) + - Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) + +* **Package Management System** + + - New options for `system prune `__ command: + + + ``--dry-run`` option to show data that will be removed + + ``--core-packages`` option to remove unnecessary core packages + + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 `_) + + - Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) + - Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) + +* **Miscellaneous** + + - Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O + - Improved listing of `multicast DNS services `_ + - Check for debugging server's "ready_pattern" in "stderr" + - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) + - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 04a05d7f..23165e88 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 04a05d7f347764e24817e37de3fe26f701660645 +Subproject commit 23165e88d7ffa4a28ee8496d374df07b9e8d9910 diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index cb311205..d0c87d84 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -13,7 +13,6 @@ # limitations under the License. import json -import os import platform import subprocess import sys @@ -27,11 +26,15 @@ from platformio.commands.system.completion import ( install_completion_code, uninstall_completion_code, ) +from platformio.commands.system.prune import ( + prune_cached_data, + prune_core_packages, + prune_platform_packages, +) from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_cache_dir @click.group("system", short_help="Miscellaneous system commands") @@ -99,22 +102,49 @@ def system_info(json_output): @cli.command("prune", short_help="Remove unused data") @click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation") -def system_prune(force): - click.secho("WARNING! This will remove:", fg="yellow") - click.echo(" - cached API requests") - click.echo(" - cached package downloads") - click.echo(" - temporary data") - if not force: - click.confirm("Do you want to continue?", abort=True) +@click.option( + "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed" +) +@click.option("--cache", is_flag=True, help="Prune only cached data") +@click.option( + "--core-packages", is_flag=True, help="Prune only unnecessary core packages" +) +@click.option( + "--platform-packages", + is_flag=True, + help="Prune only unnecessary development platform packages", +) +def system_prune(force, dry_run, cache, core_packages, platform_packages): + if dry_run: + click.secho( + "Dry run mode (do not prune, only show data that will be removed)", + fg="yellow", + ) + click.echo() - reclaimed_total = 0 - cache_dir = get_project_cache_dir() - if os.path.isdir(cache_dir): - reclaimed_total += fs.calculate_folder_size(cache_dir) - fs.rmtree(cache_dir) + reclaimed_cache = 0 + reclaimed_core_packages = 0 + reclaimed_platform_packages = 0 + prune_all = not any([cache, core_packages, platform_packages]) + + if cache or prune_all: + reclaimed_cache = prune_cached_data(force, dry_run) + click.echo() + + if core_packages or prune_all: + reclaimed_core_packages = prune_core_packages(force, dry_run) + click.echo() + + if platform_packages or prune_all: + reclaimed_platform_packages = prune_platform_packages(force, dry_run) + click.echo() click.secho( - "Total reclaimed space: %s" % fs.humanize_file_size(reclaimed_total), fg="green" + "Total reclaimed space: %s" + % fs.humanize_file_size( + reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages + ), + fg="green", ) diff --git a/platformio/commands/system/prune.py b/platformio/commands/system/prune.py new file mode 100644 index 00000000..a6bfa1d6 --- /dev/null +++ b/platformio/commands/system/prune.py @@ -0,0 +1,84 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from operator import itemgetter + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.package.manager.core import remove_unnecessary_core_packages +from platformio.package.manager.platform import remove_unnecessary_platform_packages +from platformio.project.helpers import get_project_cache_dir + + +def prune_cached_data(force, dry_run): + reclaimed_space = 0 + click.secho("Prune cached data:", bold=True) + click.echo(" - cached API requests") + click.echo(" - cached package downloads") + click.echo(" - temporary data") + cache_dir = get_project_cache_dir() + if os.path.isdir(cache_dir): + reclaimed_space += fs.calculate_folder_size(cache_dir) + if not dry_run: + if not force: + click.confirm("Do you want to continue?", abort=True) + fs.rmtree(cache_dir) + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + return reclaimed_space + + +def prune_core_packages(force, dry_run): + click.secho("Prune unnecessary core packages:", bold=True) + return _prune_packages(force, dry_run, remove_unnecessary_core_packages) + + +def prune_platform_packages(force, dry_run): + click.secho("Prune unnecessary development platform packages:", bold=True) + return _prune_packages(force, dry_run, remove_unnecessary_platform_packages) + + +def _prune_packages(force, dry_run, handler): + click.echo("Calculating...") + items = [ + ( + pkg, + fs.calculate_folder_size(pkg.path), + ) + for pkg in handler(dry_run=True) + ] + items = sorted(items, key=itemgetter(1), reverse=True) + reclaimed_space = sum([item[1] for item in items]) + if items: + click.echo( + tabulate( + [ + ( + pkg.metadata.spec.humanize(), + str(pkg.metadata.version), + fs.humanize_file_size(size), + ) + for (pkg, size) in items + ], + headers=["Package", "Version", "Size"], + ) + ) + if not dry_run: + if not force: + click.confirm("Do you want to continue?", abort=True) + handler(dry_run=False) + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + return reclaimed_space diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index af51f91a..24d494b3 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -27,6 +27,17 @@ from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import get_pythonexe_path +def get_installed_core_packages(): + result = [] + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if pkg: + result.append(pkg) + return result + + def get_core_package_dir(name, auto_install=True): if name not in __core_packages__: raise exception.PlatformioException("Please upgrade PlatformIO Core") @@ -40,7 +51,7 @@ def get_core_package_dir(name, auto_install=True): if not auto_install: return None assert pm.install(spec) - _remove_unnecessary_packages() + remove_unnecessary_core_packages() return pm.get_package(spec).path @@ -54,24 +65,40 @@ def update_core_packages(only_check=False, silent=False): if not silent or pm.outdated(pkg, spec).is_outdated(): pm.update(pkg, spec, only_check=only_check) if not only_check: - _remove_unnecessary_packages() + remove_unnecessary_core_packages() return True -def _remove_unnecessary_packages(): +def remove_unnecessary_core_packages(dry_run=False): + candidates = [] pm = ToolPackageManager() best_pkg_versions = {} + for name, requirements in __core_packages__.items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if not pkg: continue best_pkg_versions[pkg.metadata.name] = pkg.metadata.version + for pkg in pm.get_installed(): - if pkg.metadata.name not in best_pkg_versions: - continue - if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]: - pm.uninstall(pkg) + skip_conds = [ + os.path.isfile(os.path.join(pkg.path, ".piokeep")), + pkg.metadata.spec.owner != "platformio", + pkg.metadata.name not in best_pkg_versions, + pkg.metadata.name in best_pkg_versions + and pkg.metadata.version == best_pkg_versions[pkg.metadata.name], + ] + if not any(skip_conds): + candidates.append(pkg) + + if dry_run: + return candidates + + for pkg in candidates: + pm.uninstall(pkg) + + return candidates def inject_contrib_pysite(verify_openssl=False): diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 5c52630d..efe8a361 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from platformio import util from platformio.clients.http import HTTPClientError, InternetIsOffline from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.core import get_installed_core_packages from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageType from platformio.platform.exception import IncompatiblePlatform, UnknownBoard @@ -164,3 +167,37 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an ): return manifest raise UnknownBoard(id_) + + +# +# Helpers +# + + +def remove_unnecessary_platform_packages(dry_run=False): + candidates = [] + required = set() + core_packages = get_installed_core_packages() + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for pkg in p.get_installed_packages(with_optional=True): + required.add(pkg) + + pm = ToolPackageManager() + for pkg in pm.get_installed(): + skip_conds = [ + pkg.metadata.spec.url, + os.path.isfile(os.path.join(pkg.path, ".piokeep")), + pkg in required, + pkg in core_packages, + ] + if not any(skip_conds): + candidates.append(pkg) + + if dry_run: + return candidates + + for pkg in candidates: + pm.uninstall(pkg) + + return candidates diff --git a/platformio/package/meta.py b/platformio/package/meta.py index edc5d0ff..156445d3 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -405,7 +405,12 @@ class PackageItem(object): ) def __eq__(self, other): - return all([self.path == other.path, self.metadata == other.metadata]) + if not self.path or not other.path: + return self.path == other.path + return os.path.realpath(self.path) == os.path.realpath(other.path) + + def __hash__(self): + return hash(os.path.realpath(self.path)) def exists(self): return os.path.isdir(self.path) diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index ac495b48..08c40c9b 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -17,18 +17,18 @@ from platformio.package.meta import PackageSpec class PlatformPackagesMixin(object): - def get_package_spec(self, name): - version = self.packages[name].get("version", "") - if any(c in version for c in (":", "/", "@")): + def get_package_spec(self, name, version=None): + version = version or self.packages[name].get("version") + if version and any(c in version for c in (":", "/", "@")): return PackageSpec("%s=%s" % (name, version)) return PackageSpec( owner=self.packages[name].get("owner"), name=name, requirements=version ) - def get_package(self, name): + def get_package(self, name, spec=None): if not name: return None - return self.pm.get_package(self.get_package_spec(name)) + return self.pm.get_package(spec or self.get_package_spec(name)) def get_package_dir(self, name): pkg = self.get_package(name) @@ -38,12 +38,18 @@ class PlatformPackagesMixin(object): pkg = self.get_package(name) return str(pkg.metadata.version) if pkg else None - def get_installed_packages(self): + def get_installed_packages(self, with_optional=False): result = [] - for name in self.packages: - pkg = self.get_package(name) - if pkg: - result.append(pkg) + for name, options in self.packages.items(): + versions = [options.get("version")] + if with_optional: + versions.extend(options.get("optionalVersions", [])) + for version in versions: + if not version: + continue + pkg = self.get_package(name, self.get_package_spec(name, version)) + if pkg: + result.append(pkg) return result def dump_used_packages(self):