From 2dd69e21c00f7e843521b7a998063d60ae803d43 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 20:17:07 +0300 Subject: [PATCH] Implement package removing with dependencies --- platformio/clients/http.py | 14 ++-- platformio/package/manager/_install.py | 69 +++--------------- platformio/package/manager/_registry.py | 29 +++++--- platformio/package/manager/_uninstall.py | 93 ++++++++++++++++++++++++ platformio/package/manager/base.py | 16 ++-- tests/package/test_manager.py | 28 ++++++- 6 files changed, 167 insertions(+), 82 deletions(-) create mode 100644 platformio/package/manager/_uninstall.py diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 3070f749..47f6c162 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -20,7 +20,13 @@ from platformio.exception import PlatformioException class HTTPClientError(PlatformioException): - pass + def __init__(self, message, response=None): + super(HTTPClientError, self).__init__() + self.message = message + self.response = response + + def __str__(self): # pragma: no cover + return self.message class HTTPClient(object): @@ -52,7 +58,7 @@ class HTTPClient(object): try: return getattr(self._session, method)(self.base_url + path, **kwargs) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - raise HTTPClientError(e) + raise HTTPClientError(str(e)) def request_json_data(self, *args, **kwargs): response = self.send_request(*args, **kwargs) @@ -69,6 +75,4 @@ class HTTPClient(object): message = response.json()["message"] except (KeyError, ValueError): message = response.text - exc = HTTPClientError(message) - exc.response = response - raise exc + raise HTTPClientError(message, response) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 58b1de1e..ea409f2e 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -20,7 +20,7 @@ import tempfile import click from platformio import app, compat, fs, util -from platformio.package.exception import PackageException, UnknownPackageError +from platformio.package.exception import PackageException from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -28,7 +28,7 @@ from platformio.package.vcsclient import VCSClientFactory class PackageManagerInstallMixin(object): - INSTALL_HISTORY = None # avoid circle dependencies + _INSTALL_HISTORY = None # avoid circle dependencies @staticmethod def unpack(src, dst): @@ -56,10 +56,10 @@ class PackageManagerInstallMixin(object): spec = self.ensure_spec(spec) # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = {} - if spec in self.INSTALL_HISTORY: - return self.INSTALL_HISTORY[spec] + if not self._INSTALL_HISTORY: + self._INSTALL_HISTORY = {} + if spec in self._INSTALL_HISTORY: + return self._INSTALL_HISTORY[spec] # check if package is already installed pkg = self.get_package(spec) @@ -105,11 +105,11 @@ class PackageManagerInstallMixin(object): ) self.memcache_reset() - self.install_dependencies(pkg, silent) - self.INSTALL_HISTORY[spec] = pkg + self._install_dependencies(pkg, silent) + self._INSTALL_HISTORY[spec] = pkg return pkg - def install_dependencies(self, pkg, silent=False): + def _install_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageSourceItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): @@ -117,14 +117,14 @@ class PackageManagerInstallMixin(object): if not silent: self.print_message(click.style("Installing dependencies...", fg="yellow")) for dependency in manifest.get("dependencies"): - if not self.install_dependency(dependency, silent) and not silent: + if not self._install_dependency(dependency, silent) and not silent: click.secho( "Warning! Could not install dependency %s for package '%s'" % (dependency, pkg.metadata.name), fg="yellow", ) - def install_dependency(self, dependency, silent=False): + def _install_dependency(self, dependency, silent=False): spec = PackageSpec( name=dependency.get("name"), requirements=dependency.get("version") ) @@ -247,50 +247,3 @@ class PackageManagerInstallMixin(object): _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) - - def uninstall(self, pkg, silent=False): - try: - self.lock() - - if not isinstance(pkg, PackageSourceItem): - pkg = ( - PackageSourceItem(pkg) - if os.path.isdir(pkg) - else self.get_package(pkg) - ) - if not pkg or not pkg.metadata: - raise UnknownPackageError(pkg) - - if not silent: - self.print_message( - "Uninstalling %s @ %s: \t" - % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), - nl=False, - ) - if os.path.islink(pkg.path): - os.unlink(pkg.path) - else: - fs.rmtree(pkg.path) - self.memcache_reset() - - # unfix detached-package with the same name - detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) - if ( - detached_pkg - and "@" in detached_pkg.path - and not os.path.isdir( - os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) - ) - ): - shutil.move( - detached_pkg.path, - os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), - ) - self.memcache_reset() - finally: - self.unlock() - - if not silent: - click.echo("[%s]" % click.style("OK", fg="green")) - - return True diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index a14bde98..d5c9ddad 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -78,12 +78,19 @@ class RegistryFileMirrorsIterator(object): class PackageManageRegistryMixin(object): def install_from_registry(self, spec, search_filters=None, silent=False): - packages = self.search_registry_packages(spec, search_filters) - if not packages: - raise UnknownPackageError(spec.humanize()) - if len(packages) > 1 and not silent: - self.print_multi_package_issue(packages, spec) - package, version = self.find_best_registry_version(packages, spec) + if spec.owner and spec.name and not search_filters: + package = self.fetch_registry_package(spec.owner, spec.name) + if not package: + raise UnknownPackageError(spec.humanize()) + version = self._pick_best_pkg_version(package["versions"], spec) + else: + packages = self.search_registry_packages(spec, search_filters) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + package, version = self.find_best_registry_version(packages, spec) + pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None if not pkgfile: raise UnknownPackageError(spec.humanize()) @@ -117,17 +124,17 @@ class PackageManageRegistryMixin(object): filters["ids"] = str(spec.id) else: filters["types"] = self.pkg_type - filters["names"] = '"%s"' % spec.name.lower() + filters["names"] = spec.name.lower() if spec.owner: filters["owners"] = spec.owner.lower() return self.get_registry_client_instance().list_packages(filters=filters)[ "items" ] - def fetch_registry_package_versions(self, owner, name): + def fetch_registry_package(self, owner, name): return self.get_registry_client_instance().get_package( self.pkg_type, owner, name - )["versions"] + ) @staticmethod def print_multi_package_issue(packages, spec): @@ -163,9 +170,9 @@ class PackageManageRegistryMixin(object): # if the custom version requirements, check ALL package versions for package in packages: version = self._pick_best_pkg_version( - self.fetch_registry_package_versions( + self.fetch_registry_package( package["owner"]["username"], package["name"] - ), + ).get("versions"), spec, ) if version: diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py new file mode 100644 index 00000000..e754eab2 --- /dev/null +++ b/platformio/package/manager/_uninstall.py @@ -0,0 +1,93 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil + +import click + +from platformio import fs +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSourceItem, PackageSpec + + +class PackageManagerUninstallMixin(object): + def uninstall(self, pkg, silent=False, skip_dependencies=False): + try: + self.lock() + return self._uninstall(pkg, silent, skip_dependencies) + finally: + self.unlock() + + def _uninstall(self, pkg, silent=False, skip_dependencies=False): + if not isinstance(pkg, PackageSourceItem): + pkg = ( + PackageSourceItem(pkg) if os.path.isdir(pkg) else self.get_package(pkg) + ) + if not pkg or not pkg.metadata: + raise UnknownPackageError(pkg) + + if not silent: + self.print_message( + "Removing %s @ %s: \t" + % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), + nl=False, + ) + + # firstly, remove dependencies + if not skip_dependencies: + self._uninstall_dependencies(pkg, silent) + + if os.path.islink(pkg.path): + os.unlink(pkg.path) + else: + fs.rmtree(pkg.path) + self.memcache_reset() + + # unfix detached-package with the same name + detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) + if ( + detached_pkg + and "@" in detached_pkg.path + and not os.path.isdir( + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) + ) + ): + shutil.move( + detached_pkg.path, + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), + ) + self.memcache_reset() + + if not silent: + click.echo("[%s]" % click.style("OK", fg="green")) + + return True + + def _uninstall_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageSourceItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message(click.style("Removing dependencies...", fg="yellow")) + for dependency in manifest.get("dependencies"): + pkg = self.get_package( + PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + ) + if not pkg: + continue + self._uninstall(pkg, silent=silent) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 93599200..ca065833 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -25,6 +25,7 @@ from platformio.package.lockfile import LockFile from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager._registry import PackageManageRegistryMixin +from platformio.package.manager._uninstall import PackageManagerUninstallMixin from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import ( PackageMetaData, @@ -36,14 +37,17 @@ from platformio.project.helpers import get_project_cache_dir class BasePackageManager( # pylint: disable=too-many-public-methods - PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin + PackageManagerDownloadMixin, + PackageManageRegistryMixin, + PackageManagerInstallMixin, + PackageManagerUninstallMixin, ): - MEMORY_CACHE = {} + _MEMORY_CACHE = {} def __init__(self, pkg_type, package_dir): self.pkg_type = pkg_type self.package_dir = self.ensure_dir_exists(package_dir) - self.MEMORY_CACHE = {} + self._MEMORY_CACHE = {} self._lockfile = None self._download_dir = None @@ -65,13 +69,13 @@ class BasePackageManager( # pylint: disable=too-many-public-methods self.unlock() def memcache_get(self, key, default=None): - return self.MEMORY_CACHE.get(key, default) + return self._MEMORY_CACHE.get(key, default) def memcache_set(self, key, value): - self.MEMORY_CACHE[key] = value + self._MEMORY_CACHE[key] = value def memcache_reset(self): - self.MEMORY_CACHE.clear() + self._MEMORY_CACHE.clear() @staticmethod def is_system_compatible(value): diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 2d8a7b14..5898ae2b 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -18,7 +18,10 @@ import time import pytest from platformio import fs, util -from platformio.package.exception import MissingPackageManifestError +from platformio.package.exception import ( + MissingPackageManifestError, + UnknownPackageError, +) from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager @@ -193,14 +196,24 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): # mbed library assert lm.install("wolfSSL", silent=True) assert len(lm.get_installed()) == 4 + # case sensitive author name + assert lm.install("DallasTemperature", silent=True) + assert lm.get_package("OneWire").metadata.version.major >= 2 + assert len(lm.get_installed()) == 6 # Tools tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) - pkg = tm.install("tool-stlink @ ~1.10400.0", silent=True) + pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True) manifest = tm.load_manifest(pkg) assert tm.is_system_compatible(manifest.get("system")) assert util.get_systype() in manifest.get("system", []) + # Test unknown + with pytest.raises(UnknownPackageError): + tm.install("unknown-package-tool @ 9.1.1", silent=True) + with pytest.raises(UnknownPackageError): + tm.install("owner/unknown-package-tool", silent=True) + def test_install_force(isolated_pio_core, tmpdir_factory): lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) @@ -316,3 +329,14 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert lm.uninstall(bar_pkg, silent=True) assert len(lm.get_installed()) == 0 + + # test uninstall dependencies + assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert len(lm.get_installed()) == 3 + assert lm.uninstall("AsyncMqttClient-esphome", silent=True, skip_dependencies=True) + assert len(lm.get_installed()) == 2 + + lm = LibraryPackageManager(str(storage_dir)) + assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert lm.uninstall("AsyncMqttClient-esphome", silent=True) + assert len(lm.get_installed()) == 0