Implement package removing with dependencies

platformio/clients/http.py

@@ -20,7 +20,13 @@ from platformio.exception import PlatformioException
 
 
 class HTTPClientError(PlatformioException):
-    pass
+    def __init__(self, message, response=None):
+        super(HTTPClientError, self).__init__()
+        self.message = message
+        self.response = response
+
+    def __str__(self):  # pragma: no cover
+        return self.message
 
 
 class HTTPClient(object):
@@ -52,7 +58,7 @@ class HTTPClient(object):
         try:
             return getattr(self._session, method)(self.base_url + path, **kwargs)
         except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
-            raise HTTPClientError(e)
+            raise HTTPClientError(str(e))
 
     def request_json_data(self, *args, **kwargs):
         response = self.send_request(*args, **kwargs)
@@ -69,6 +75,4 @@ class HTTPClient(object):
             message = response.json()["message"]
         except (KeyError, ValueError):
             message = response.text
-        exc = HTTPClientError(message)
+        raise HTTPClientError(message, response)
-        exc.response = response
-        raise exc

platformio/package/manager/_install.py

@@ -20,7 +20,7 @@ import tempfile
 import click
 
 from platformio import app, compat, fs, util
-from platformio.package.exception import PackageException, UnknownPackageError
+from platformio.package.exception import PackageException
 from platformio.package.meta import PackageSourceItem, PackageSpec
 from platformio.package.unpack import FileUnpacker
 from platformio.package.vcsclient import VCSClientFactory
@@ -28,7 +28,7 @@ from platformio.package.vcsclient import VCSClientFactory
 
 class PackageManagerInstallMixin(object):
 
-    INSTALL_HISTORY = None  # avoid circle dependencies
+    _INSTALL_HISTORY = None  # avoid circle dependencies
 
     @staticmethod
     def unpack(src, dst):
@@ -56,10 +56,10 @@ class PackageManagerInstallMixin(object):
         spec = self.ensure_spec(spec)
 
         # avoid circle dependencies
-        if not self.INSTALL_HISTORY:
+        if not self._INSTALL_HISTORY:
-            self.INSTALL_HISTORY = {}
+            self._INSTALL_HISTORY = {}
-        if spec in self.INSTALL_HISTORY:
+        if spec in self._INSTALL_HISTORY:
-            return self.INSTALL_HISTORY[spec]
+            return self._INSTALL_HISTORY[spec]
 
         # check if package is already installed
         pkg = self.get_package(spec)
@@ -105,11 +105,11 @@ class PackageManagerInstallMixin(object):
             )
 
         self.memcache_reset()
-        self.install_dependencies(pkg, silent)
+        self._install_dependencies(pkg, silent)
-        self.INSTALL_HISTORY[spec] = pkg
+        self._INSTALL_HISTORY[spec] = pkg
         return pkg
 
-    def install_dependencies(self, pkg, silent=False):
+    def _install_dependencies(self, pkg, silent=False):
         assert isinstance(pkg, PackageSourceItem)
         manifest = self.load_manifest(pkg)
         if not manifest.get("dependencies"):
@@ -117,14 +117,14 @@ class PackageManagerInstallMixin(object):
         if not silent:
             self.print_message(click.style("Installing dependencies...", fg="yellow"))
         for dependency in manifest.get("dependencies"):
-            if not self.install_dependency(dependency, silent) and not silent:
+            if not self._install_dependency(dependency, silent) and not silent:
                 click.secho(
                     "Warning! Could not install dependency %s for package '%s'"
                     % (dependency, pkg.metadata.name),
                     fg="yellow",
                 )
 
-    def install_dependency(self, dependency, silent=False):
+    def _install_dependency(self, dependency, silent=False):
         spec = PackageSpec(
             name=dependency.get("name"), requirements=dependency.get("version")
         )
@@ -247,50 +247,3 @@ class PackageManagerInstallMixin(object):
         _cleanup_dir(dst_pkg.path)
         shutil.move(tmp_pkg.path, dst_pkg.path)
         return PackageSourceItem(dst_pkg.path)
-
-    def uninstall(self, pkg, silent=False):
-        try:
-            self.lock()
-
-            if not isinstance(pkg, PackageSourceItem):
-                pkg = (
-                    PackageSourceItem(pkg)
-                    if os.path.isdir(pkg)
-                    else self.get_package(pkg)
-                )
-            if not pkg or not pkg.metadata:
-                raise UnknownPackageError(pkg)
-
-            if not silent:
-                self.print_message(
-                    "Uninstalling %s @ %s: \t"
-                    % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version),
-                    nl=False,
-                )
-            if os.path.islink(pkg.path):
-                os.unlink(pkg.path)
-            else:
-                fs.rmtree(pkg.path)
-            self.memcache_reset()
-
-            # unfix detached-package with the same name
-            detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name))
-            if (
-                detached_pkg
-                and "@" in detached_pkg.path
-                and not os.path.isdir(
-                    os.path.join(self.package_dir, detached_pkg.get_safe_dirname())
-                )
-            ):
-                shutil.move(
-                    detached_pkg.path,
-                    os.path.join(self.package_dir, detached_pkg.get_safe_dirname()),
-                )
-                self.memcache_reset()
-        finally:
-            self.unlock()
-
-        if not silent:
-            click.echo("[%s]" % click.style("OK", fg="green"))
-
-        return True

platformio/package/manager/_registry.py

@@ -78,12 +78,19 @@ class RegistryFileMirrorsIterator(object):
 
 class PackageManageRegistryMixin(object):
     def install_from_registry(self, spec, search_filters=None, silent=False):
-        packages = self.search_registry_packages(spec, search_filters)
+        if spec.owner and spec.name and not search_filters:
-        if not packages:
+            package = self.fetch_registry_package(spec.owner, spec.name)
-            raise UnknownPackageError(spec.humanize())
+            if not package:
-        if len(packages) > 1 and not silent:
+                raise UnknownPackageError(spec.humanize())
-            self.print_multi_package_issue(packages, spec)
+            version = self._pick_best_pkg_version(package["versions"], spec)
-        package, version = self.find_best_registry_version(packages, spec)
+        else:
+            packages = self.search_registry_packages(spec, search_filters)
+            if not packages:
+                raise UnknownPackageError(spec.humanize())
+            if len(packages) > 1 and not silent:
+                self.print_multi_package_issue(packages, spec)
+            package, version = self.find_best_registry_version(packages, spec)
+
         pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None
         if not pkgfile:
             raise UnknownPackageError(spec.humanize())
@@ -117,17 +124,17 @@ class PackageManageRegistryMixin(object):
             filters["ids"] = str(spec.id)
         else:
             filters["types"] = self.pkg_type
-            filters["names"] = '"%s"' % spec.name.lower()
+            filters["names"] = spec.name.lower()
             if spec.owner:
                 filters["owners"] = spec.owner.lower()
         return self.get_registry_client_instance().list_packages(filters=filters)[
             "items"
         ]
 
-    def fetch_registry_package_versions(self, owner, name):
+    def fetch_registry_package(self, owner, name):
         return self.get_registry_client_instance().get_package(
             self.pkg_type, owner, name
-        )["versions"]
+        )
 
     @staticmethod
     def print_multi_package_issue(packages, spec):
@@ -163,9 +170,9 @@ class PackageManageRegistryMixin(object):
         # if the custom version requirements, check ALL package versions
         for package in packages:
             version = self._pick_best_pkg_version(
-                self.fetch_registry_package_versions(
+                self.fetch_registry_package(
                     package["owner"]["username"], package["name"]
-                ),
+                ).get("versions"),
                 spec,
             )
             if version:

platformio/package/manager/_uninstall.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+
+import click
+
+from platformio import fs
+from platformio.package.exception import UnknownPackageError
+from platformio.package.meta import PackageSourceItem, PackageSpec
+
+
+class PackageManagerUninstallMixin(object):
+    def uninstall(self, pkg, silent=False, skip_dependencies=False):
+        try:
+            self.lock()
+            return self._uninstall(pkg, silent, skip_dependencies)
+        finally:
+            self.unlock()
+
+    def _uninstall(self, pkg, silent=False, skip_dependencies=False):
+        if not isinstance(pkg, PackageSourceItem):
+            pkg = (
+                PackageSourceItem(pkg) if os.path.isdir(pkg) else self.get_package(pkg)
+            )
+        if not pkg or not pkg.metadata:
+            raise UnknownPackageError(pkg)
+
+        if not silent:
+            self.print_message(
+                "Removing %s @ %s: \t"
+                % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version),
+                nl=False,
+            )
+
+        # firstly, remove dependencies
+        if not skip_dependencies:
+            self._uninstall_dependencies(pkg, silent)
+
+        if os.path.islink(pkg.path):
+            os.unlink(pkg.path)
+        else:
+            fs.rmtree(pkg.path)
+        self.memcache_reset()
+
+        # unfix detached-package with the same name
+        detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name))
+        if (
+            detached_pkg
+            and "@" in detached_pkg.path
+            and not os.path.isdir(
+                os.path.join(self.package_dir, detached_pkg.get_safe_dirname())
+            )
+        ):
+            shutil.move(
+                detached_pkg.path,
+                os.path.join(self.package_dir, detached_pkg.get_safe_dirname()),
+            )
+            self.memcache_reset()
+
+        if not silent:
+            click.echo("[%s]" % click.style("OK", fg="green"))
+
+        return True
+
+    def _uninstall_dependencies(self, pkg, silent=False):
+        assert isinstance(pkg, PackageSourceItem)
+        manifest = self.load_manifest(pkg)
+        if not manifest.get("dependencies"):
+            return
+        if not silent:
+            self.print_message(click.style("Removing dependencies...", fg="yellow"))
+        for dependency in manifest.get("dependencies"):
+            pkg = self.get_package(
+                PackageSpec(
+                    name=dependency.get("name"), requirements=dependency.get("version")
+                )
+            )
+            if not pkg:
+                continue
+            self._uninstall(pkg, silent=silent)

platformio/package/manager/base.py

@@ -25,6 +25,7 @@ from platformio.package.lockfile import LockFile
 from platformio.package.manager._download import PackageManagerDownloadMixin
 from platformio.package.manager._install import PackageManagerInstallMixin
 from platformio.package.manager._registry import PackageManageRegistryMixin
+from platformio.package.manager._uninstall import PackageManagerUninstallMixin
 from platformio.package.manifest.parser import ManifestParserFactory
 from platformio.package.meta import (
     PackageMetaData,
@@ -36,14 +37,17 @@ from platformio.project.helpers import get_project_cache_dir
 
 
 class BasePackageManager(  # pylint: disable=too-many-public-methods
-    PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin
+    PackageManagerDownloadMixin,
+    PackageManageRegistryMixin,
+    PackageManagerInstallMixin,
+    PackageManagerUninstallMixin,
 ):
-    MEMORY_CACHE = {}
+    _MEMORY_CACHE = {}
 
     def __init__(self, pkg_type, package_dir):
         self.pkg_type = pkg_type
         self.package_dir = self.ensure_dir_exists(package_dir)
-        self.MEMORY_CACHE = {}
+        self._MEMORY_CACHE = {}
 
         self._lockfile = None
         self._download_dir = None
@@ -65,13 +69,13 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
         self.unlock()
 
     def memcache_get(self, key, default=None):
-        return self.MEMORY_CACHE.get(key, default)
+        return self._MEMORY_CACHE.get(key, default)
 
     def memcache_set(self, key, value):
-        self.MEMORY_CACHE[key] = value
+        self._MEMORY_CACHE[key] = value
 
     def memcache_reset(self):
-        self.MEMORY_CACHE.clear()
+        self._MEMORY_CACHE.clear()
 
     @staticmethod
     def is_system_compatible(value):

tests/package/test_manager.py

@@ -18,7 +18,10 @@ import time
 import pytest
 
 from platformio import fs, util
-from platformio.package.exception import MissingPackageManifestError
+from platformio.package.exception import (
+    MissingPackageManifestError,
+    UnknownPackageError,
+)
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.manager.tool import ToolPackageManager
@@ -193,14 +196,24 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory):
     # mbed library
     assert lm.install("wolfSSL", silent=True)
     assert len(lm.get_installed()) == 4
+    # case sensitive author name
+    assert lm.install("DallasTemperature", silent=True)
+    assert lm.get_package("OneWire").metadata.version.major >= 2
+    assert len(lm.get_installed()) == 6
 
     # Tools
     tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage")))
-    pkg = tm.install("tool-stlink @ ~1.10400.0", silent=True)
+    pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True)
     manifest = tm.load_manifest(pkg)
     assert tm.is_system_compatible(manifest.get("system"))
     assert util.get_systype() in manifest.get("system", [])
 
+    # Test unknown
+    with pytest.raises(UnknownPackageError):
+        tm.install("unknown-package-tool @ 9.1.1", silent=True)
+    with pytest.raises(UnknownPackageError):
+        tm.install("owner/unknown-package-tool", silent=True)
+
 
 def test_install_force(isolated_pio_core, tmpdir_factory):
     lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage")))
@@ -316,3 +329,14 @@ def test_uninstall(isolated_pio_core, tmpdir_factory):
     assert lm.uninstall(bar_pkg, silent=True)
 
     assert len(lm.get_installed()) == 0
+
+    # test uninstall dependencies
+    assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True)
+    assert len(lm.get_installed()) == 3
+    assert lm.uninstall("AsyncMqttClient-esphome", silent=True, skip_dependencies=True)
+    assert len(lm.get_installed()) == 2
+
+    lm = LibraryPackageManager(str(storage_dir))
+    assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True)
+    assert lm.uninstall("AsyncMqttClient-esphome", silent=True)
+    assert len(lm.get_installed()) == 0