Allow a forced package installation with removing existing package

platformio/package/manager/_download.py

@@ -14,6 +14,7 @@
 
 import hashlib
 import os
+import shutil
 import tempfile
 import time
 
@@ -85,7 +86,7 @@ class PackageManagerDownloadMixin(object):
                         raise e
             if checksum:
                 fd.verify(checksum)
-            os.rename(tmp_path, dl_path)
+            shutil.copyfile(tmp_path, dl_path)
         finally:
             if os.path.isfile(tmp_path):
                 os.remove(tmp_path)

platformio/package/manager/_install.py

@@ -21,7 +21,6 @@ import click
 
 from platformio import app, compat, fs, util
 from platformio.package.exception import PackageException, UnknownPackageError
-from platformio.package.lockfile import LockFile
 from platformio.package.meta import PackageSourceItem, PackageSpec
 from platformio.package.unpack import FileUnpacker
 from platformio.package.vcsclient import VCSClientFactory
@@ -43,25 +42,33 @@ class PackageManagerInstallMixin(object):
             with FileUnpacker(src) as fu:
                 return fu.unpack(dst, with_progress=False)
 
-    def install(self, spec, silent=False):
+    def install(self, spec, silent=False, force=False):
-        with LockFile(self.package_dir):
+        try:
-            pkg = self._install(spec, silent=silent)
+            self.lock()
+            pkg = self._install(spec, silent=silent, force=force)
             self.memcache_reset()
             self.cleanup_expired_downloads()
             return pkg
+        finally:
+            self.unlock()
 
-    def _install(self, spec, search_filters=None, silent=False):
+    def _install(self, spec, search_filters=None, silent=False, force=False):
         spec = self.ensure_spec(spec)
 
         # avoid circle dependencies
         if not self.INSTALL_HISTORY:
-            self.INSTALL_HISTORY = []
+            self.INSTALL_HISTORY = {}
         if spec in self.INSTALL_HISTORY:
-            return None
+            return self.INSTALL_HISTORY[spec]
-        self.INSTALL_HISTORY.append(spec)
 
         # check if package is already installed
         pkg = self.get_package(spec)
+
+        # if a forced installation
+        if pkg and force:
+            self.uninstall(pkg, silent=silent)
+            pkg = None
+
         if pkg:
             if not silent:
                 click.secho(
@@ -99,6 +106,7 @@ class PackageManagerInstallMixin(object):
 
         self.memcache_reset()
         self.install_dependencies(pkg, silent)
+        self.INSTALL_HISTORY[spec] = pkg
         return pkg
 
     def install_dependencies(self, pkg, silent=False):
@@ -240,15 +248,18 @@ class PackageManagerInstallMixin(object):
         shutil.move(tmp_pkg.path, dst_pkg.path)
         return PackageSourceItem(dst_pkg.path)
 
-    def uninstall(self, path_or_spec, silent=False):
+    def uninstall(self, pkg, silent=False):
-        with LockFile(self.package_dir):
+        try:
-            pkg = (
+            self.lock()
-                PackageSourceItem(path_or_spec)
+
-                if os.path.isdir(path_or_spec)
+            if not isinstance(pkg, PackageSourceItem):
-                else self.get_package(path_or_spec)
+                pkg = (
-            )
+                    PackageSourceItem(pkg)
+                    if os.path.isdir(pkg)
+                    else self.get_package(pkg)
+                )
             if not pkg or not pkg.metadata:
-                raise UnknownPackageError(path_or_spec)
+                raise UnknownPackageError(pkg)
 
             if not silent:
                 self.print_message(
@@ -276,7 +287,10 @@ class PackageManagerInstallMixin(object):
                     os.path.join(self.package_dir, detached_pkg.get_safe_dirname()),
                 )
                 self.memcache_reset()
+        finally:
+            self.unlock()
+
+        if not silent:
+            click.echo("[%s]" % click.style("OK", fg="green"))
 
-            if not silent:
-                click.echo("[%s]" % click.style("OK", fg="green"))
         return True

platformio/package/manager/_registry.py

@@ -37,7 +37,7 @@ class RegistryFileMirrorsIterator(object):
         self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc)
         self._visited_mirrors = []
 
-    def __iter__(self):
+    def __iter__(self):  # pylint: disable=non-iterator-returned
         return self
 
     def __next__(self):

platformio/package/manager/base.py

@@ -21,6 +21,7 @@ import semantic_version
 from platformio import fs, util
 from platformio.commands import PlatformioCLI
 from platformio.package.exception import ManifestException, MissingPackageManifestError
+from platformio.package.lockfile import LockFile
 from platformio.package.manager._download import PackageManagerDownloadMixin
 from platformio.package.manager._install import PackageManagerInstallMixin
 from platformio.package.manager._registry import PackageManageRegistryMixin
@@ -34,7 +35,7 @@ from platformio.package.meta import (
 from platformio.project.helpers import get_project_cache_dir
 
 
-class BasePackageManager(
+class BasePackageManager(  # pylint: disable=too-many-public-methods
     PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin
 ):
     MEMORY_CACHE = {}
@@ -43,10 +44,26 @@ class BasePackageManager(
         self.pkg_type = pkg_type
         self.package_dir = self.ensure_dir_exists(package_dir)
         self.MEMORY_CACHE = {}
+
+        self._lockfile = None
         self._download_dir = None
         self._tmp_dir = None
         self._registry_client = None
 
+    def lock(self):
+        if self._lockfile:
+            return
+        self._lockfile = LockFile(self.package_dir)
+        self._lockfile.acquire()
+
+    def unlock(self):
+        if hasattr(self, "_lockfile") and self._lockfile:
+            self._lockfile.release()
+            self._lockfile = None
+
+    def __del__(self):
+        self.unlock()
+
     def memcache_get(self, key, default=None):
         return self.MEMORY_CACHE.get(key, default)
 

platformio/package/meta.py

@@ -16,10 +16,11 @@ import json
 import os
 import re
 import tarfile
+from binascii import crc32
 
 import semantic_version
 
-from platformio.compat import get_object_members, string_types
+from platformio.compat import get_object_members, hashlib_encode_data, string_types
 from platformio.package.manifest.parser import ManifestFileType
 
 try:
@@ -89,6 +90,14 @@ class PackageSpec(object):
             ]
         )
 
+    def __hash__(self):
+        return crc32(
+            hashlib_encode_data(
+                "%s-%s-%s-%s-%s"
+                % (self.owner, self.id, self.name, self.requirements, self.url)
+            )
+        )
+
     def __repr__(self):
         return (
             "PackageSpec <owner={owner} id={id} name={name} "

tests/package/test_manager.py

@@ -202,6 +202,21 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory):
     assert util.get_systype() in manifest.get("system", [])
 
 
+def test_install_force(isolated_pio_core, tmpdir_factory):
+    lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage")))
+    # install #64 ArduinoJson
+    pkg = lm.install("64 @ ^5", silent=True)
+    assert pkg.metadata.version.major == 5
+    # try install the latest without specification
+    pkg = lm.install("64", silent=True)
+    assert pkg.metadata.version.major == 5
+    assert len(lm.get_installed()) == 1
+    # re-install the latest
+    pkg = lm.install(64, silent=True, force=True)
+    assert len(lm.get_installed()) == 1
+    assert pkg.metadata.version.major > 5
+
+
 def test_get_installed(isolated_pio_core, tmpdir_factory):
     storage_dir = tmpdir_factory.mktemp("storage")
     lm = LibraryPackageManager(str(storage_dir))
@@ -276,7 +291,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory):
     # foo @ 1.0.0
     pkg_dir = tmp_dir.join("foo").mkdir()
     pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}')
-    lm.install_from_url("file://%s" % pkg_dir, "foo")
+    foo_1_0_0_pkg = lm.install_from_url("file://%s" % pkg_dir, "foo")
     # foo @ 1.3.0
     pkg_dir = tmp_dir.join("foo-1.3.0").mkdir()
     pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}')
@@ -284,7 +299,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory):
     # bar
     pkg_dir = tmp_dir.join("bar").mkdir()
     pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}')
-    lm.install("file://%s" % pkg_dir, silent=True)
+    bar_pkg = lm.install("file://%s" % pkg_dir, silent=True)
 
     assert len(lm.get_installed()) == 3
     assert os.path.isdir(os.path.join(str(storage_dir), "foo"))
@@ -297,7 +312,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory):
     assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0"))
 
     # uninstall the rest
-    assert lm.uninstall("foo", silent=True)
+    assert lm.uninstall(foo_1_0_0_pkg.path, silent=True)
-    assert lm.uninstall("bar", silent=True)
+    assert lm.uninstall(bar_pkg, silent=True)
 
     assert len(lm.get_installed()) == 0