diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index a052da09..2ab8c143 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -14,6 +14,7 @@ import hashlib import os +import shutil import tempfile import time @@ -85,7 +86,7 @@ class PackageManagerDownloadMixin(object): raise e if checksum: fd.verify(checksum) - os.rename(tmp_path, dl_path) + shutil.copyfile(tmp_path, dl_path) finally: if os.path.isfile(tmp_path): os.remove(tmp_path) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index a5aa1c38..58b1de1e 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,6 @@ import click from platformio import app, compat, fs, util from platformio.package.exception import PackageException, UnknownPackageError -from platformio.package.lockfile import LockFile from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -43,25 +42,33 @@ class PackageManagerInstallMixin(object): with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=False) - def install(self, spec, silent=False): - with LockFile(self.package_dir): - pkg = self._install(spec, silent=silent) + def install(self, spec, silent=False, force=False): + try: + self.lock() + pkg = self._install(spec, silent=silent, force=force) self.memcache_reset() self.cleanup_expired_downloads() return pkg + finally: + self.unlock() - def _install(self, spec, search_filters=None, silent=False): + def _install(self, spec, search_filters=None, silent=False, force=False): spec = self.ensure_spec(spec) # avoid circle dependencies if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] + self.INSTALL_HISTORY = {} if spec in self.INSTALL_HISTORY: - return None - self.INSTALL_HISTORY.append(spec) + return self.INSTALL_HISTORY[spec] # check if package is already installed pkg = self.get_package(spec) + + # if a forced installation + if pkg and force: + self.uninstall(pkg, silent=silent) + pkg = None + if pkg: if not silent: click.secho( @@ -99,6 +106,7 @@ class PackageManagerInstallMixin(object): self.memcache_reset() self.install_dependencies(pkg, silent) + self.INSTALL_HISTORY[spec] = pkg return pkg def install_dependencies(self, pkg, silent=False): @@ -240,15 +248,18 @@ class PackageManagerInstallMixin(object): shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) - def uninstall(self, path_or_spec, silent=False): - with LockFile(self.package_dir): - pkg = ( - PackageSourceItem(path_or_spec) - if os.path.isdir(path_or_spec) - else self.get_package(path_or_spec) - ) + def uninstall(self, pkg, silent=False): + try: + self.lock() + + if not isinstance(pkg, PackageSourceItem): + pkg = ( + PackageSourceItem(pkg) + if os.path.isdir(pkg) + else self.get_package(pkg) + ) if not pkg or not pkg.metadata: - raise UnknownPackageError(path_or_spec) + raise UnknownPackageError(pkg) if not silent: self.print_message( @@ -276,7 +287,10 @@ class PackageManagerInstallMixin(object): os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), ) self.memcache_reset() + finally: + self.unlock() + + if not silent: + click.echo("[%s]" % click.style("OK", fg="green")) - if not silent: - click.echo("[%s]" % click.style("OK", fg="green")) return True diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 3f3ae813..cdd46cd0 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -37,7 +37,7 @@ class RegistryFileMirrorsIterator(object): self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) self._visited_mirrors = [] - def __iter__(self): + def __iter__(self): # pylint: disable=non-iterator-returned return self def __next__(self): diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index e48fc250..93599200 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -21,6 +21,7 @@ import semantic_version from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.package.exception import ManifestException, MissingPackageManifestError +from platformio.package.lockfile import LockFile from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager._registry import PackageManageRegistryMixin @@ -34,7 +35,7 @@ from platformio.package.meta import ( from platformio.project.helpers import get_project_cache_dir -class BasePackageManager( +class BasePackageManager( # pylint: disable=too-many-public-methods PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin ): MEMORY_CACHE = {} @@ -43,10 +44,26 @@ class BasePackageManager( self.pkg_type = pkg_type self.package_dir = self.ensure_dir_exists(package_dir) self.MEMORY_CACHE = {} + + self._lockfile = None self._download_dir = None self._tmp_dir = None self._registry_client = None + def lock(self): + if self._lockfile: + return + self._lockfile = LockFile(self.package_dir) + self._lockfile.acquire() + + def unlock(self): + if hasattr(self, "_lockfile") and self._lockfile: + self._lockfile.release() + self._lockfile = None + + def __del__(self): + self.unlock() + def memcache_get(self, key, default=None): return self.MEMORY_CACHE.get(key, default) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 0f1214e1..6cd2904b 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -16,10 +16,11 @@ import json import os import re import tarfile +from binascii import crc32 import semantic_version -from platformio.compat import get_object_members, string_types +from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType try: @@ -89,6 +90,14 @@ class PackageSpec(object): ] ) + def __hash__(self): + return crc32( + hashlib_encode_data( + "%s-%s-%s-%s-%s" + % (self.owner, self.id, self.name, self.requirements, self.url) + ) + ) + def __repr__(self): return ( "PackageSpec 5 + + def test_get_installed(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) @@ -276,7 +291,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): # foo @ 1.0.0 pkg_dir = tmp_dir.join("foo").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') - lm.install_from_url("file://%s" % pkg_dir, "foo") + foo_1_0_0_pkg = lm.install_from_url("file://%s" % pkg_dir, "foo") # foo @ 1.3.0 pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') @@ -284,7 +299,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): # bar pkg_dir = tmp_dir.join("bar").mkdir() pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') - lm.install("file://%s" % pkg_dir, silent=True) + bar_pkg = lm.install("file://%s" % pkg_dir, silent=True) assert len(lm.get_installed()) == 3 assert os.path.isdir(os.path.join(str(storage_dir), "foo")) @@ -297,7 +312,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) # uninstall the rest - assert lm.uninstall("foo", silent=True) - assert lm.uninstall("bar", silent=True) + assert lm.uninstall(foo_1_0_0_pkg.path, silent=True) + assert lm.uninstall(bar_pkg, silent=True) assert len(lm.get_installed()) == 0