Significantly improve Package Manager // Resolve #913

* Handle dependencies when installing non-registry package/library (VCS, archive, local folder)
This commit is contained in:
Ivan Kravets committed 2017-03-08 17:24:58 +02:00
1 parent 41cea76603
commit 58942c3f38
13 files changed
+657 -433

No files matched your search

+17 -14
View File
@@ -15,7 +15,7 @@
# pylint: disable=too-many-branches, too-many-locals
import json
from os.path import join
from os.path import isdir, join
from time import sleep
from urllib import quote
@@ -116,22 +116,23 @@ def lib_uninstall(lm, libraries):
@click.pass_obj
def lib_update(lm, libraries, only_check, json_output):
if not libraries:
libraries = []
for manifest in lm.get_installed():
if "@vcs-" in manifest['__pkg_dir']:
libraries.append("%s=%s" % (manifest['name'], manifest['url']))
else:
libraries.append(str(manifest.get("id", manifest['name'])))
libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()]
if only_check and json_output:
result = []
for library in libraries:
name, requirements, url = lm.parse_pkg_name(library)
latest = lm.outdated(name, requirements, url)
pkg_dir = library if isdir(library) else None
requirements = None
url = None
if not pkg_dir:
name, requirements, url = lm.parse_pkg_input(library)
pkg_dir = lm.get_package_dir(name, requirements, url)
if not pkg_dir:
continue
latest = lm.outdated(pkg_dir, requirements)
if not latest:
continue
manifest = lm.load_manifest(
lm.get_package_dir(name, requirements, url))
manifest = lm.load_manifest(pkg_dir)
manifest['versionLatest'] = latest
result.append(manifest)
return click.echo(json.dumps(result))
@@ -167,6 +168,9 @@ def print_lib_item(item):
click.echo("Authors: %s" % ", ".join(
item.get("authornames",
[a.get("name", "") for a in item.get("authors", [])])))
if "__src_url" in item:
click.secho("Source: %s" % item['__src_url'])
click.echo()
@@ -270,8 +274,7 @@ def get_builtin_libs(storage_names=None):
storage_names = storage_names or []
pm = PlatformManager()
for manifest in pm.get_installed():
p = PlatformFactory.newPlatform(
pm.get_manifest_path(manifest['__pkg_dir']))
p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
for storage in p.get_lib_storages():
if storage_names and storage['name'] not in storage_names:
continue
@@ -308,7 +311,7 @@ def lib_builtin(storage, json_output):
@click.option("--json-output", is_flag=True)
def lib_show(library, json_output):
lm = LibraryManager()
name, requirements, _ = lm.parse_pkg_name(library)
name, requirements, _ = lm.parse_pkg_input(library)
lib_id = lm.get_pkg_id_by_name(
name, requirements, silent=json_output, interactive=not json_output)
lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
+20 -15
View File
@@ -13,7 +13,7 @@
# limitations under the License.
import json
from os.path import dirname, isfile, join
from os.path import dirname, isdir
import click
@@ -94,10 +94,12 @@ def _get_installed_platform_data(platform,
# del data['version']
# return data
data['__pkg_dir'] = dirname(p.manifest_path)
# if VCS cloned platform
if not isfile(join(data['__pkg_dir'], "platform.json")):
data['__pkg_dir'] = dirname(data['__pkg_dir'])
# overwrite VCS version and add extra fields
manifest = PlatformManager().load_manifest(dirname(p.manifest_path))
assert manifest
for key in manifest:
if key == "version" or key.startswith("__"):
data[key] = manifest[key]
if with_boards:
data['boards'] = [c.get_brief_data() for c in p.get_boards().values()]
@@ -214,7 +216,7 @@ def platform_list(json_output):
for manifest in pm.get_installed():
platforms.append(
_get_installed_platform_data(
pm.get_manifest_path(manifest['__pkg_dir']),
manifest['__pkg_dir'],
with_boards=False,
expose_packages=False))
if json_output:
@@ -336,22 +338,25 @@ def platform_update(platforms, only_packages, only_check, json_output):
pm = PlatformManager()
if not platforms:
platforms = []
for manifest in pm.get_installed():
if "@vcs-" in manifest['__pkg_dir']:
platforms.append("%s=%s" % (manifest['name'], manifest['url']))
else:
platforms.append(manifest['name'])
platforms = [manifest['__pkg_dir'] for manifest in pm.get_installed()]
if only_check and json_output:
result = []
for platform in platforms:
name, requirements, url = pm.parse_pkg_name(platform)
latest = pm.outdated(name, requirements, url)
pkg_dir = platform if isdir(platform) else None
requirements = None
url = None
if not pkg_dir:
name, requirements, url = pm.parse_pkg_input(platform)
pkg_dir = pm.get_package_dir(name, requirements, url)
if not pkg_dir:
continue
latest = pm.outdated(pkg_dir, requirements)
if not latest:
continue
data = _get_installed_platform_data(
name, with_boards=False, expose_packages=False)
data['versionLatest'] = latest or "Unknown"
pkg_dir, with_boards=False, expose_packages=False)
data['versionLatest'] = latest
result.append(data)
return click.echo(json.dumps(result))
else:
+2 -3
View File
@@ -163,8 +163,7 @@ def after_upgrade(ctx):
# update development platforms
pm = PlatformManager()
for manifest in pm.get_installed():
# pm.update(manifest['name'], "^" + manifest['version'])
pm.update(manifest['name'])
pm.update(manifest['__pkg_dir'])
# update PlatformIO Plus tool if installed
pioplus_update()
@@ -262,7 +261,7 @@ def check_internal_updates(ctx, what):
outdated_items = []
for manifest in pm.get_installed():
if manifest['name'] not in outdated_items and \
pm.outdated(manifest['name']):
pm.outdated(manifest['__pkg_dir']):
outdated_items.append(manifest['name'])
if not outdated_items:
+12 -41
View File
@@ -15,10 +15,8 @@
# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
import json
import os
import re
from glob import glob
from hashlib import md5
from os.path import isdir, join
import arrow
@@ -61,8 +59,8 @@ class LibraryManager(BasePkgManager):
return None
def load_manifest(self, path):
manifest = BasePkgManager.load_manifest(self, path)
def load_manifest(self, pkg_dir):
manifest = BasePkgManager.load_manifest(self, pkg_dir)
if not manifest:
return manifest
@@ -76,6 +74,9 @@ class LibraryManager(BasePkgManager):
manifest['authors'] = [{"name": manifest['author']}]
del manifest['author']
if "authors" in manifest and not isinstance(manifest['authors'], list):
manifest['authors'] = [manifest['authors']]
if "keywords" not in manifest:
keywords = []
for keyword in re.split(r"[\s/]+",
@@ -123,31 +124,6 @@ class LibraryManager(BasePkgManager):
return manifest
def check_pkg_structure(self, pkg_dir):
try:
return BasePkgManager.check_pkg_structure(self, pkg_dir)
except exception.MissingPackageManifest:
# we will generate manifest automatically
# if library doesn't contain any
pass
manifest = {
"name": "Library_" + md5(pkg_dir).hexdigest()[:5],
"version": "0.0.0"
}
for root, dirs, files in os.walk(pkg_dir):
if len(dirs) == 1 and not files:
manifest['name'] = dirs[0]
continue
if dirs or files:
pkg_dir = root
break
with open(join(pkg_dir, self.manifest_names[0]), "w") as fp:
json.dump(manifest, fp)
return pkg_dir
@staticmethod
def normalize_dependencies(dependencies):
if not dependencies:
@@ -239,7 +215,7 @@ class LibraryManager(BasePkgManager):
}, silent, interactive)['id'])
def _install_from_piorepo(self, name, requirements):
assert name.startswith("id=")
assert name.startswith("id="), name
version = self.get_latest_repo_version(name, requirements)
if not version:
raise exception.UndefinedPackageVersion(requirements or "latest",
@@ -260,28 +236,23 @@ class LibraryManager(BasePkgManager):
silent=False,
trigger_event=True,
interactive=False):
already_installed = False
_name, _requirements, _url = self.parse_pkg_name(name, requirements)
try:
_name, _requirements, _url = self.parse_pkg_input(name,
requirements)
if not _url:
_name = "id=%d" % self.get_pkg_id_by_name(
name = "id=%d" % self.get_pkg_id_by_name(
_name,
_requirements,
silent=silent,
interactive=interactive)
already_installed = self.get_package(_name, _requirements, _url)
pkg_dir = BasePkgManager.install(
self, _name
if not _url else name, _requirements, silent, trigger_event)
requirements = _requirements
pkg_dir = BasePkgManager.install(self, name, requirements, silent,
trigger_event)
except exception.InternetIsOffline as e:
if not silent:
click.secho(str(e), fg="yellow")
return
if already_installed:
return
manifest = self.load_manifest(pkg_dir)
if "dependencies" not in manifest:
return pkg_dir
+268 -230
View File
@@ -17,7 +17,7 @@ import hashlib
import json
import os
import shutil
from os.path import basename, dirname, getsize, isdir, isfile, islink, join
from os.path import basename, getsize, isdir, isfile, islink, join
from tempfile import mkdtemp
import click
@@ -127,16 +127,36 @@ class PkgRepoMixin(object):
class PkgInstallerMixin(object):
VCS_MANIFEST_NAME = ".piopkgmanager.json"
SRC_MANIFEST_NAME = ".piopkgmanager.json"
FILE_CACHE_VALID = "1m" # 1 month
FILE_CACHE_MAX_SIZE = 1024 * 1024
_INSTALLED_CACHE = {}
MEMORY_CACHE = {}
def reset_cache(self):
if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
del PkgInstallerMixin._INSTALLED_CACHE[self.package_dir]
@staticmethod
def cache_get(key, default=None):
return PkgInstallerMixin.MEMORY_CACHE.get(key, default)
@staticmethod
def cache_set(key, value):
PkgInstallerMixin.MEMORY_CACHE[key] = value
@staticmethod
def cache_reset():
PkgInstallerMixin.MEMORY_CACHE = {}
def read_dirs(self, src_dir):
cache_key = "read_dirs-%s" % src_dir
result = self.cache_get(cache_key)
if result:
return result
result = [
join(src_dir, name) for name in sorted(os.listdir(src_dir))
if isdir(join(src_dir, name))
]
self.cache_set(cache_key, result)
return result
def download(self, url, dest_dir, sha1=None):
cache_key_fname = app.ContentCache.key_from_args(url, "fname")
@@ -171,26 +191,23 @@ class PkgInstallerMixin(object):
return fu.start()
@staticmethod
def generate_install_dirname(manifest):
def get_install_dirname(manifest):
name = manifest['name']
if "id" in manifest:
name += "_ID%d" % manifest['id']
return name
def get_vcs_manifest_path(self, pkg_dir):
def get_src_manifest_path(self, pkg_dir):
for item in os.listdir(pkg_dir):
if not isdir(join(pkg_dir, item)):
continue
if isfile(join(pkg_dir, item, self.VCS_MANIFEST_NAME)):
return join(pkg_dir, item, self.VCS_MANIFEST_NAME)
if isfile(join(pkg_dir, item, self.SRC_MANIFEST_NAME)):
return join(pkg_dir, item, self.SRC_MANIFEST_NAME)
return None
def get_manifest_path(self, pkg_dir):
if not isdir(pkg_dir):
return None
manifest_path = self.get_vcs_manifest_path(pkg_dir)
if manifest_path:
return manifest_path
for name in self.manifest_names:
manifest_path = join(pkg_dir, name)
if isfile(manifest_path):
@@ -198,73 +215,104 @@ class PkgInstallerMixin(object):
return None
def manifest_exists(self, pkg_dir):
return self.get_manifest_path(pkg_dir) is not None
return self.get_manifest_path(pkg_dir) or \
self.get_src_manifest_path(pkg_dir)
def load_manifest(self, path): # pylint: disable=too-many-branches
assert path
pkg_dir = path
if isdir(path):
path = self.get_manifest_path(path)
if not path:
return None
else:
pkg_dir = dirname(pkg_dir)
def load_manifest(self, pkg_dir):
cache_key = "load_manifest-%s" % pkg_dir
result = self.cache_get(cache_key)
if result:
return result
is_vcs_pkg = False
if isfile(path) and path.endswith(self.VCS_MANIFEST_NAME):
is_vcs_pkg = True
pkg_dir = dirname(dirname(path))
manifest_path = self.get_manifest_path(pkg_dir)
if not manifest_path:
return None
# return from cache
if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
for manifest in PkgInstallerMixin._INSTALLED_CACHE[
self.package_dir]:
if not is_vcs_pkg and manifest['__pkg_dir'] == pkg_dir:
return manifest
# if non-registry packages: VCS or archive
src_manifest_path = self.get_src_manifest_path(pkg_dir)
src_manifest = None
if src_manifest_path:
src_manifest = util.load_json(src_manifest_path)
manifest = {}
if path.endswith(".json"):
manifest = util.load_json(path)
elif path.endswith(".properties"):
with codecs.open(path, encoding="utf-8") as fp:
if manifest_path.endswith(".json"):
manifest = util.load_json(manifest_path)
elif manifest_path.endswith(".properties"):
with codecs.open(manifest_path, encoding="utf-8") as fp:
for line in fp.readlines():
if "=" not in line:
continue
key, value = line.split("=", 1)
manifest[key.strip()] = value.strip()
else:
if src_manifest:
if "name" not in manifest:
manifest['name'] = basename(pkg_dir)
if "version" not in manifest:
manifest['version'] = "0.0.0"
manifest['name'] = src_manifest['name']
if "version" in src_manifest:
manifest['version'] = src_manifest['version']
manifest['__src_url'] = src_manifest['url']
if "name" not in manifest:
manifest['name'] = basename(pkg_dir)
if "version" not in manifest:
manifest['version'] = "0.0.0"
manifest['__pkg_dir'] = pkg_dir
self.cache_set(cache_key, manifest)
return manifest
def get_installed(self):
if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
return PkgInstallerMixin._INSTALLED_CACHE[self.package_dir]
items = []
for p in sorted(os.listdir(self.package_dir)):
pkg_dir = join(self.package_dir, p)
if not isdir(pkg_dir):
continue
for pkg_dir in self.read_dirs(self.package_dir):
manifest = self.load_manifest(pkg_dir)
if not manifest:
continue
assert "name" in manifest
items.append(manifest)
PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] = items
return items
def check_pkg_structure(self, pkg_dir):
if self.manifest_exists(pkg_dir):
return pkg_dir
def get_package(self, name, requirements=None, url=None):
pkg_id = int(name[3:]) if name.startswith("id=") else 0
best = None
for manifest in self.get_installed():
if url:
if manifest.get("__src_url") != url:
continue
elif pkg_id and manifest.get("id") != pkg_id:
continue
elif not pkg_id and manifest['name'] != name:
continue
for root, _, _ in os.walk(pkg_dir):
# strict version or VCS HASH
if requirements and requirements == manifest['version']:
return manifest
try:
if requirements and not semantic_version.Spec(
requirements).match(
semantic_version.Version(
manifest['version'], partial=True)):
continue
elif not best or (semantic_version.Version(
manifest['version'], partial=True) >
semantic_version.Version(
best['version'], partial=True)):
best = manifest
except ValueError:
pass
return best
def get_package_dir(self, name, requirements=None, url=None):
manifest = self.get_package(name, requirements, url)
return manifest.get("__pkg_dir") if manifest else None
def find_pkg_root(self, src_dir):
if self.manifest_exists(src_dir):
return src_dir
for root, _, _ in os.walk(src_dir):
if self.manifest_exists(root):
return root
raise exception.MissingPackageManifest(", ".join(self.manifest_names))
def _install_from_piorepo(self, name, requirements):
@@ -291,18 +339,25 @@ class PkgInstallerMixin(object):
util.get_systype())
return pkg_dir
def _install_from_url(self, name, url, requirements=None, sha1=None):
def _install_from_url(self,
name,
url,
requirements=None,
sha1=None,
track=False):
pkg_dir = None
tmp_dir = mkdtemp("-package", "installing-", self.package_dir)
src_manifest_dir = None
src_manifest = {"name": name, "url": url, "requirements": requirements}
try:
if url.startswith("file://"):
url = url[7:]
if isfile(url):
self.unpack(url, tmp_dir)
_url = url[7:]
if isfile(_url):
self.unpack(_url, tmp_dir)
else:
util.rmtree_(tmp_dir)
shutil.copytree(url, tmp_dir)
shutil.copytree(_url, tmp_dir)
elif url.startswith(("http://", "https://")):
dlpath = self.download(url, tmp_dir, sha1)
assert isfile(dlpath)
@@ -311,29 +366,52 @@ class PkgInstallerMixin(object):
else:
vcs = VCSClientFactory.newClient(tmp_dir, url)
assert vcs.export()
with open(join(vcs.storage_dir, self.VCS_MANIFEST_NAME),
"w") as fp:
json.dump({
"name": name,
"version": vcs.get_current_revision(),
"url": url,
"requirements": requirements
}, fp)
src_manifest_dir = vcs.storage_dir
src_manifest['version'] = vcs.get_current_revision()
pkg_dir = self.find_pkg_root(tmp_dir)
# write source data to a special manifest
if track:
if not src_manifest_dir:
src_manifest_dir = join(pkg_dir, ".pio")
self._update_src_manifest(src_manifest, src_manifest_dir)
pkg_dir = self.check_pkg_structure(tmp_dir)
pkg_dir = self._install_from_tmp_dir(pkg_dir, requirements)
finally:
if isdir(tmp_dir):
util.rmtree_(tmp_dir)
return pkg_dir
def _install_from_tmp_dir(self, tmp_dir, requirements=None):
tmp_manifest_path = self.get_manifest_path(tmp_dir)
tmp_manifest = self.load_manifest(tmp_manifest_path)
def _update_src_manifest(self, data, src_dir):
if not isdir(src_dir):
os.makedirs(src_dir)
src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME)
_data = data
if isfile(src_manifest_path):
_data.update(util.load_json(src_manifest_path))
with open(src_manifest_path, "w") as fp:
json.dump(_data, fp)
def _install_from_tmp_dir( # pylint: disable=too-many-branches
self, tmp_dir, requirements=None):
tmp_manifest = self.load_manifest(tmp_dir)
assert set(["name", "version"]) <= set(tmp_manifest.keys())
name = self.generate_install_dirname(tmp_manifest)
pkg_dir = join(self.package_dir, name)
pkg_dirname = self.get_install_dirname(tmp_manifest)
pkg_dir = join(self.package_dir, pkg_dirname)
cur_manifest = self.load_manifest(pkg_dir)
tmp_semver = None
cur_semver = None
try:
tmp_semver = semantic_version.Version(
tmp_manifest['version'], partial=True)
if cur_manifest:
cur_semver = semantic_version.Version(
cur_manifest['version'], partial=True)
except ValueError:
pass
# package should satisfy requirements
if requirements:
@@ -341,45 +419,51 @@ class PkgInstallerMixin(object):
"Package version %s doesn't satisfy requirements %s" % (
tmp_manifest['version'], requirements))
try:
reqspec = semantic_version.Spec(requirements)
tmp_version = semantic_version.Version(
tmp_manifest['version'], partial=True)
assert tmp_version in reqspec, mismatch_error
except ValueError:
assert tmp_semver and tmp_semver in semantic_version.Spec(
requirements), mismatch_error
except (AssertionError, ValueError):
assert tmp_manifest['version'] == requirements, mismatch_error
if self.manifest_exists(pkg_dir):
cur_manifest_path = self.get_manifest_path(pkg_dir)
cur_manifest = self.load_manifest(cur_manifest_path)
if tmp_manifest_path.endswith(self.VCS_MANIFEST_NAME):
if cur_manifest.get("url") != tmp_manifest['url']:
pkg_dir = join(self.package_dir, "%s@vcs-%s" % (
name, hashlib.md5(tmp_manifest['url']).hexdigest()))
# check if package already exists
if cur_manifest:
# 0-overwrite, 1-rename, 2-fix to a version
action = 0
if "__src_url" in cur_manifest:
if cur_manifest['__src_url'] != tmp_manifest.get("__src_url"):
action = 1
elif "__src_url" in tmp_manifest:
action = 2
else:
try:
tmp_version = semantic_version.Version(
tmp_manifest['version'], partial=True)
cur_version = semantic_version.Version(
cur_manifest['version'], partial=True)
if tmp_semver and (not cur_semver or tmp_semver > cur_semver):
action = 1
elif tmp_semver and cur_semver and tmp_semver != cur_semver:
action = 2
# if current package version < new package, backup it
if tmp_version > cur_version:
os.rename(pkg_dir,
join(self.package_dir, "%s@%s" %
(name, cur_manifest['version'])))
elif tmp_version < cur_version:
pkg_dir = join(self.package_dir, "%s@%s" %
(name, tmp_manifest['version']))
except ValueError:
pkg_dir = join(self.package_dir,
"%s@%s" % (name, tmp_manifest['version']))
# rename
if action == 1:
target_dirname = "%s@%s" % (pkg_dirname,
cur_manifest['version'])
if "__src_url" in cur_manifest:
target_dirname = "%s@src-%s" % (
pkg_dirname,
hashlib.md5(cur_manifest['__src_url']).hexdigest())
os.rename(pkg_dir, join(self.package_dir, target_dirname))
# fix to a version
elif action == 2:
target_dirname = "%s@%s" % (pkg_dirname,
tmp_manifest['version'])
if "__src_url" in tmp_manifest:
target_dirname = "%s@src-%s" % (
pkg_dirname,
hashlib.md5(tmp_manifest['__src_url']).hexdigest())
pkg_dir = join(self.package_dir, target_dirname)
# remove previous/not-satisfied package
if isdir(pkg_dir):
util.rmtree_(pkg_dir)
os.rename(tmp_dir, pkg_dir)
assert isdir(pkg_dir)
self.cache_reset()
return pkg_dir
@@ -400,14 +484,20 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl)
@staticmethod
def parse_pkg_name( # pylint: disable=too-many-branches
def parse_pkg_input( # pylint: disable=too-many-branches
text, requirements=None):
text = str(text)
url_marker = "://"
if not any([
requirements, "@" not in text, text.startswith("git@"),
url_marker in text
]):
# git@github.com:user/package.git
url_marker = text[:4]
if url_marker not in ("git@", "git+") or ":" not in text:
url_marker = "://"
req_conditions = [
not requirements, "@" in text,
(url_marker != "git@" and "://git@" not in text) or
text.count("@") > 1
]
if all(req_conditions):
text, requirements = text.rsplit("@", 1)
if text.isdigit():
text = "id=" + text
@@ -423,22 +513,18 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
url.startswith("http") and
(url.split("#", 1)[0] if "#" in url else url).endswith(".git")
]
if any(git_conditions):
url = "git+" + url
# Handle Developer Mbed URL
# (https://developer.mbed.org/users/user/code/package/)
elif url.startswith("https://developer.mbed.org"):
if url.startswith("https://developer.mbed.org"):
url = "hg+" + url
# git@github.com:user/package.git
if url.startswith("git@"):
url_marker = "git@"
if any([s in url for s in ("\\", "/")]) and url_marker not in url:
if isfile(url) or isdir(url):
url = "file://" + url
elif url.count("/") == 1 and not url.startswith("git@"):
elif url.count("/") == 1 and "git" not in url_marker:
url = "git+https://github.com/" + url
# determine name
@@ -448,55 +534,13 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
_url = _url[:-1]
name = basename(_url)
if "." in name and not name.startswith("."):
name = name.split(".", 1)[0]
name = name.rsplit(".", 1)[0]
if url_marker not in url:
url = None
return (name or text, requirements, url)
def get_package(self, name, requirements=None, url=None):
pkg_id = int(name[3:]) if name.startswith("id=") else 0
best = None
reqspec = None
if requirements:
try:
reqspec = semantic_version.Spec(requirements)
except ValueError:
pass
for manifest in self.get_installed():
if pkg_id and manifest.get("id") != pkg_id:
continue
elif not pkg_id and manifest['name'] != name:
continue
elif not reqspec and (requirements or url):
conds = [
requirements == manifest['version'], url and
url in manifest.get("url", "")
]
if not best or any(conds):
best = manifest
continue
try:
if reqspec and not reqspec.match(
semantic_version.Version(
manifest['version'], partial=True)):
continue
elif not best or (semantic_version.Version(
manifest['version'], partial=True) >
semantic_version.Version(
best['version'], partial=True)):
best = manifest
except ValueError:
pass
return best
def get_package_dir(self, name, requirements=None, url=None):
package = self.get_package(name, requirements, url)
return package.get("__pkg_dir") if package else None
def outdated(self, name, requirements=None, url=None):
def outdated(self, pkg_dir, requirements=None):
"""
Has 3 different results:
`None` - unknown package, VCS is fixed to commit
@@ -504,27 +548,26 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
`String` - a found latest version
"""
latest = None
package_dir = self.get_package_dir(name, requirements, url)
if not package_dir or ("@" in package_dir and
"@vcs-" not in package_dir):
manifest = self.load_manifest(pkg_dir)
# skip a fixed package to a specific version
if "@" in pkg_dir and "__src_url" not in manifest:
return None
is_vcs_pkg = False
manifest_path = self.get_vcs_manifest_path(package_dir)
if manifest_path:
is_vcs_pkg = True
manifest = self.load_manifest(manifest_path)
else:
manifest = self.load_manifest(package_dir)
if is_vcs_pkg:
vcs = VCSClientFactory.newClient(
package_dir, manifest['url'], silent=True)
if "__src_url" in manifest:
try:
vcs = VCSClientFactory.newClient(
pkg_dir, manifest['__src_url'], silent=True)
except (AttributeError, exception.PlatformioException):
return None
if not vcs.can_be_updated:
return None
latest = vcs.get_latest_revision()
else:
try:
latest = self.get_latest_repo_version(
name, requirements, silent=True)
"id=%d" % manifest['id']
if "id" in manifest else manifest['name'],
requirements,
silent=True)
except (exception.PlatformioException, ValueError):
return None
if not latest:
@@ -543,7 +586,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
silent=False,
trigger_event=True,
interactive=False): # pylint: disable=unused-argument
name, requirements, url = self.parse_pkg_name(name, requirements)
name, requirements, url = self.parse_pkg_input(name, requirements)
package_dir = self.get_package_dir(name, requirements, url)
if not package_dir or not silent:
@@ -560,15 +603,16 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
return package_dir
if url:
pkg_dir = self._install_from_url(name, url, requirements)
pkg_dir = self._install_from_url(
name, url, requirements, track=True)
else:
pkg_dir = self._install_from_piorepo(name, requirements)
if not pkg_dir or not self.manifest_exists(pkg_dir):
raise exception.PackageInstallError(name, requirements or "*",
util.get_systype())
self.reset_cache()
manifest = self.load_manifest(pkg_dir)
assert manifest
if trigger_event:
telemetry.on_event(
@@ -576,42 +620,45 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
action="Install",
label=manifest['name'])
click.secho(
"{name} @ {version} has been successfully installed!".format(
**manifest),
fg="green")
if not silent:
click.secho(
"{name} @ {version} has been successfully installed!".format(
**manifest),
fg="green")
return pkg_dir
def uninstall(self, name, requirements=None, trigger_event=True):
name, requirements, url = self.parse_pkg_name(name, requirements)
package_dir = self.get_package_dir(name, requirements, url)
if not package_dir:
click.secho(
"%s @ %s is not installed" % (name, requirements or "*"),
fg="yellow")
return
def uninstall(self, package, requirements=None, trigger_event=True):
if isdir(package):
pkg_dir = package
else:
name, requirements, url = self.parse_pkg_input(package,
requirements)
pkg_dir = self.get_package_dir(name, requirements, url)
manifest = self.load_manifest(package_dir)
if not pkg_dir:
raise exception.UnknownPackage("%s @ %s" %
(package, requirements or "*"))
manifest = self.load_manifest(pkg_dir)
click.echo(
"Uninstalling %s @ %s: \t" % (click.style(
manifest['name'], fg="cyan"), manifest['version']),
nl=False)
if isdir(package_dir):
if islink(package_dir):
os.unlink(package_dir)
else:
util.rmtree_(package_dir)
self.reset_cache()
if islink(pkg_dir):
os.unlink(pkg_dir)
else:
util.rmtree_(pkg_dir)
self.cache_reset()
# unfix package with the same name
package_dir = self.get_package_dir(manifest['name'])
if package_dir and "@" in package_dir:
os.rename(package_dir,
join(self.package_dir,
self.generate_install_dirname(manifest)))
self.reset_cache()
pkg_dir = self.get_package_dir(manifest['name'])
if pkg_dir and "@" in pkg_dir:
os.rename(
pkg_dir,
join(self.package_dir, self.get_install_dirname(manifest)))
self.cache_reset()
click.echo("[%s]" % click.style("OK", fg="green"))
@@ -624,25 +671,23 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
def update( # pylint: disable=too-many-return-statements
self,
name,
package,
requirements=None,
only_check=False):
name, requirements, url = self.parse_pkg_name(name, requirements)
package_dir = self.get_package_dir(name, requirements, url)
if not package_dir:
click.secho(
"%s @ %s is not installed" % (name, requirements or "*"),
fg="yellow")
return
is_vcs_pkg = False
if self.get_vcs_manifest_path(package_dir):
is_vcs_pkg = True
manifest_path = self.get_vcs_manifest_path(package_dir)
if isdir(package):
pkg_dir = package
else:
manifest_path = self.get_manifest_path(package_dir)
name, requirements, url = self.parse_pkg_input(package,
requirements)
pkg_dir = self.get_package_dir(name, requirements, url)
if not pkg_dir:
raise exception.UnknownPackage("%s @ %s" %
(package, requirements or "*"))
manifest = self.load_manifest(pkg_dir)
name = manifest['name']
manifest = self.load_manifest(manifest_path)
click.echo(
"{} {:<40} @ {:<15}".format(
"Checking" if only_check else "Updating",
@@ -651,7 +696,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
if not util.internet_on():
click.echo("[%s]" % (click.style("Off-line", fg="yellow")))
return
latest = self.outdated(name, requirements, url)
latest = self.outdated(pkg_dir, requirements)
if latest:
click.echo("[%s]" % (click.style(latest, fg="red")))
elif latest is False:
@@ -659,26 +705,18 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
else:
click.echo("[%s]" % (click.style("Skip", fg="yellow")))
if only_check or latest is False or (not is_vcs_pkg and not latest):
if only_check or not latest:
return
if is_vcs_pkg:
vcs = VCSClientFactory.newClient(package_dir, manifest['url'])
if not vcs.can_be_updated:
click.secho(
"Skip update because repository is fixed "
"to %s revision" % manifest['version'],
fg="yellow")
return
if "__src_url" in manifest:
vcs = VCSClientFactory.newClient(pkg_dir, manifest['__vcs_url'])
assert vcs.update()
with open(manifest_path, "w") as fp:
manifest['version'] = vcs.get_current_revision()
json.dump(manifest, fp)
self._update_src_manifest(
dict(version=vcs.get_current_revision()), vcs.storage_dir)
else:
self.uninstall(name, manifest['version'], trigger_event=False)
self.uninstall(pkg_dir, trigger_event=False)
self.install(name, latest, trigger_event=False)
self.reset_cache()
telemetry.on_event(
category=self.__class__.__name__,
action="Update",
+54 -42
View File
@@ -63,7 +63,7 @@ class PlatformManager(BasePkgManager):
trigger_event=True,
**_): # pylint: disable=too-many-arguments
platform_dir = BasePkgManager.install(self, name, requirements)
p = PlatformFactory.newPlatform(self.get_manifest_path(platform_dir))
p = PlatformFactory.newPlatform(platform_dir)
# @Hook: when 'update' operation (trigger_event is False),
# don't cleanup packages or install them
@@ -75,10 +75,16 @@ class PlatformManager(BasePkgManager):
self.cleanup_packages(p.packages.keys())
return True
def uninstall(self, name, requirements=None, trigger_event=True):
name, requirements, _ = self.parse_pkg_name(name, requirements)
p = PlatformFactory.newPlatform(name, requirements)
BasePkgManager.uninstall(self, name, requirements)
def uninstall(self, package, requirements=None, trigger_event=True):
if isdir(package):
pkg_dir = package
else:
name, requirements, url = self.parse_pkg_input(package,
requirements)
pkg_dir = self.get_package_dir(name, requirements, url)
p = PlatformFactory.newPlatform(pkg_dir)
BasePkgManager.uninstall(self, pkg_dir, requirements)
# @Hook: when 'update' operation (trigger_event is False),
# don't cleanup packages or install them
@@ -90,18 +96,23 @@ class PlatformManager(BasePkgManager):
def update( # pylint: disable=arguments-differ
self,
name,
package,
requirements=None,
only_packages=False,
only_check=False):
name, requirements, _ = self.parse_pkg_name(name, requirements)
only_check=False,
only_packages=False):
if isdir(package):
pkg_dir = package
else:
name, requirements, url = self.parse_pkg_input(package,
requirements)
pkg_dir = self.get_package_dir(name, requirements, url)
p = PlatformFactory.newPlatform(name, requirements)
p = PlatformFactory.newPlatform(pkg_dir)
pkgs_before = pkgs_after = p.get_installed_packages().keys()
if not only_packages:
BasePkgManager.update(self, name, requirements, only_check)
p = PlatformFactory.newPlatform(name, requirements)
BasePkgManager.update(self, pkg_dir, requirements, only_check)
p = PlatformFactory.newPlatform(pkg_dir)
pkgs_after = p.get_installed_packages().keys()
p.update_packages(only_check)
@@ -115,11 +126,10 @@ class PlatformManager(BasePkgManager):
return True
def cleanup_packages(self, names):
self.reset_cache()
self.cache_reset()
deppkgs = {}
for manifest in PlatformManager().get_installed():
p = PlatformFactory.newPlatform(manifest['name'],
manifest['version'])
p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
for pkgname, pkgmanifest in p.get_installed_packages().items():
if pkgname not in deppkgs:
deppkgs[pkgname] = set()
@@ -131,17 +141,15 @@ class PlatformManager(BasePkgManager):
continue
if (manifest['name'] not in deppkgs or
manifest['version'] not in deppkgs[manifest['name']]):
pm.uninstall(
manifest['name'], manifest['version'], trigger_event=False)
pm.uninstall(manifest['__pkg_dir'], trigger_event=False)
self.reset_cache()
self.cache_reset()
return True
def get_installed_boards(self):
boards = []
for manifest in self.get_installed():
p = PlatformFactory.newPlatform(
self.get_manifest_path(manifest['__pkg_dir']))
p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
for config in p.get_boards().values():
board = config.get_brief_data()
if board not in boards:
@@ -183,7 +191,10 @@ class PlatformFactory(object):
@classmethod
def newPlatform(cls, name, requirements=None):
platform_dir = None
if name.endswith("platform.json") and isfile(name):
if isdir(name):
platform_dir = name
name = PlatformManager().load_manifest(platform_dir)['name']
elif name.endswith("platform.json") and isfile(name):
platform_dir = dirname(name)
name = util.load_json(name)['name']
else:
@@ -249,8 +260,8 @@ class PlatformPackagesMixin(object):
if self.is_valid_requirements(version):
package = self.pm.get_package(name, version)
else:
package = self.pm.get_package(*self._parse_pkg_name(name,
version))
package = self.pm.get_package(*self._parse_pkg_input(name,
version))
if package:
items[name] = package
return items
@@ -267,46 +278,47 @@ class PlatformPackagesMixin(object):
self.pm.update("%s=%s" % (name, version), requirements,
only_check)
def are_outdated_packages(self):
latest = None
for name in self.get_installed_packages():
version = self.packages[name].get("version", "")
if self.is_valid_requirements(version):
latest = self.pm.outdated(name, version)
else:
requirements = None
if "@" in version:
version, requirements = version.rsplit("@", 1)
latest = self.pm.outdated(name, requirements, version)
if latest or latest is None:
return True
return False
# def are_outdated_packages(self):
# latest = None
# for name in self.get_installed_packages():
# version = self.packages[name].get("version", "")
# if self.is_valid_requirements(version):
# latest = self.pm.outdated(name, version)
# else:
# requirements = None
# if "@" in version:
# version, requirements = version.rsplit("@", 1)
# latest = self.pm.outdated(name, requirements, version)
# if latest or latest is None:
# return True
# return False
def get_package_dir(self, name):
version = self.packages[name].get("version", "")
if self.is_valid_requirements(version):
return self.pm.get_package_dir(name, version)
else:
return self.pm.get_package_dir(*self._parse_pkg_name(name,
version))
return self.pm.get_package_dir(*self._parse_pkg_input(name,
version))
def get_package_version(self, name):
version = self.packages[name].get("version", "")
if self.is_valid_requirements(version):
package = self.pm.get_package(name, version)
else:
package = self.pm.get_package(*self._parse_pkg_name(name, version))
package = self.pm.get_package(*self._parse_pkg_input(name,
version))
return package['version'] if package else None
@staticmethod
def is_valid_requirements(requirements):
return requirements and "://" not in requirements
def _parse_pkg_name(self, name, version):
def _parse_pkg_input(self, name, version):
requirements = None
if "@" in version:
version, requirements = version.rsplit("@", 1)
return self.pm.parse_pkg_name("%s=%s" % (name, version), requirements)
return self.pm.parse_pkg_input("%s=%s" % (name, version), requirements)
class PlatformRunMixin(object):
+6 -1
View File
@@ -141,7 +141,12 @@ class memoized(object):
def __get__(self, obj, objtype):
'''Support instance methods.'''
return functools.partial(self.__call__, obj)
fn = functools.partial(self.__call__, obj)
fn.reset = self._reset
return fn
def _reset(self):
self.cache = {}
def singleton(cls):