feedc0de/platformio-core
1
0
Fork 0
forked from platformio/platformio-core
Code Pull Requests Activity

“pio lib update” and “pio platform update” in JSON format

Browse Source
This commit is contained in:
Ivan Kravets
2017-01-30 01:04:06 +02:00
parent c1e14b671c
commit 34eab69e85
8 changed files with 231 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs

Submodule docs updated: 24e041602f...d019f41070

2
platformio/__init__.py
View File

@ -14,7 +14,7 @@
import sys
VERSION = (3, 3, "0a7")
VERSION = (3, 3, "0a8")
__version__ = ".".join([str(s) for s in VERSION])
__title__ = "platformio"

31
platformio/commands/lib.py
View File

@ -112,12 +112,35 @@ def lib_uninstall(lm, libraries):
"--only-check",
is_flag=True,
help="Do not update, only check for new version")
@click.option("--json-output", is_flag=True)
@click.pass_obj
def lib_update(lm, libraries, only_check):
def lib_update(lm, libraries, only_check, json_output):
if not libraries:
libraries = [str(m.get("id", m['name'])) for m in lm.get_installed()]
for library in libraries:
lm.update(library, only_check=only_check)
libraries = []
for manifest in lm.get_installed():
pkg_dir = manifest['__pkg_dir']
if "@" in pkg_dir and "@vcs-" not in pkg_dir:
continue
elif "@vcs-" in pkg_dir:
libraries.append("%s=%s" % (manifest['name'], manifest['url']))
else:
libraries.append(str(manifest.get("id", manifest['name'])))
if only_check and json_output:
result = []
for library in libraries:
name, requirements, url = lm.parse_pkg_name(library)
latest = lm.outdated(name, requirements, url)
if latest is False:
continue
manifest = lm.load_manifest(
lm.get_package_dir(name, requirements, url))
manifest['versionLatest'] = latest or "Unknown"
result.append(manifest)
return click.echo(json.dumps(result))
else:
for library in libraries:
lm.update(library, only_check=only_check)
def print_lib_item(item):

41
platformio/commands/platform.py
View File

@ -114,15 +114,42 @@ def platform_uninstall(platforms):
"--only-check",
is_flag=True,
help="Do not update, only check for new version")
def platform_update(platforms, only_packages, only_check):
@click.option("--json-output", is_flag=True)
def platform_update(platforms, only_packages, only_check, json_output):
pm = PlatformManager()
if not platforms:
platforms = set([m['name'] for m in pm.get_installed()])
for platform in platforms:
click.echo("Platform %s" % click.style(platform, fg="cyan"))
click.echo("--------")
pm.update(platform, only_packages=only_packages, only_check=only_check)
click.echo()
platforms = []
for manifest in pm.get_installed():
pkg_dir = manifest['__pkg_dir']
if "@" in pkg_dir and "@vcs-" not in pkg_dir:
continue
elif "@vcs-" in pkg_dir:
platforms.append("%s=%s" % (manifest['name'], manifest['url']))
else:
platforms.append(manifest['name'])
if only_check and json_output:
result = []
for platform in platforms:
name, requirements, url = pm.parse_pkg_name(platform)
latest = pm.outdated(name, requirements, url)
if latest is False:
continue
manifest = pm.load_manifest(
pm.get_package_dir(name, requirements, url))
if latest is True:
manifest['versionLatest'] = "Out-of-date"
else:
manifest['versionLatest'] = latest or "Unknown"
result.append(manifest)
return click.echo(json.dumps(result))
else:
for platform in platforms:
click.echo("Platform %s" % click.style(platform, fg="cyan"))
click.echo("--------")
pm.update(
platform, only_packages=only_packages, only_check=only_check)
click.echo()
@cli.command("list", short_help="List installed development platforms")

2
platformio/maintenance.py
View File

@ -268,7 +268,7 @@ def check_internal_updates(ctx, what):
outdated_items = []
for manifest in pm.get_installed():
if manifest['name'] not in outdated_items and \
pm.is_outdated(manifest['name']):
pm.outdated(manifest['name']):
outdated_items.append(manifest['name'])
if not outdated_items:

226
platformio/managers/package.py
View File

@ -119,6 +119,47 @@ class PkgInstallerMixin(object):
VCS_MANIFEST_NAME = ".piopkgmanager.json"
FILE_CACHE_VALID = "1m" # 1 month
FILE_CACHE_MAX_SIZE = 1024 * 1024
_INSTALLED_CACHE = {}
def reset_cache(self):
if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
del PkgInstallerMixin._INSTALLED_CACHE[self.package_dir]
def download(self, url, dest_dir, sha1=None):
cache_key_fname = app.ContentCache.key_from_args(url, "fname")
cache_key_data = app.ContentCache.key_from_args(url, "data")
if self.FILE_CACHE_VALID:
with app.ContentCache() as cc:
fname = cc.get(cache_key_fname)
cache_path = cc.get_cache_path(cache_key_data)
if fname and isfile(cache_path):
dst_path = join(dest_dir, fname)
shutil.copy(cache_path, dst_path)
return dst_path
fd = FileDownloader(url, dest_dir)
fd.start()
if sha1:
fd.verify(sha1)
dst_path = fd.get_filepath()
if not self.FILE_CACHE_VALID or getsize(
dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE:
return dst_path
with app.ContentCache() as cc:
cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
return dst_path
@staticmethod
def unpack(source_path, dest_dir):
fu = FileUnpacker(source_path, dest_dir)
return fu.start()
def get_vcs_manifest_path(self, pkg_dir):
for item in os.listdir(pkg_dir):
if not isdir(join(pkg_dir, item)):
@ -142,7 +183,7 @@ class PkgInstallerMixin(object):
def manifest_exists(self, pkg_dir):
return self.get_manifest_path(pkg_dir) is not None
def load_manifest(self, path):
def load_manifest(self, path): # pylint: disable=too-many-branches
assert path
pkg_dir = path
if isdir(path):
@ -155,6 +196,13 @@ class PkgInstallerMixin(object):
if isfile(path) and path.endswith(self.VCS_MANIFEST_NAME):
pkg_dir = dirname(dirname(path))
# return from cache
if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
for manifest in PkgInstallerMixin._INSTALLED_CACHE[self.
package_dir]:
if manifest['__pkg_dir'] == pkg_dir:
return manifest
manifest = {}
if path.endswith(".json"):
manifest = util.load_json(path)
@ -174,6 +222,22 @@ class PkgInstallerMixin(object):
manifest['__pkg_dir'] = pkg_dir
return manifest
def get_installed(self):
if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
return PkgInstallerMixin._INSTALLED_CACHE[self.package_dir]
items = []
for p in sorted(os.listdir(self.package_dir)):
pkg_dir = join(self.package_dir, p)
if not isdir(pkg_dir):
continue
manifest = self.load_manifest(pkg_dir)
if not manifest:
continue
assert "name" in manifest
items.append(manifest)
PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] = items
return items
def check_pkg_structure(self, pkg_dir):
if self.manifest_exists(pkg_dir):
return pkg_dir
@ -305,11 +369,6 @@ class PkgInstallerMixin(object):
class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
_INSTALLED_CACHE = {}
FILE_CACHE_VALID = "1m" # 1 month
FILE_CACHE_MAX_SIZE = 1024 * 1024
def __init__(self, package_dir, repositories=None):
self.repositories = repositories
self.package_dir = package_dir
@ -321,42 +380,6 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
def manifest_names(self):
raise NotImplementedError()
def download(self, url, dest_dir, sha1=None):
cache_key_fname = app.ContentCache.key_from_args(url, "fname")
cache_key_data = app.ContentCache.key_from_args(url, "data")
if self.FILE_CACHE_VALID:
with app.ContentCache() as cc:
fname = cc.get(cache_key_fname)
cache_path = cc.get_cache_path(cache_key_data)
if fname and isfile(cache_path):
dst_path = join(dest_dir, fname)
shutil.copy(cache_path, dst_path)
return dst_path
fd = FileDownloader(url, dest_dir)
fd.start()
if sha1:
fd.verify(sha1)
dst_path = fd.get_filepath()
if not self.FILE_CACHE_VALID or getsize(
dst_path) > BasePkgManager.FILE_CACHE_MAX_SIZE:
return dst_path
with app.ContentCache() as cc:
cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
return dst_path
@staticmethod
def unpack(source_path, dest_dir):
fu = FileUnpacker(source_path, dest_dir)
return fu.start()
def reset_cache(self):
if self.package_dir in BasePkgManager._INSTALLED_CACHE:
del BasePkgManager._INSTALLED_CACHE[self.package_dir]
def print_message(self, message, nl=True):
click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl)
@ -415,22 +438,6 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
url = None
return (name or text, requirements, url)
def get_installed(self):
if self.package_dir in BasePkgManager._INSTALLED_CACHE:
return BasePkgManager._INSTALLED_CACHE[self.package_dir]
items = []
for p in sorted(os.listdir(self.package_dir)):
pkg_dir = join(self.package_dir, p)
if not isdir(pkg_dir):
continue
manifest = self.load_manifest(pkg_dir)
if not manifest:
continue
assert "name" in manifest
items.append(manifest)
BasePkgManager._INSTALLED_CACHE[self.package_dir] = items
return items
def get_package(self, name, requirements=None, url=None):
pkg_id = int(name[3:]) if name.startswith("id=") else 0
best = None
@ -473,20 +480,38 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
package = self.get_package(name, requirements, url)
return package.get("__pkg_dir") if package else None
def is_outdated(self, name, requirements=None, silent=False):
package_dir = self.get_package_dir(name, requirements)
def outdated(self, name, requirements=None, url=None):
"""
Has 3 different results:
`None` - unknown package, VCS is fixed to commit
`False` - package is up-to-date
`String` - a found latest version
"""
latest = None
package_dir = self.get_package_dir(name, requirements, url)
if not package_dir:
if silent:
return
click.secho(
"%s @ %s is not installed" % (name, requirements or "*"),
fg="yellow")
return
if self.get_vcs_manifest_path(package_dir):
return False
return None
manifest = self.load_manifest(package_dir)
latest = self.get_latest_repo_version(name, requirements)
return latest and manifest['version'] != latest
if self.get_vcs_manifest_path(package_dir):
vcs = VCSClientFactory.newClient(
package_dir, manifest['url'], silent=True)
if not vcs.can_be_updated:
return None
latest = vcs.get_latest_revision()
else:
try:
latest = self.get_latest_repo_version(name, requirements)
except (exception.PlatformioException, ValueError):
return None
if not latest:
return None
up_to_date = False
try:
up_to_date = (semantic_version.Version.coerce(manifest['version'])
>= semantic_version.Version.coerce(latest))
except ValueError:
up_to_date = latest == manifest['version']
return False if up_to_date else latest
def install(self,
name,
@ -571,7 +596,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
requirements=None,
only_check=False):
name, requirements, url = self.parse_pkg_name(name, requirements)
package_dir = self.get_package_dir(name, None, url)
package_dir = self.get_package_dir(name, requirements, url)
if not package_dir:
click.secho(
"%s @ %s is not installed" % (name, requirements or "*"),
@ -587,16 +612,29 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
manifest = self.load_manifest(manifest_path)
click.echo(
"%s %s @ %s: \t" % ("Checking"
if only_check else "Updating", click.style(
manifest['name'], fg="cyan"),
manifest['version']),
"{} {:<40} @ {:<15}".format(
"Checking" if only_check else "Updating",
click.style(
manifest['name'], fg="cyan"),
manifest['version']),
nl=False)
if not util.internet_on():
click.echo("[%s]" % (click.style("Off-line", fg="yellow")))
return
latest = self.outdated(name, requirements, url)
if latest is True:
click.echo("[%s]" % (click.style("Out-of-date", fg="red")))
elif latest:
click.echo("[%s]" % (click.style(latest, fg="red")))
elif latest is False:
click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
else:
click.echo("[%s]" % (click.style("Unknown", fg="yellow")))
if only_check or latest is False or (not is_vcs_pkg and not latest):
return
if is_vcs_pkg:
if only_check:
click.echo("[%s]" % (click.style("Skip", fg="yellow")))
return
click.echo("[%s]" % (click.style("VCS", fg="yellow")))
vcs = VCSClientFactory.newClient(package_dir, manifest['url'])
if not vcs.can_be_updated:
click.secho(
@ -608,36 +646,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
with open(manifest_path, "w") as fp:
manifest['version'] = vcs.get_current_revision()
json.dump(manifest, fp)
self.reset_cache()
else:
latest_version = None
try:
latest_version = self.get_latest_repo_version(name,
requirements)
except exception.PlatformioException:
pass
if not latest_version:
click.echo("[%s]" % (click.style(
"Off-line" if not util.internet_on() else "Unknown",
fg="yellow")))
return
up_to_date = False
try:
up_to_date = (
semantic_version.Version.coerce(manifest['version']) >=
semantic_version.Version.coerce(latest_version))
except ValueError:
up_to_date = latest_version == manifest['version']
if up_to_date:
click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
return
click.echo("[%s]" % (click.style("Out-of-date", fg="red")))
if only_check:
return
self.uninstall(name, manifest['version'], trigger_event=False)
self.install(name, latest_version, trigger_event=False)
self.install(name, latest, trigger_event=False)
telemetry.on_event(
category=self.__class__.__name__,

35
platformio/managers/platform.py
View File

@ -92,9 +92,10 @@ class PlatformManager(BasePkgManager):
self.cleanup_packages(p.packages.keys())
return True
def is_outdated(self, name, requirements=None, silent=False):
if BasePkgManager.is_outdated(self, name, requirements, silent):
return True
def outdated(self, name, requirements=None, url=None):
latest = BasePkgManager.outdated(self, name, requirements, url)
if latest:
return latest
p = PlatformFactory.newPlatform(name, requirements)
return p.are_outdated_packages()
@ -213,7 +214,7 @@ class PlatformPackagesMixin(object):
continue
elif (name in with_packages or
not (skip_default_package or opts.get("optional", False))):
if self.validate_version_requirements(version):
if self.is_valid_requirements(version):
self.pm.install(name, version, silent=silent)
else:
requirements = None
@ -228,7 +229,7 @@ class PlatformPackagesMixin(object):
items = {}
for name, opts in self.packages.items():
version = opts.get("version", "")
if self.validate_version_requirements(version):
if self.is_valid_requirements(version):
package = self.pm.get_package(name, version)
else:
package = self.pm.get_package(*self._parse_pkg_name(name,
@ -240,7 +241,7 @@ class PlatformPackagesMixin(object):
def update_packages(self, only_check=False):
for name in self.get_installed_packages():
version = self.packages[name].get("version", "")
if self.validate_version_requirements(version):
if self.is_valid_requirements(version):
self.pm.update(name, version, only_check)
else:
requirements = None
@ -250,17 +251,23 @@ class PlatformPackagesMixin(object):
only_check)
def are_outdated_packages(self):
for name, opts in self.packages.items():
version = opts.get("version", "")
if not self.validate_version_requirements(version):
continue
if self.pm.is_outdated(name, version, silent=True):
latest = None
for name in self.get_installed_packages():
version = self.packages[name].get("version", "")
if self.is_valid_requirements(version):
latest = self.pm.outdated(name, version)
else:
requirements = None
if "@" in version:
version, requirements = version.rsplit("@", 1)
latest = self.pm.outdated(name, requirements, version)
if latest or latest is None:
return True
return False
def get_package_dir(self, name):
version = self.packages[name].get("version", "")
if self.validate_version_requirements(version):
if self.is_valid_requirements(version):
return self.pm.get_package_dir(name, version)
else:
return self.pm.get_package_dir(*self._parse_pkg_name(name,
@ -268,14 +275,14 @@ class PlatformPackagesMixin(object):
def get_package_version(self, name):
version = self.packages[name].get("version", "")
if self.validate_version_requirements(version):
if self.is_valid_requirements(version):
package = self.pm.get_package(name, version)
else:
package = self.pm.get_package(*self._parse_pkg_name(name, version))
return package['version'] if package else None
@staticmethod
def validate_version_requirements(requirements):
def is_valid_requirements(requirements):
return requirements and "://" not in requirements
def _parse_pkg_name(self, name, version):

32
platformio/vcsclient.py
View File

@ -25,7 +25,7 @@ from platformio.exception import PlatformioException
class VCSClientFactory(object):
@staticmethod
def newClient(src_dir, remote_url):
def newClient(src_dir, remote_url, silent=False):
result = urlparse(remote_url)
type_ = result.scheme
tag = None
@ -40,7 +40,7 @@ class VCSClientFactory(object):
raise PlatformioException("VCS: Unknown repository type %s" %
remote_url)
obj = getattr(modules[__name__], "%sClient" % type_.title())(
src_dir, remote_url, tag)
src_dir, remote_url, tag, silent)
assert isinstance(obj, VCSClientBase)
return obj
@ -49,17 +49,21 @@ class VCSClientBase(object):
command = None
def __init__(self, src_dir, remote_url=None, tag=None):
def __init__(self, src_dir, remote_url=None, tag=None, silent=False):
self.src_dir = src_dir
self.remote_url = remote_url
self.tag = tag
self.silent = silent
self.check_client()
def check_client(self):
try:
assert self.command
assert self.run_cmd(["--version"])
except (AssertionError, OSError):
if self.silent:
self.get_cmd_output(["--version"])
else:
assert self.run_cmd(["--version"])
except (AssertionError, OSError, PlatformioException):
raise PlatformioException(
"VCS: `%s` client is not installed in your system" %
self.command)
@ -82,6 +86,9 @@ class VCSClientBase(object):
def get_current_revision(self):
raise NotImplementedError
def get_latest_revision(self):
return None if self.can_be_updated else self.get_current_revision()
def run_cmd(self, args, **kwargs):
args = [self.command] + args
if "cwd" not in kwargs:
@ -141,6 +148,16 @@ class GitClient(VCSClientBase):
def get_current_revision(self):
return self.get_cmd_output(["rev-parse", "--short", "HEAD"])
def get_latest_revision(self):
if not self.can_be_updated:
return self.get_latest_revision()
result = self.get_cmd_output(["ls-remote"])
for line in result.split("\n"):
line = line.strip()
if "HEAD" in line:
return line.split("HEAD", 1)[0].strip()[:7]
return None
class HgClient(VCSClientBase):
@ -160,6 +177,11 @@ class HgClient(VCSClientBase):
def get_current_revision(self):
return self.get_cmd_output(["identify", "--id"])
def get_latest_revision(self):
if not self.can_be_updated:
return self.get_latest_revision()
return self.get_cmd_output(["identify", "--id", self.remote_url])
class SvnClient(VCSClientBase):