diff --git a/docs b/docs index 24e04160..d019f410 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 24e041602f3d0765963772083b023a00b7cf039f +Subproject commit d019f41070f64d026b80c9cc612015c5b16d1cda diff --git a/platformio/__init__.py b/platformio/__init__.py index 72ecffd5..87c5830d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 3, "0a7") +VERSION = (3, 3, "0a8") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 5e3965fb..0e75548d 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -112,12 +112,35 @@ def lib_uninstall(lm, libraries): "--only-check", is_flag=True, help="Do not update, only check for new version") +@click.option("--json-output", is_flag=True) @click.pass_obj -def lib_update(lm, libraries, only_check): +def lib_update(lm, libraries, only_check, json_output): if not libraries: - libraries = [str(m.get("id", m['name'])) for m in lm.get_installed()] - for library in libraries: - lm.update(library, only_check=only_check) + libraries = [] + for manifest in lm.get_installed(): + pkg_dir = manifest['__pkg_dir'] + if "@" in pkg_dir and "@vcs-" not in pkg_dir: + continue + elif "@vcs-" in pkg_dir: + libraries.append("%s=%s" % (manifest['name'], manifest['url'])) + else: + libraries.append(str(manifest.get("id", manifest['name']))) + + if only_check and json_output: + result = [] + for library in libraries: + name, requirements, url = lm.parse_pkg_name(library) + latest = lm.outdated(name, requirements, url) + if latest is False: + continue + manifest = lm.load_manifest( + lm.get_package_dir(name, requirements, url)) + manifest['versionLatest'] = latest or "Unknown" + result.append(manifest) + return click.echo(json.dumps(result)) + else: + for library in libraries: + lm.update(library, only_check=only_check) def print_lib_item(item): diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 7821d68b..4f493229 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -114,15 +114,42 @@ def platform_uninstall(platforms): "--only-check", is_flag=True, help="Do not update, only check for new version") -def platform_update(platforms, only_packages, only_check): +@click.option("--json-output", is_flag=True) +def platform_update(platforms, only_packages, only_check, json_output): pm = PlatformManager() if not platforms: - platforms = set([m['name'] for m in pm.get_installed()]) - for platform in platforms: - click.echo("Platform %s" % click.style(platform, fg="cyan")) - click.echo("--------") - pm.update(platform, only_packages=only_packages, only_check=only_check) - click.echo() + platforms = [] + for manifest in pm.get_installed(): + pkg_dir = manifest['__pkg_dir'] + if "@" in pkg_dir and "@vcs-" not in pkg_dir: + continue + elif "@vcs-" in pkg_dir: + platforms.append("%s=%s" % (manifest['name'], manifest['url'])) + else: + platforms.append(manifest['name']) + + if only_check and json_output: + result = [] + for platform in platforms: + name, requirements, url = pm.parse_pkg_name(platform) + latest = pm.outdated(name, requirements, url) + if latest is False: + continue + manifest = pm.load_manifest( + pm.get_package_dir(name, requirements, url)) + if latest is True: + manifest['versionLatest'] = "Out-of-date" + else: + manifest['versionLatest'] = latest or "Unknown" + result.append(manifest) + return click.echo(json.dumps(result)) + else: + for platform in platforms: + click.echo("Platform %s" % click.style(platform, fg="cyan")) + click.echo("--------") + pm.update( + platform, only_packages=only_packages, only_check=only_check) + click.echo() @cli.command("list", short_help="List installed development platforms") diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 3925741e..cd1e3562 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -268,7 +268,7 @@ def check_internal_updates(ctx, what): outdated_items = [] for manifest in pm.get_installed(): if manifest['name'] not in outdated_items and \ - pm.is_outdated(manifest['name']): + pm.outdated(manifest['name']): outdated_items.append(manifest['name']) if not outdated_items: diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 8450ff52..10b0cd18 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -119,6 +119,47 @@ class PkgInstallerMixin(object): VCS_MANIFEST_NAME = ".piopkgmanager.json" + FILE_CACHE_VALID = "1m" # 1 month + FILE_CACHE_MAX_SIZE = 1024 * 1024 + + _INSTALLED_CACHE = {} + + def reset_cache(self): + if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE: + del PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] + + def download(self, url, dest_dir, sha1=None): + cache_key_fname = app.ContentCache.key_from_args(url, "fname") + cache_key_data = app.ContentCache.key_from_args(url, "data") + if self.FILE_CACHE_VALID: + with app.ContentCache() as cc: + fname = cc.get(cache_key_fname) + cache_path = cc.get_cache_path(cache_key_data) + if fname and isfile(cache_path): + dst_path = join(dest_dir, fname) + shutil.copy(cache_path, dst_path) + return dst_path + + fd = FileDownloader(url, dest_dir) + fd.start() + if sha1: + fd.verify(sha1) + dst_path = fd.get_filepath() + if not self.FILE_CACHE_VALID or getsize( + dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE: + return dst_path + + with app.ContentCache() as cc: + cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID) + cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID) + shutil.copy(dst_path, cc.get_cache_path(cache_key_data)) + return dst_path + + @staticmethod + def unpack(source_path, dest_dir): + fu = FileUnpacker(source_path, dest_dir) + return fu.start() + def get_vcs_manifest_path(self, pkg_dir): for item in os.listdir(pkg_dir): if not isdir(join(pkg_dir, item)): @@ -142,7 +183,7 @@ class PkgInstallerMixin(object): def manifest_exists(self, pkg_dir): return self.get_manifest_path(pkg_dir) is not None - def load_manifest(self, path): + def load_manifest(self, path): # pylint: disable=too-many-branches assert path pkg_dir = path if isdir(path): @@ -155,6 +196,13 @@ class PkgInstallerMixin(object): if isfile(path) and path.endswith(self.VCS_MANIFEST_NAME): pkg_dir = dirname(dirname(path)) + # return from cache + if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE: + for manifest in PkgInstallerMixin._INSTALLED_CACHE[self. + package_dir]: + if manifest['__pkg_dir'] == pkg_dir: + return manifest + manifest = {} if path.endswith(".json"): manifest = util.load_json(path) @@ -174,6 +222,22 @@ class PkgInstallerMixin(object): manifest['__pkg_dir'] = pkg_dir return manifest + def get_installed(self): + if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE: + return PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] + items = [] + for p in sorted(os.listdir(self.package_dir)): + pkg_dir = join(self.package_dir, p) + if not isdir(pkg_dir): + continue + manifest = self.load_manifest(pkg_dir) + if not manifest: + continue + assert "name" in manifest + items.append(manifest) + PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] = items + return items + def check_pkg_structure(self, pkg_dir): if self.manifest_exists(pkg_dir): return pkg_dir @@ -305,11 +369,6 @@ class PkgInstallerMixin(object): class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): - _INSTALLED_CACHE = {} - - FILE_CACHE_VALID = "1m" # 1 month - FILE_CACHE_MAX_SIZE = 1024 * 1024 - def __init__(self, package_dir, repositories=None): self.repositories = repositories self.package_dir = package_dir @@ -321,42 +380,6 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): def manifest_names(self): raise NotImplementedError() - def download(self, url, dest_dir, sha1=None): - cache_key_fname = app.ContentCache.key_from_args(url, "fname") - cache_key_data = app.ContentCache.key_from_args(url, "data") - if self.FILE_CACHE_VALID: - with app.ContentCache() as cc: - fname = cc.get(cache_key_fname) - cache_path = cc.get_cache_path(cache_key_data) - if fname and isfile(cache_path): - dst_path = join(dest_dir, fname) - shutil.copy(cache_path, dst_path) - return dst_path - - fd = FileDownloader(url, dest_dir) - fd.start() - if sha1: - fd.verify(sha1) - dst_path = fd.get_filepath() - if not self.FILE_CACHE_VALID or getsize( - dst_path) > BasePkgManager.FILE_CACHE_MAX_SIZE: - return dst_path - - with app.ContentCache() as cc: - cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID) - cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID) - shutil.copy(dst_path, cc.get_cache_path(cache_key_data)) - return dst_path - - @staticmethod - def unpack(source_path, dest_dir): - fu = FileUnpacker(source_path, dest_dir) - return fu.start() - - def reset_cache(self): - if self.package_dir in BasePkgManager._INSTALLED_CACHE: - del BasePkgManager._INSTALLED_CACHE[self.package_dir] - def print_message(self, message, nl=True): click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) @@ -415,22 +438,6 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): url = None return (name or text, requirements, url) - def get_installed(self): - if self.package_dir in BasePkgManager._INSTALLED_CACHE: - return BasePkgManager._INSTALLED_CACHE[self.package_dir] - items = [] - for p in sorted(os.listdir(self.package_dir)): - pkg_dir = join(self.package_dir, p) - if not isdir(pkg_dir): - continue - manifest = self.load_manifest(pkg_dir) - if not manifest: - continue - assert "name" in manifest - items.append(manifest) - BasePkgManager._INSTALLED_CACHE[self.package_dir] = items - return items - def get_package(self, name, requirements=None, url=None): pkg_id = int(name[3:]) if name.startswith("id=") else 0 best = None @@ -473,20 +480,38 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): package = self.get_package(name, requirements, url) return package.get("__pkg_dir") if package else None - def is_outdated(self, name, requirements=None, silent=False): - package_dir = self.get_package_dir(name, requirements) + def outdated(self, name, requirements=None, url=None): + """ + Has 3 different results: + `None` - unknown package, VCS is fixed to commit + `False` - package is up-to-date + `String` - a found latest version + """ + latest = None + package_dir = self.get_package_dir(name, requirements, url) if not package_dir: - if silent: - return - click.secho( - "%s @ %s is not installed" % (name, requirements or "*"), - fg="yellow") - return - if self.get_vcs_manifest_path(package_dir): - return False + return None manifest = self.load_manifest(package_dir) - latest = self.get_latest_repo_version(name, requirements) - return latest and manifest['version'] != latest + if self.get_vcs_manifest_path(package_dir): + vcs = VCSClientFactory.newClient( + package_dir, manifest['url'], silent=True) + if not vcs.can_be_updated: + return None + latest = vcs.get_latest_revision() + else: + try: + latest = self.get_latest_repo_version(name, requirements) + except (exception.PlatformioException, ValueError): + return None + if not latest: + return None + up_to_date = False + try: + up_to_date = (semantic_version.Version.coerce(manifest['version']) + >= semantic_version.Version.coerce(latest)) + except ValueError: + up_to_date = latest == manifest['version'] + return False if up_to_date else latest def install(self, name, @@ -571,7 +596,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): requirements=None, only_check=False): name, requirements, url = self.parse_pkg_name(name, requirements) - package_dir = self.get_package_dir(name, None, url) + package_dir = self.get_package_dir(name, requirements, url) if not package_dir: click.secho( "%s @ %s is not installed" % (name, requirements or "*"), @@ -587,16 +612,29 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): manifest = self.load_manifest(manifest_path) click.echo( - "%s %s @ %s: \t" % ("Checking" - if only_check else "Updating", click.style( - manifest['name'], fg="cyan"), - manifest['version']), + "{} {:<40} @ {:<15}".format( + "Checking" if only_check else "Updating", + click.style( + manifest['name'], fg="cyan"), + manifest['version']), nl=False) + if not util.internet_on(): + click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) + return + latest = self.outdated(name, requirements, url) + if latest is True: + click.echo("[%s]" % (click.style("Out-of-date", fg="red"))) + elif latest: + click.echo("[%s]" % (click.style(latest, fg="red"))) + elif latest is False: + click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) + else: + click.echo("[%s]" % (click.style("Unknown", fg="yellow"))) + + if only_check or latest is False or (not is_vcs_pkg and not latest): + return + if is_vcs_pkg: - if only_check: - click.echo("[%s]" % (click.style("Skip", fg="yellow"))) - return - click.echo("[%s]" % (click.style("VCS", fg="yellow"))) vcs = VCSClientFactory.newClient(package_dir, manifest['url']) if not vcs.can_be_updated: click.secho( @@ -608,36 +646,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): with open(manifest_path, "w") as fp: manifest['version'] = vcs.get_current_revision() json.dump(manifest, fp) + self.reset_cache() else: - latest_version = None - try: - latest_version = self.get_latest_repo_version(name, - requirements) - except exception.PlatformioException: - pass - if not latest_version: - click.echo("[%s]" % (click.style( - "Off-line" if not util.internet_on() else "Unknown", - fg="yellow"))) - return - - up_to_date = False - try: - up_to_date = ( - semantic_version.Version.coerce(manifest['version']) >= - semantic_version.Version.coerce(latest_version)) - except ValueError: - up_to_date = latest_version == manifest['version'] - - if up_to_date: - click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) - return - - click.echo("[%s]" % (click.style("Out-of-date", fg="red"))) - if only_check: - return self.uninstall(name, manifest['version'], trigger_event=False) - self.install(name, latest_version, trigger_event=False) + self.install(name, latest, trigger_event=False) telemetry.on_event( category=self.__class__.__name__, diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 85d963a2..caf41b99 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -92,9 +92,10 @@ class PlatformManager(BasePkgManager): self.cleanup_packages(p.packages.keys()) return True - def is_outdated(self, name, requirements=None, silent=False): - if BasePkgManager.is_outdated(self, name, requirements, silent): - return True + def outdated(self, name, requirements=None, url=None): + latest = BasePkgManager.outdated(self, name, requirements, url) + if latest: + return latest p = PlatformFactory.newPlatform(name, requirements) return p.are_outdated_packages() @@ -213,7 +214,7 @@ class PlatformPackagesMixin(object): continue elif (name in with_packages or not (skip_default_package or opts.get("optional", False))): - if self.validate_version_requirements(version): + if self.is_valid_requirements(version): self.pm.install(name, version, silent=silent) else: requirements = None @@ -228,7 +229,7 @@ class PlatformPackagesMixin(object): items = {} for name, opts in self.packages.items(): version = opts.get("version", "") - if self.validate_version_requirements(version): + if self.is_valid_requirements(version): package = self.pm.get_package(name, version) else: package = self.pm.get_package(*self._parse_pkg_name(name, @@ -240,7 +241,7 @@ class PlatformPackagesMixin(object): def update_packages(self, only_check=False): for name in self.get_installed_packages(): version = self.packages[name].get("version", "") - if self.validate_version_requirements(version): + if self.is_valid_requirements(version): self.pm.update(name, version, only_check) else: requirements = None @@ -250,17 +251,23 @@ class PlatformPackagesMixin(object): only_check) def are_outdated_packages(self): - for name, opts in self.packages.items(): - version = opts.get("version", "") - if not self.validate_version_requirements(version): - continue - if self.pm.is_outdated(name, version, silent=True): + latest = None + for name in self.get_installed_packages(): + version = self.packages[name].get("version", "") + if self.is_valid_requirements(version): + latest = self.pm.outdated(name, version) + else: + requirements = None + if "@" in version: + version, requirements = version.rsplit("@", 1) + latest = self.pm.outdated(name, requirements, version) + if latest or latest is None: return True return False def get_package_dir(self, name): version = self.packages[name].get("version", "") - if self.validate_version_requirements(version): + if self.is_valid_requirements(version): return self.pm.get_package_dir(name, version) else: return self.pm.get_package_dir(*self._parse_pkg_name(name, @@ -268,14 +275,14 @@ class PlatformPackagesMixin(object): def get_package_version(self, name): version = self.packages[name].get("version", "") - if self.validate_version_requirements(version): + if self.is_valid_requirements(version): package = self.pm.get_package(name, version) else: package = self.pm.get_package(*self._parse_pkg_name(name, version)) return package['version'] if package else None @staticmethod - def validate_version_requirements(requirements): + def is_valid_requirements(requirements): return requirements and "://" not in requirements def _parse_pkg_name(self, name, version): diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index b06df1f3..a5bba681 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -25,7 +25,7 @@ from platformio.exception import PlatformioException class VCSClientFactory(object): @staticmethod - def newClient(src_dir, remote_url): + def newClient(src_dir, remote_url, silent=False): result = urlparse(remote_url) type_ = result.scheme tag = None @@ -40,7 +40,7 @@ class VCSClientFactory(object): raise PlatformioException("VCS: Unknown repository type %s" % remote_url) obj = getattr(modules[__name__], "%sClient" % type_.title())( - src_dir, remote_url, tag) + src_dir, remote_url, tag, silent) assert isinstance(obj, VCSClientBase) return obj @@ -49,17 +49,21 @@ class VCSClientBase(object): command = None - def __init__(self, src_dir, remote_url=None, tag=None): + def __init__(self, src_dir, remote_url=None, tag=None, silent=False): self.src_dir = src_dir self.remote_url = remote_url self.tag = tag + self.silent = silent self.check_client() def check_client(self): try: assert self.command - assert self.run_cmd(["--version"]) - except (AssertionError, OSError): + if self.silent: + self.get_cmd_output(["--version"]) + else: + assert self.run_cmd(["--version"]) + except (AssertionError, OSError, PlatformioException): raise PlatformioException( "VCS: `%s` client is not installed in your system" % self.command) @@ -82,6 +86,9 @@ class VCSClientBase(object): def get_current_revision(self): raise NotImplementedError + def get_latest_revision(self): + return None if self.can_be_updated else self.get_current_revision() + def run_cmd(self, args, **kwargs): args = [self.command] + args if "cwd" not in kwargs: @@ -141,6 +148,16 @@ class GitClient(VCSClientBase): def get_current_revision(self): return self.get_cmd_output(["rev-parse", "--short", "HEAD"]) + def get_latest_revision(self): + if not self.can_be_updated: + return self.get_latest_revision() + result = self.get_cmd_output(["ls-remote"]) + for line in result.split("\n"): + line = line.strip() + if "HEAD" in line: + return line.split("HEAD", 1)[0].strip()[:7] + return None + class HgClient(VCSClientBase): @@ -160,6 +177,11 @@ class HgClient(VCSClientBase): def get_current_revision(self): return self.get_cmd_output(["identify", "--id"]) + def get_latest_revision(self): + if not self.can_be_updated: + return self.get_latest_revision() + return self.get_cmd_output(["identify", "--id", self.remote_url]) + class SvnClient(VCSClientBase):