“pio lib update” and “pio platform update” in JSON format

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 3, "0a7")
+VERSION = (3, 3, "0a8")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/commands/lib.py

@@ -112,12 +112,35 @@ def lib_uninstall(lm, libraries):
     "--only-check",
     is_flag=True,
     help="Do not update, only check for new version")
+@click.option("--json-output", is_flag=True)
 @click.pass_obj
-def lib_update(lm, libraries, only_check):
+def lib_update(lm, libraries, only_check, json_output):
     if not libraries:
-        libraries = [str(m.get("id", m['name'])) for m in lm.get_installed()]
+        libraries = []
-    for library in libraries:
+        for manifest in lm.get_installed():
-        lm.update(library, only_check=only_check)
+            pkg_dir = manifest['__pkg_dir']
+            if "@" in pkg_dir and "@vcs-" not in pkg_dir:
+                continue
+            elif "@vcs-" in pkg_dir:
+                libraries.append("%s=%s" % (manifest['name'], manifest['url']))
+            else:
+                libraries.append(str(manifest.get("id", manifest['name'])))
+
+    if only_check and json_output:
+        result = []
+        for library in libraries:
+            name, requirements, url = lm.parse_pkg_name(library)
+            latest = lm.outdated(name, requirements, url)
+            if latest is False:
+                continue
+            manifest = lm.load_manifest(
+                lm.get_package_dir(name, requirements, url))
+            manifest['versionLatest'] = latest or "Unknown"
+            result.append(manifest)
+        return click.echo(json.dumps(result))
+    else:
+        for library in libraries:
+            lm.update(library, only_check=only_check)
 
 
 def print_lib_item(item):

platformio/commands/platform.py

@@ -114,15 +114,42 @@ def platform_uninstall(platforms):
     "--only-check",
     is_flag=True,
     help="Do not update, only check for new version")
-def platform_update(platforms, only_packages, only_check):
+@click.option("--json-output", is_flag=True)
+def platform_update(platforms, only_packages, only_check, json_output):
     pm = PlatformManager()
     if not platforms:
-        platforms = set([m['name'] for m in pm.get_installed()])
+        platforms = []
-    for platform in platforms:
+        for manifest in pm.get_installed():
-        click.echo("Platform %s" % click.style(platform, fg="cyan"))
+            pkg_dir = manifest['__pkg_dir']
-        click.echo("--------")
+            if "@" in pkg_dir and "@vcs-" not in pkg_dir:
-        pm.update(platform, only_packages=only_packages, only_check=only_check)
+                continue
-        click.echo()
+            elif "@vcs-" in pkg_dir:
+                platforms.append("%s=%s" % (manifest['name'], manifest['url']))
+            else:
+                platforms.append(manifest['name'])
+
+    if only_check and json_output:
+        result = []
+        for platform in platforms:
+            name, requirements, url = pm.parse_pkg_name(platform)
+            latest = pm.outdated(name, requirements, url)
+            if latest is False:
+                continue
+            manifest = pm.load_manifest(
+                pm.get_package_dir(name, requirements, url))
+            if latest is True:
+                manifest['versionLatest'] = "Out-of-date"
+            else:
+                manifest['versionLatest'] = latest or "Unknown"
+            result.append(manifest)
+        return click.echo(json.dumps(result))
+    else:
+        for platform in platforms:
+            click.echo("Platform %s" % click.style(platform, fg="cyan"))
+            click.echo("--------")
+            pm.update(
+                platform, only_packages=only_packages, only_check=only_check)
+            click.echo()
 
 
 @cli.command("list", short_help="List installed development platforms")

platformio/maintenance.py

@@ -268,7 +268,7 @@ def check_internal_updates(ctx, what):
     outdated_items = []
     for manifest in pm.get_installed():
         if manifest['name'] not in outdated_items and \
-                pm.is_outdated(manifest['name']):
+                pm.outdated(manifest['name']):
             outdated_items.append(manifest['name'])
 
     if not outdated_items:

platformio/managers/package.py

@@ -119,6 +119,47 @@ class PkgInstallerMixin(object):
 
     VCS_MANIFEST_NAME = ".piopkgmanager.json"
 
+    FILE_CACHE_VALID = "1m"  # 1 month
+    FILE_CACHE_MAX_SIZE = 1024 * 1024
+
+    _INSTALLED_CACHE = {}
+
+    def reset_cache(self):
+        if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
+            del PkgInstallerMixin._INSTALLED_CACHE[self.package_dir]
+
+    def download(self, url, dest_dir, sha1=None):
+        cache_key_fname = app.ContentCache.key_from_args(url, "fname")
+        cache_key_data = app.ContentCache.key_from_args(url, "data")
+        if self.FILE_CACHE_VALID:
+            with app.ContentCache() as cc:
+                fname = cc.get(cache_key_fname)
+                cache_path = cc.get_cache_path(cache_key_data)
+                if fname and isfile(cache_path):
+                    dst_path = join(dest_dir, fname)
+                    shutil.copy(cache_path, dst_path)
+                    return dst_path
+
+        fd = FileDownloader(url, dest_dir)
+        fd.start()
+        if sha1:
+            fd.verify(sha1)
+        dst_path = fd.get_filepath()
+        if not self.FILE_CACHE_VALID or getsize(
+                dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE:
+            return dst_path
+
+        with app.ContentCache() as cc:
+            cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
+            cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
+            shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
+        return dst_path
+
+    @staticmethod
+    def unpack(source_path, dest_dir):
+        fu = FileUnpacker(source_path, dest_dir)
+        return fu.start()
+
     def get_vcs_manifest_path(self, pkg_dir):
         for item in os.listdir(pkg_dir):
             if not isdir(join(pkg_dir, item)):
@@ -142,7 +183,7 @@ class PkgInstallerMixin(object):
     def manifest_exists(self, pkg_dir):
         return self.get_manifest_path(pkg_dir) is not None
 
-    def load_manifest(self, path):
+    def load_manifest(self, path):  # pylint: disable=too-many-branches
         assert path
         pkg_dir = path
         if isdir(path):
@@ -155,6 +196,13 @@ class PkgInstallerMixin(object):
         if isfile(path) and path.endswith(self.VCS_MANIFEST_NAME):
             pkg_dir = dirname(dirname(path))
 
+        # return from cache
+        if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
+            for manifest in PkgInstallerMixin._INSTALLED_CACHE[self.
+                                                               package_dir]:
+                if manifest['__pkg_dir'] == pkg_dir:
+                    return manifest
+
         manifest = {}
         if path.endswith(".json"):
             manifest = util.load_json(path)
@@ -174,6 +222,22 @@ class PkgInstallerMixin(object):
         manifest['__pkg_dir'] = pkg_dir
         return manifest
 
+    def get_installed(self):
+        if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE:
+            return PkgInstallerMixin._INSTALLED_CACHE[self.package_dir]
+        items = []
+        for p in sorted(os.listdir(self.package_dir)):
+            pkg_dir = join(self.package_dir, p)
+            if not isdir(pkg_dir):
+                continue
+            manifest = self.load_manifest(pkg_dir)
+            if not manifest:
+                continue
+            assert "name" in manifest
+            items.append(manifest)
+        PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] = items
+        return items
+
     def check_pkg_structure(self, pkg_dir):
         if self.manifest_exists(pkg_dir):
             return pkg_dir
@@ -305,11 +369,6 @@ class PkgInstallerMixin(object):
 
 class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
-    _INSTALLED_CACHE = {}
-
-    FILE_CACHE_VALID = "1m"  # 1 month
-    FILE_CACHE_MAX_SIZE = 1024 * 1024
-
     def __init__(self, package_dir, repositories=None):
         self.repositories = repositories
         self.package_dir = package_dir
@@ -321,42 +380,6 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
     def manifest_names(self):
         raise NotImplementedError()
 
-    def download(self, url, dest_dir, sha1=None):
-        cache_key_fname = app.ContentCache.key_from_args(url, "fname")
-        cache_key_data = app.ContentCache.key_from_args(url, "data")
-        if self.FILE_CACHE_VALID:
-            with app.ContentCache() as cc:
-                fname = cc.get(cache_key_fname)
-                cache_path = cc.get_cache_path(cache_key_data)
-                if fname and isfile(cache_path):
-                    dst_path = join(dest_dir, fname)
-                    shutil.copy(cache_path, dst_path)
-                    return dst_path
-
-        fd = FileDownloader(url, dest_dir)
-        fd.start()
-        if sha1:
-            fd.verify(sha1)
-        dst_path = fd.get_filepath()
-        if not self.FILE_CACHE_VALID or getsize(
-                dst_path) > BasePkgManager.FILE_CACHE_MAX_SIZE:
-            return dst_path
-
-        with app.ContentCache() as cc:
-            cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
-            cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
-            shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
-        return dst_path
-
-    @staticmethod
-    def unpack(source_path, dest_dir):
-        fu = FileUnpacker(source_path, dest_dir)
-        return fu.start()
-
-    def reset_cache(self):
-        if self.package_dir in BasePkgManager._INSTALLED_CACHE:
-            del BasePkgManager._INSTALLED_CACHE[self.package_dir]
-
     def print_message(self, message, nl=True):
         click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl)
 
@@ -415,22 +438,6 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             url = None
         return (name or text, requirements, url)
 
-    def get_installed(self):
-        if self.package_dir in BasePkgManager._INSTALLED_CACHE:
-            return BasePkgManager._INSTALLED_CACHE[self.package_dir]
-        items = []
-        for p in sorted(os.listdir(self.package_dir)):
-            pkg_dir = join(self.package_dir, p)
-            if not isdir(pkg_dir):
-                continue
-            manifest = self.load_manifest(pkg_dir)
-            if not manifest:
-                continue
-            assert "name" in manifest
-            items.append(manifest)
-        BasePkgManager._INSTALLED_CACHE[self.package_dir] = items
-        return items
-
     def get_package(self, name, requirements=None, url=None):
         pkg_id = int(name[3:]) if name.startswith("id=") else 0
         best = None
@@ -473,20 +480,38 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         package = self.get_package(name, requirements, url)
         return package.get("__pkg_dir") if package else None
 
-    def is_outdated(self, name, requirements=None, silent=False):
+    def outdated(self, name, requirements=None, url=None):
-        package_dir = self.get_package_dir(name, requirements)
+        """
+        Has 3 different results:
+        `None` - unknown package, VCS is fixed to commit
+        `False` - package is up-to-date
+        `String` - a found latest version
+        """
+        latest = None
+        package_dir = self.get_package_dir(name, requirements, url)
         if not package_dir:
-            if silent:
+            return None
-                return
-            click.secho(
-                "%s @ %s is not installed" % (name, requirements or "*"),
-                fg="yellow")
-            return
-        if self.get_vcs_manifest_path(package_dir):
-            return False
         manifest = self.load_manifest(package_dir)
-        latest = self.get_latest_repo_version(name, requirements)
+        if self.get_vcs_manifest_path(package_dir):
-        return latest and manifest['version'] != latest
+            vcs = VCSClientFactory.newClient(
+                package_dir, manifest['url'], silent=True)
+            if not vcs.can_be_updated:
+                return None
+            latest = vcs.get_latest_revision()
+        else:
+            try:
+                latest = self.get_latest_repo_version(name, requirements)
+            except (exception.PlatformioException, ValueError):
+                return None
+        if not latest:
+            return None
+        up_to_date = False
+        try:
+            up_to_date = (semantic_version.Version.coerce(manifest['version'])
+                          >= semantic_version.Version.coerce(latest))
+        except ValueError:
+            up_to_date = latest == manifest['version']
+        return False if up_to_date else latest
 
     def install(self,
                 name,
@@ -571,7 +596,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             requirements=None,
             only_check=False):
         name, requirements, url = self.parse_pkg_name(name, requirements)
-        package_dir = self.get_package_dir(name, None, url)
+        package_dir = self.get_package_dir(name, requirements, url)
         if not package_dir:
             click.secho(
                 "%s @ %s is not installed" % (name, requirements or "*"),
@@ -587,16 +612,29 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
         manifest = self.load_manifest(manifest_path)
         click.echo(
-            "%s %s @ %s: \t" % ("Checking"
+            "{} {:<40} @ {:<15}".format(
-                                if only_check else "Updating", click.style(
+                "Checking" if only_check else "Updating",
-                                    manifest['name'], fg="cyan"),
+                click.style(
-                                manifest['version']),
+                    manifest['name'], fg="cyan"),
+                manifest['version']),
             nl=False)
+        if not util.internet_on():
+            click.echo("[%s]" % (click.style("Off-line", fg="yellow")))
+            return
+        latest = self.outdated(name, requirements, url)
+        if latest is True:
+            click.echo("[%s]" % (click.style("Out-of-date", fg="red")))
+        elif latest:
+            click.echo("[%s]" % (click.style(latest, fg="red")))
+        elif latest is False:
+            click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
+        else:
+            click.echo("[%s]" % (click.style("Unknown", fg="yellow")))
+
+        if only_check or latest is False or (not is_vcs_pkg and not latest):
+            return
+
         if is_vcs_pkg:
-            if only_check:
-                click.echo("[%s]" % (click.style("Skip", fg="yellow")))
-                return
-            click.echo("[%s]" % (click.style("VCS", fg="yellow")))
             vcs = VCSClientFactory.newClient(package_dir, manifest['url'])
             if not vcs.can_be_updated:
                 click.secho(
@@ -608,36 +646,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             with open(manifest_path, "w") as fp:
                 manifest['version'] = vcs.get_current_revision()
                 json.dump(manifest, fp)
+            self.reset_cache()
         else:
-            latest_version = None
-            try:
-                latest_version = self.get_latest_repo_version(name,
-                                                              requirements)
-            except exception.PlatformioException:
-                pass
-            if not latest_version:
-                click.echo("[%s]" % (click.style(
-                    "Off-line" if not util.internet_on() else "Unknown",
-                    fg="yellow")))
-                return
-
-            up_to_date = False
-            try:
-                up_to_date = (
-                    semantic_version.Version.coerce(manifest['version']) >=
-                    semantic_version.Version.coerce(latest_version))
-            except ValueError:
-                up_to_date = latest_version == manifest['version']
-
-            if up_to_date:
-                click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
-                return
-
-            click.echo("[%s]" % (click.style("Out-of-date", fg="red")))
-            if only_check:
-                return
             self.uninstall(name, manifest['version'], trigger_event=False)
-            self.install(name, latest_version, trigger_event=False)
+            self.install(name, latest, trigger_event=False)
 
         telemetry.on_event(
             category=self.__class__.__name__,

platformio/managers/platform.py

@@ -92,9 +92,10 @@ class PlatformManager(BasePkgManager):
         self.cleanup_packages(p.packages.keys())
         return True
 
-    def is_outdated(self, name, requirements=None, silent=False):
+    def outdated(self, name, requirements=None, url=None):
-        if BasePkgManager.is_outdated(self, name, requirements, silent):
+        latest = BasePkgManager.outdated(self, name, requirements, url)
-            return True
+        if latest:
+            return latest
         p = PlatformFactory.newPlatform(name, requirements)
         return p.are_outdated_packages()
 
@@ -213,7 +214,7 @@ class PlatformPackagesMixin(object):
                 continue
             elif (name in with_packages or
                   not (skip_default_package or opts.get("optional", False))):
-                if self.validate_version_requirements(version):
+                if self.is_valid_requirements(version):
                     self.pm.install(name, version, silent=silent)
                 else:
                     requirements = None
@@ -228,7 +229,7 @@ class PlatformPackagesMixin(object):
         items = {}
         for name, opts in self.packages.items():
             version = opts.get("version", "")
-            if self.validate_version_requirements(version):
+            if self.is_valid_requirements(version):
                 package = self.pm.get_package(name, version)
             else:
                 package = self.pm.get_package(*self._parse_pkg_name(name,
@@ -240,7 +241,7 @@ class PlatformPackagesMixin(object):
     def update_packages(self, only_check=False):
         for name in self.get_installed_packages():
             version = self.packages[name].get("version", "")
-            if self.validate_version_requirements(version):
+            if self.is_valid_requirements(version):
                 self.pm.update(name, version, only_check)
             else:
                 requirements = None
@@ -250,17 +251,23 @@ class PlatformPackagesMixin(object):
                                only_check)
 
     def are_outdated_packages(self):
-        for name, opts in self.packages.items():
+        latest = None
-            version = opts.get("version", "")
+        for name in self.get_installed_packages():
-            if not self.validate_version_requirements(version):
+            version = self.packages[name].get("version", "")
-                continue
+            if self.is_valid_requirements(version):
-            if self.pm.is_outdated(name, version, silent=True):
+                latest = self.pm.outdated(name, version)
+            else:
+                requirements = None
+                if "@" in version:
+                    version, requirements = version.rsplit("@", 1)
+                latest = self.pm.outdated(name, requirements, version)
+            if latest or latest is None:
                 return True
         return False
 
     def get_package_dir(self, name):
         version = self.packages[name].get("version", "")
-        if self.validate_version_requirements(version):
+        if self.is_valid_requirements(version):
             return self.pm.get_package_dir(name, version)
         else:
             return self.pm.get_package_dir(*self._parse_pkg_name(name,
@@ -268,14 +275,14 @@ class PlatformPackagesMixin(object):
 
     def get_package_version(self, name):
         version = self.packages[name].get("version", "")
-        if self.validate_version_requirements(version):
+        if self.is_valid_requirements(version):
             package = self.pm.get_package(name, version)
         else:
             package = self.pm.get_package(*self._parse_pkg_name(name, version))
         return package['version'] if package else None
 
     @staticmethod
-    def validate_version_requirements(requirements):
+    def is_valid_requirements(requirements):
         return requirements and "://" not in requirements
 
     def _parse_pkg_name(self, name, version):

platformio/vcsclient.py

@@ -25,7 +25,7 @@ from platformio.exception import PlatformioException
 class VCSClientFactory(object):
 
     @staticmethod
-    def newClient(src_dir, remote_url):
+    def newClient(src_dir, remote_url, silent=False):
         result = urlparse(remote_url)
         type_ = result.scheme
         tag = None
@@ -40,7 +40,7 @@ class VCSClientFactory(object):
             raise PlatformioException("VCS: Unknown repository type %s" %
                                       remote_url)
         obj = getattr(modules[__name__], "%sClient" % type_.title())(
-            src_dir, remote_url, tag)
+            src_dir, remote_url, tag, silent)
         assert isinstance(obj, VCSClientBase)
         return obj
 
@@ -49,17 +49,21 @@ class VCSClientBase(object):
 
     command = None
 
-    def __init__(self, src_dir, remote_url=None, tag=None):
+    def __init__(self, src_dir, remote_url=None, tag=None, silent=False):
         self.src_dir = src_dir
         self.remote_url = remote_url
         self.tag = tag
+        self.silent = silent
         self.check_client()
 
     def check_client(self):
         try:
             assert self.command
-            assert self.run_cmd(["--version"])
+            if self.silent:
-        except (AssertionError, OSError):
+                self.get_cmd_output(["--version"])
+            else:
+                assert self.run_cmd(["--version"])
+        except (AssertionError, OSError, PlatformioException):
             raise PlatformioException(
                 "VCS: `%s` client is not installed in your system" %
                 self.command)
@@ -82,6 +86,9 @@ class VCSClientBase(object):
     def get_current_revision(self):
         raise NotImplementedError
 
+    def get_latest_revision(self):
+        return None if self.can_be_updated else self.get_current_revision()
+
     def run_cmd(self, args, **kwargs):
         args = [self.command] + args
         if "cwd" not in kwargs:
@@ -141,6 +148,16 @@ class GitClient(VCSClientBase):
     def get_current_revision(self):
         return self.get_cmd_output(["rev-parse", "--short", "HEAD"])
 
+    def get_latest_revision(self):
+        if not self.can_be_updated:
+            return self.get_latest_revision()
+        result = self.get_cmd_output(["ls-remote"])
+        for line in result.split("\n"):
+            line = line.strip()
+            if "HEAD" in line:
+                return line.split("HEAD", 1)[0].strip()[:7]
+        return None
+
 
 class HgClient(VCSClientBase):
 
@@ -160,6 +177,11 @@ class HgClient(VCSClientBase):
     def get_current_revision(self):
         return self.get_cmd_output(["identify", "--id"])
 
+    def get_latest_revision(self):
+        if not self.can_be_updated:
+            return self.get_latest_revision()
+        return self.get_cmd_output(["identify", "--id", self.remote_url])
+
 
 class SvnClient(VCSClientBase):