Added Migration Manager which simplifies process with upgrading to a major release

.pylintrc

@@ -38,7 +38,7 @@ load-plugins=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=C0103,C0111,E0611,F0401,I0011,R0801,R0903
+disable=C0103,C0111,E0611,F0401,I0011,R0801,R0903,R0922
 
 
 [REPORTS]

HISTORY.rst

@@ -4,6 +4,8 @@ Release History
 0.9.0 (?)
 ---------
 
+* Added *Migration Manager* which simplifies process with upgrading to a
+  major release
 * Added *Telemetry Service* which should help us make *PlatformIO* better
 * Implemented *PlatformIO AppState Manager* which allow to have multiple
   ``.platformio`` states.
@@ -59,11 +61,11 @@ Release History
 * Added auto-conversation from \*.ino to valid \*.cpp for Arduino/Energia
   frameworks (`issue #7 <https://github.com/ivankravets/platformio/issues/7>`_)
 * Added `Arduino example <https://github.com/ivankravets/platformio/tree/develop/examples/arduino-adafruit-library>`_
-  with external library (Adafruit CC3000)
+  with external library (*Adafruit CC3000*)
 * Implemented `platformio upgrade <http://docs.platformio.ikravets.com/en/latest/userguide/cmd_upgrade.html>`_
   command and "auto-check" for the latest
   version (`issue #8 <https://github.com/ivankravets/platformio/issues/8>`_)
-* Fixed an issue with "auto-reset" for Raspduino board
+* Fixed an issue with "auto-reset" for *Raspduino* board
 * Fixed a bug with nested libs building
 
 0.4.0 (2014-07-31)

platformio/__main__.py

@@ -1,21 +1,19 @@
 # Copyright (C) Ivan Kravets <me@ikravets.com>
 # See LICENSE for details.
 
-from os import listdir, makedirs
+from os import listdir
-from os.path import getmtime, isdir, isfile, join
+from os.path import join
 from sys import exit as sys_exit
-from time import time
 from traceback import format_exc
 
-from click import command, MultiCommand, secho, version_option
+import click
 
-from platformio import __version__
+from platformio import __version__, maintenance
-from platformio.commands.upgrade import get_latest_version
 from platformio.exception import PlatformioException, UnknownCLICommand
-from platformio.util import get_home_dir, get_source_dir
+from platformio.util import get_source_dir
 
 
-class PlatformioCLI(MultiCommand):  # pylint: disable=R0904
+class PlatformioCLI(click.MultiCommand):  # pylint: disable=R0904
 
     def list_commands(self, ctx):
         cmds = []
@@ -28,6 +26,7 @@ class PlatformioCLI(MultiCommand):  # pylint: disable=R0904
         return cmds
 
     def get_command(self, ctx, name):
+        mod = None
         try:
             mod = __import__("platformio.commands." + name,
                              None, None, ["cli"])
@@ -36,35 +35,27 @@ class PlatformioCLI(MultiCommand):  # pylint: disable=R0904
         return mod.cli
 
 
-@command(cls=PlatformioCLI)
+@click.command(cls=PlatformioCLI)
-@version_option(__version__, prog_name="PlatformIO")
+@click.version_option(__version__, prog_name="PlatformIO")
-def cli():
+@click.pass_context
-    pass
+def cli(ctx):
+    maintenance.on_platformio_start(ctx)
 
 
-def autocheck_latest_version():
+@cli.resultcallback()
-    check_interval = 3600 * 24 * 7  # 1 week
+@click.pass_context
-    checkfile = join(get_home_dir(), ".pioupgrade")
+def process_result(ctx, result):
-    if isfile(checkfile) and getmtime(checkfile) > (time() - check_interval):
+    maintenance.on_platformio_end(ctx, result)
-        return False
-    if not isdir(get_home_dir()):
-        makedirs(get_home_dir())
-    with open(checkfile, "w") as f:
-        f.write(str(time()))
-    return get_latest_version() != __version__
 
 
 def main():
     try:
-        if autocheck_latest_version():
+        cli(None)
-            secho("\nThere is a new version of PlatformIO available.\n"
-                  "Please upgrade it via `platformio upgrade` command.\n",
-                  fg="yellow")
-
-        cli()
     except Exception as e:  # pylint: disable=W0703
+        maintenance.on_platformio_exception(e)
         if isinstance(e, PlatformioException):
-            sys_exit("Error: " + str(e))
+            click.echo("Error: " + str(e))
+            sys_exit(1)
         else:
             print format_exc()
 

platformio/commands/lib.py

@@ -191,13 +191,7 @@ def lib_show(libid):
 @click.pass_context
 def lib_update(ctx):
     lm = LibraryManager(get_lib_dir())
-
+    for id_, latest_version in (lm.get_latest_versions() or {}).items():
-    lib_ids = [str(item['id']) for item in lm.get_installed().values()]
-    if not lib_ids:
-        return
-
-    versions = get_api_result("/lib/version/" + str(",".join(lib_ids)))
-    for id_ in lib_ids:
         info = lm.get_info(int(id_))
 
         click.echo("Updating  [ %s ] %s library:" % (
@@ -205,8 +199,6 @@ def lib_update(ctx):
             click.style(info['name'], fg="cyan")))
 
         current_version = info['version']
-        latest_version = versions[id_]
-
         if latest_version is None:
             click.secho("Unknown library", fg="red")
             continue

platformio/commands/list.py

@@ -11,7 +11,7 @@ def cli():
 
     installed_platforms = PlatformFactory.get_platforms(
         installed=True).keys()
-    installed_platforms = sorted(installed_platforms)
+    installed_platforms.sort()
 
     for platform in installed_platforms:
         p = PlatformFactory().newPlatform(platform)

platformio/commands/update.py

@@ -11,7 +11,7 @@ def cli():
 
     installed_platforms = PlatformFactory.get_platforms(
         installed=True).keys()
-    installed_platforms = sorted(installed_platforms)
+    installed_platforms.sort()
 
     for platform in installed_platforms:
         echo("\nPlatform %s" % style(platform, fg="cyan"))

platformio/exception.py

@@ -139,3 +139,8 @@ class InvalidSettingName(PlatformioException):
 class InvalidSettingValue(PlatformioException):
 
     MESSAGE = "Invalid value '%s' for the setting '%s'"
+
+
+class UpgraderFailed(PlatformioException):
+
+    MESSAGE = "An error occurred while upgrading PlatformIO"

platformio/libmanager.py

@@ -45,6 +45,20 @@ class LibraryManager(object):
                 items[dirname] = json.load(f)
         return items
 
+    def get_latest_versions(self):
+        lib_ids = [str(item['id']) for item in self.get_installed().values()]
+        if not lib_ids:
+            return None
+        return get_api_result("/lib/version/" + str(",".join(lib_ids)))
+
+    def get_outdated(self):
+        outdated = []
+        for id_, latest_version in (self.get_latest_versions() or {}).items():
+            info = self.get_info(int(id_))
+            if latest_version != info['version']:
+                outdated.append(info['name'])
+        return outdated
+
     def get_info(self, id_):
         for item in self.get_installed().values():
             if "id" in item and item['id'] == id_:

platformio/maintenance.py

@@ -0,0 +1,185 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+import re
+from os import remove
+from os.path import isdir, isfile, join
+from shutil import rmtree
+from time import time
+
+import click
+
+from platformio import __version__, app, telemetry
+from platformio.commands.install import cli as cli_install
+from platformio.commands.lib import lib_update as cli_libraries_update
+from platformio.commands.update import cli as cli_platforms_update
+from platformio.commands.upgrade import get_latest_version
+from platformio.exception import UpgraderFailed
+from platformio.libmanager import LibraryManager
+from platformio.platforms.base import PlatformFactory
+from platformio.util import get_home_dir, get_lib_dir
+
+
+def on_platformio_start(ctx):
+    telemetry.on_command(ctx)
+    after_upgrade(ctx)
+    check_platformio_upgrade()
+    check_internal_updates(ctx, "platforms")
+    check_internal_updates(ctx, "libraries")
+
+
+def on_platformio_end(ctx, result):  # pylint: disable=W0613
+    pass
+
+
+def on_platformio_exception(e):
+    telemetry.on_exception(e)
+
+
+class Upgrader(object):
+
+    def __init__(self, from_version, to_version):
+        self.from_version = self.version_to_int(from_version)
+        self.to_version = self.version_to_int(to_version)
+
+    @staticmethod
+    def version_to_int(version):
+        return int(re.sub(r"[^\d]+", "", version))
+
+    def run(self, ctx):
+        if self.from_version > self.to_version:
+            return True
+
+        result = [True]
+        for v in (90, ):
+            if self.from_version >= v:
+                continue
+            result.append(getattr(self, "_upgrade_to_%d" % v)(ctx))
+
+        return all(result)
+
+    def _upgrade_to_90(self, ctx):  # pylint: disable=R0201
+        prev_platforms = []
+
+        # remove platform's folder (obsoleted package structure)
+        for name in PlatformFactory.get_platforms().keys():
+            pdir = join(get_home_dir(), name)
+            if not isdir(pdir):
+                continue
+            prev_platforms.append(name)
+            rmtree(pdir)
+
+        # remove unused files
+        for fname in (".pioupgrade", "installed.json"):
+            if isfile(join(get_home_dir(), fname)):
+                remove(join(get_home_dir(), fname))
+
+        if prev_platforms:
+            ctx.invoke(cli_install, platforms=prev_platforms)
+
+        return True
+
+
+def after_upgrade(ctx):
+    if app.get_state_item("last_version", None) == __version__:
+        return
+
+    # promotion
+    click.echo("\nIf you like %s, please:" % (
+        click.style("PlatformIO", fg="cyan")
+    ))
+    click.echo(
+        "- %s us on Twitter to stay up-to-date "
+        "on the latest project news > %s" %
+        (click.style("follow", fg="cyan"),
+         click.style("https://twitter.com/platformiotool", fg="blue"))
+    )
+    click.echo("- %s us a star on GitHub > %s" % (
+        click.style("give", fg="cyan"),
+        click.style("https://github.com/ivankravets/platformio", fg="blue")
+    ))
+    click.secho("Thanks a lot!\n", fg="green", blink=True)
+
+    if not isdir(get_home_dir()):
+        return
+
+    click.secho("Please wait while upgrading PlatformIO ...",
+                fg="yellow")
+
+    last_version = app.get_state_item("last_version", "0.0.0")
+    u = Upgrader(last_version, __version__)
+    if u.run(ctx):
+        app.set_state_item("last_version", __version__)
+        click.secho("PlatformIO has been successfully upgraded to %s!\n" %
+                    __version__, fg="green")
+
+        telemetry.on_event(category="Auto", action="Upgrade",
+                           label="%s > %s" % (last_version, __version__))
+    else:
+        raise UpgraderFailed()
+    click.echo("")
+
+
+def check_platformio_upgrade():
+    last_check = app.get_state_item("last_check", {})
+    interval = int(app.get_setting("check_platformio_interval")) * 3600 * 24
+    if (time() - interval) < last_check.get("platformio_upgrade", 0):
+        return
+
+    last_check['platformio_upgrade'] = int(time())
+    app.set_state_item("last_check", last_check)
+
+    latest_version = get_latest_version()
+    if latest_version == __version__:
+        return
+
+    click.secho("There is a new version %s of PlatformIO available.\n"
+                "Please upgrade it via " % latest_version,
+                fg="yellow", nl=False)
+    click.secho("`platformio upgrade`", fg="cyan", nl=False)
+    click.secho(" command.\nChanges: ", fg="yellow", nl=False)
+    click.secho("http://docs.platformio.ikravets.com/en/latest/history.html\n",
+                fg="blue")
+
+
+def check_internal_updates(ctx, what):
+    last_check = app.get_state_item("last_check", {})
+    interval = int(app.get_setting("check_%s_interval" % what)) * 3600 * 24
+    if (time() - interval) < last_check.get(what + "_update", 0):
+        return
+
+    last_check[what + '_update'] = int(time())
+    app.set_state_item("last_check", last_check)
+
+    outdated_items = []
+    if what == "platforms":
+        for platform in PlatformFactory.get_platforms(installed=True).keys():
+            p = PlatformFactory().newPlatform(platform)
+            if p.is_outdated():
+                outdated_items.append(platform)
+    elif what == "libraries":
+        lm = LibraryManager(get_lib_dir())
+        outdated_items = lm.get_outdated()
+
+    if not outdated_items:
+        return
+
+    click.secho("There are the new updates for %s (%s)" %
+                (what, ", ".join(outdated_items)), fg="yellow")
+
+    if not app.get_setting("auto_update_" + what):
+        click.secho("Please update them via ", fg="yellow", nl=False)
+        click.secho("`platformio %supdate`" %
+                    ("lib " if what == "libraries" else ""),
+                    fg="cyan", nl=False)
+        click.secho(" command.\n", fg="yellow")
+    else:
+        click.secho("Please wait while updating %s ..." % what, fg="yellow")
+        if what == "platforms":
+            ctx.invoke(cli_platforms_update)
+        elif what == "libraries":
+            ctx.invoke(cli_libraries_update)
+        click.echo()
+
+        telemetry.on_event(category="Auto", action="Update",
+                           label=what.title())

platformio/pkgmanager.py

@@ -19,21 +19,19 @@ from platformio.util import get_api_result, get_home_dir, get_systype
 
 class PackageManager(object):
 
-    DBFILE_PATH = join(get_home_dir(), "installed.json")
-
     def __init__(self):
         self._package_dir = join(get_home_dir(), "packages")
         if not isdir(self._package_dir):
             makedirs(self._package_dir)
         assert isdir(self._package_dir)
 
-    @staticmethod
+    @classmethod
-    def get_manifest():
+    def get_manifest(cls):
         try:
-            return PackageManager._cached_manifest
+            return cls._cached_manifest
         except AttributeError:
-            PackageManager._cached_manifest = get_api_result("/packages")
+            cls._cached_manifest = get_api_result("/packages")
-        return PackageManager._cached_manifest
+        return cls._cached_manifest
 
     @staticmethod
     def download(url, dest_dir, sha1=None):
@@ -144,11 +142,11 @@ class PackageManager(object):
         data = self.get_installed()
         data[name] = {
             "version": version,
-            "time": time()
+            "time": int(time())
         }
-        self.update_appstate_instpkgs(data)
+        set_state_item("installed_packages", data)
 
     def _unregister(self, name):
         data = self.get_installed()
         del data[name]
-        self.update_appstate_instpkgs(data)
+        set_state_item("installed_packages", data)