Implement LocalCache system for API and improve a work in off-line mode

HISTORY.rst

@@ -4,14 +4,14 @@ Release Notes
 PlatformIO 3.0
 --------------
 
-3.0.2 (2016-09-??)
+3.1.0 (2016-09-??)
 ~~~~~~~~~~~~~~~~~~
 
-* Improved a work in off-line mode
+* Implemented LocalCache system for API and improved a work in off-line mode
 * Improved Project Generator when custom ``--project-option`` is passed to
   `platformio init <http://docs.platformio.org/en/stable/userguide/cmd_init.html>`__
   command
-* Disable SSL Server-Name-Indication for Python < 2.7.9
+* Fixed SSL Server-Name-Indication for Python < 2.7.9
 * Return valid exit code from ``plaformio test`` command
 
 * Development platform `Espressif 8266 <https://github.com/platformio/platform-espressif8266>`__

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 0, "2a3")
+VERSION = (3, 1, "0a1")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/app.py

@@ -14,10 +14,11 @@
 
 import hashlib
 import json
+import os
 import uuid
 from copy import deepcopy
-from os import environ, getenv
+from os import environ, getenv, listdir, remove
-from os.path import getmtime, isfile, join
+from os.path import dirname, getmtime, isdir, isfile, join
 from time import time
 
 from lockfile import LockFile
@@ -111,6 +112,85 @@ class State(object):
             self._lockfile.release()
 
 
+class LocalCache(object):
+
+    def __init__(self, cache_dir=None):
+        self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache")
+        if not self.cache_dir:
+            os.makedirs(self.cache_dir)
+        self.db_path = join(self.cache_dir, "db.data")
+
+    def __enter__(self):
+        if not isfile(self.db_path):
+            return self
+        newlines = []
+        found = False
+        with open(self.db_path) as fp:
+            for line in fp.readlines():
+                if "=" not in line:
+                    continue
+                line = line.strip()
+                expire, path = line.split("=")
+                if time() < int(expire):
+                    newlines.append(line)
+                    continue
+                found = True
+                if isfile(path):
+                    remove(path)
+                    if not len(listdir(dirname(path))):
+                        util.rmtree_(dirname(path))
+        if found:
+            with open(self.db_path, "w") as fp:
+                fp.write("\n".join(newlines) + "\n")
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        pass
+
+    def get_cache_path(self, key):
+        assert len(key) > 3
+        return join(self.cache_dir, key[-2:], key)
+
+    @staticmethod
+    def key_from_args(*args):
+        h = hashlib.md5()
+        for data in args:
+            h.update(str(data))
+        return h.hexdigest()
+
+    def get(self, key):
+        cache_path = self.get_cache_path(key)
+        if not isfile(cache_path):
+            return None
+        with open(cache_path) as fp:
+            data = fp.read()
+            if data[0] in ("{", "["):
+                return json.loads(data)
+            return data
+
+    def set(self, key, data, valid):
+        if not data:
+            return
+        tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
+        assert valid.endswith(tuple(tdmap.keys()))
+        cache_path = self.get_cache_path(key)
+        if not isdir(dirname(cache_path)):
+            os.makedirs(dirname(cache_path))
+        with open(cache_path, "w") as fp:
+            if isinstance(data, dict) or isinstance(data, list):
+                json.dump(data, fp)
+            else:
+                fp.write(str(data))
+        expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
+        with open(self.db_path, "w+") as fp:
+            fp.write("%s=%s\n" % (str(expire_time), cache_path))
+        return True
+
+    def clean(self):
+        if isdir(self.cache_dir):
+            util.rmtree_(self.cache_dir)
+
+
 def sanitize_setting(name, value):
     if name not in DEFAULT_SETTINGS:
         raise InvalidSettingName(name)

platformio/commands/ci.py

@@ -21,7 +21,8 @@ from tempfile import mkdtemp
 import click
 
 from platformio import app, util
-from platformio.commands.init import cli as cmd_init, validate_boards
+from platformio.commands.init import cli as cmd_init
+from platformio.commands.init import validate_boards
 from platformio.commands.run import cli as cmd_run
 from platformio.exception import CIBuildEnvsEmpty
 

platformio/commands/lib.py

@@ -180,8 +180,10 @@ def lib_search(query, json_output, page, noninteractive, **filters):
             query.append('%s:"%s"' % (key, value))
 
     result = get_api_result(
-        "/lib/search", dict(
+        "/lib/search",
-            query=" ".join(query), page=page))
+        dict(
+            query=" ".join(query), page=page),
+        cache_valid="3d")
 
     if json_output:
         click.echo(json.dumps(result))
@@ -232,7 +234,8 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         result = get_api_result(
             "/lib/search",
             dict(
-                query=" ".join(query), page=int(result['page']) + 1))
+                query=" ".join(query), page=int(result['page']) + 1),
+            cache_valid="3d")
 
 
 @cli.command("list", short_help="List installed libraries")

platformio/commands/platform.py

@@ -48,7 +48,7 @@ def _print_platforms(platforms):
 @click.option("--json-output", is_flag=True)
 def platform_search(query, json_output):
     platforms = []
-    for platform in util.get_api_result("/platforms"):
+    for platform in util.get_api_result("/platforms", cache_valid="365d"):
         if query == "all":
             query = ""
 

platformio/maintenance.py

@@ -42,6 +42,12 @@ def in_silence(ctx=None):
             (ctx.args[0] == "upgrade" or "--json-output" in ctx_args))
 
 
+def clean_cache(ctx):
+    if ctx.args and (ctx.args[0] == "upgrade" or "update" in ctx.args):
+        with app.LocalCache() as lc:
+            lc.clean()
+
+
 def on_platformio_start(ctx, force, caller):
     if not caller:
         if getenv("PLATFORMIO_CALLER"):
@@ -52,6 +58,7 @@ def on_platformio_start(ctx, force, caller):
     app.set_session_var("command_ctx", ctx)
     app.set_session_var("force_option", force)
     app.set_session_var("caller_id", caller)
+    clean_cache(ctx)
     telemetry.on_command()
 
     if not in_silence(ctx):

platformio/util.py

@@ -409,12 +409,24 @@ def _get_api_result(
     return result
 
 
-def get_api_result(path, params=None, data=None):
+def get_api_result(path, params=None, data=None, cache_valid=None):
-    max_retries = 5
+    from platformio.app import LocalCache
     total = 0
+    max_retries = 5
+    cache_key = (LocalCache.key_from_args(path, params, data)
+                 if cache_valid else None)
     while total < max_retries:
         try:
-            return _get_api_result(path, params, data)
+            with LocalCache() as lc:
+                if cache_key:
+                    result = lc.get(cache_key)
+                    if result is not None:
+                        return result
+            result = _get_api_result(path, params, data)
+            if cache_valid:
+                with LocalCache() as lc:
+                    lc.set(cache_key, result, cache_valid)
+            return result
         except (requests.exceptions.ConnectionError,
                 requests.exceptions.Timeout) as e:
             from platformio.maintenance import in_silence