diff --git a/HISTORY.rst b/HISTORY.rst index f8d2a437..0f5f57d7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,14 +4,14 @@ Release Notes PlatformIO 3.0 -------------- -3.0.2 (2016-09-??) +3.1.0 (2016-09-??) ~~~~~~~~~~~~~~~~~~ -* Improved a work in off-line mode +* Implemented LocalCache system for API and improved a work in off-line mode * Improved Project Generator when custom ``--project-option`` is passed to `platformio init `__ command -* Disable SSL Server-Name-Indication for Python < 2.7.9 +* Fixed SSL Server-Name-Indication for Python < 2.7.9 * Return valid exit code from ``plaformio test`` command * Development platform `Espressif 8266 `__ diff --git a/platformio/__init__.py b/platformio/__init__.py index aa995d20..d8d69720 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 0, "2a3") +VERSION = (3, 1, "0a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/app.py b/platformio/app.py index 013c0745..59e50e61 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,10 +14,11 @@ import hashlib import json +import os import uuid from copy import deepcopy -from os import environ, getenv -from os.path import getmtime, isfile, join +from os import environ, getenv, listdir, remove +from os.path import dirname, getmtime, isdir, isfile, join from time import time from lockfile import LockFile @@ -111,6 +112,85 @@ class State(object): self._lockfile.release() +class LocalCache(object): + + def __init__(self, cache_dir=None): + self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache") + if not self.cache_dir: + os.makedirs(self.cache_dir) + self.db_path = join(self.cache_dir, "db.data") + + def __enter__(self): + if not isfile(self.db_path): + return self + newlines = [] + found = False + with open(self.db_path) as fp: + for line in fp.readlines(): + if "=" not in line: + continue + line = line.strip() + expire, path = line.split("=") + if time() < int(expire): + newlines.append(line) + continue + found = True + if isfile(path): + remove(path) + if not len(listdir(dirname(path))): + util.rmtree_(dirname(path)) + if found: + with open(self.db_path, "w") as fp: + fp.write("\n".join(newlines) + "\n") + return self + + def __exit__(self, type_, value, traceback): + pass + + def get_cache_path(self, key): + assert len(key) > 3 + return join(self.cache_dir, key[-2:], key) + + @staticmethod + def key_from_args(*args): + h = hashlib.md5() + for data in args: + h.update(str(data)) + return h.hexdigest() + + def get(self, key): + cache_path = self.get_cache_path(key) + if not isfile(cache_path): + return None + with open(cache_path) as fp: + data = fp.read() + if data[0] in ("{", "["): + return json.loads(data) + return data + + def set(self, key, data, valid): + if not data: + return + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert valid.endswith(tuple(tdmap.keys())) + cache_path = self.get_cache_path(key) + if not isdir(dirname(cache_path)): + os.makedirs(dirname(cache_path)) + with open(cache_path, "w") as fp: + if isinstance(data, dict) or isinstance(data, list): + json.dump(data, fp) + else: + fp.write(str(data)) + expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) + with open(self.db_path, "w+") as fp: + fp.write("%s=%s\n" % (str(expire_time), cache_path)) + return True + + def clean(self): + if isdir(self.cache_dir): + util.rmtree_(self.cache_dir) + + def sanitize_setting(name, value): if name not in DEFAULT_SETTINGS: raise InvalidSettingName(name) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index de3349fa..cdfc004c 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -21,7 +21,8 @@ from tempfile import mkdtemp import click from platformio import app, util -from platformio.commands.init import cli as cmd_init, validate_boards +from platformio.commands.init import cli as cmd_init +from platformio.commands.init import validate_boards from platformio.commands.run import cli as cmd_run from platformio.exception import CIBuildEnvsEmpty diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 79a082c3..e087d061 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -180,8 +180,10 @@ def lib_search(query, json_output, page, noninteractive, **filters): query.append('%s:"%s"' % (key, value)) result = get_api_result( - "/lib/search", dict( - query=" ".join(query), page=page)) + "/lib/search", + dict( + query=" ".join(query), page=page), + cache_valid="3d") if json_output: click.echo(json.dumps(result)) @@ -232,7 +234,8 @@ def lib_search(query, json_output, page, noninteractive, **filters): result = get_api_result( "/lib/search", dict( - query=" ".join(query), page=int(result['page']) + 1)) + query=" ".join(query), page=int(result['page']) + 1), + cache_valid="3d") @cli.command("list", short_help="List installed libraries") diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 825422cb..53bb5515 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -48,7 +48,7 @@ def _print_platforms(platforms): @click.option("--json-output", is_flag=True) def platform_search(query, json_output): platforms = [] - for platform in util.get_api_result("/platforms"): + for platform in util.get_api_result("/platforms", cache_valid="365d"): if query == "all": query = "" diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 7b4c66a0..4ca07c4e 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -42,6 +42,12 @@ def in_silence(ctx=None): (ctx.args[0] == "upgrade" or "--json-output" in ctx_args)) +def clean_cache(ctx): + if ctx.args and (ctx.args[0] == "upgrade" or "update" in ctx.args): + with app.LocalCache() as lc: + lc.clean() + + def on_platformio_start(ctx, force, caller): if not caller: if getenv("PLATFORMIO_CALLER"): @@ -52,6 +58,7 @@ def on_platformio_start(ctx, force, caller): app.set_session_var("command_ctx", ctx) app.set_session_var("force_option", force) app.set_session_var("caller_id", caller) + clean_cache(ctx) telemetry.on_command() if not in_silence(ctx): diff --git a/platformio/util.py b/platformio/util.py index 98d18182..19fb8e02 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -409,12 +409,24 @@ def _get_api_result( return result -def get_api_result(path, params=None, data=None): - max_retries = 5 +def get_api_result(path, params=None, data=None, cache_valid=None): + from platformio.app import LocalCache total = 0 + max_retries = 5 + cache_key = (LocalCache.key_from_args(path, params, data) + if cache_valid else None) while total < max_retries: try: - return _get_api_result(path, params, data) + with LocalCache() as lc: + if cache_key: + result = lc.get(cache_key) + if result is not None: + return result + result = _get_api_result(path, params, data) + if cache_valid: + with LocalCache() as lc: + lc.set(cache_key, result, cache_valid) + return result except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: from platformio.maintenance import in_silence