feedc0de/platformio-core
1
0
Fork 0
forked from platformio/platformio-core
Code Pull Requests Activity

Enable caching by default for API requests and Library Manager

Browse Source
This commit is contained in:
Ivan Kravets
2016-12-05 18:51:25 +02:00
parent 6de8325130
commit 08cda5e7ea
8 changed files with 87 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.rst
View File

@@ -44,6 +44,7 @@ PlatformIO 3.0
* Added global ``lib_extra_dirs`` option to ``[platformio]`` section for * Added global ``lib_extra_dirs`` option to ``[platformio]`` section for
`Project Configuration File "platformio.ini" <http://docs.platformio.org/en/stable/projectconf.html>`__ `Project Configuration File "platformio.ini" <http://docs.platformio.org/en/stable/projectconf.html>`__
(`issue #842 <https://github.com/platformio/platformio-core/issues/842>`_) (`issue #842 <https://github.com/platformio/platformio-core/issues/842>`_)
* Enabled caching by default for API requests and Library Manager (see `enable_cache <http://docs.platformio.org/en/latest/userguide/cmd_settings.html#enable-cache>`__ setting)
* Changed a default exit combination for Device Monitor from ``Ctrl+]`` to ``Ctrl+C`` * Changed a default exit combination for Device Monitor from ``Ctrl+]`` to ``Ctrl+C``
* Improved detecting of ARM mbed media disk for uploading * Improved detecting of ARM mbed media disk for uploading
* Improved Project Generator for CLion IDE when source folder contains nested items * Improved Project Generator for CLion IDE when source folder contains nested items

30
docs/userguide/cmd_settings.rst
View File

@@ -97,6 +97,16 @@ Check for the platform updates interval.
Enable SSL for PlatformIO Services Enable SSL for PlatformIO Services
.. _enable_cache:
``enable_cache``
^^^^^^^^^^^^^^^^
:Default: Yes
:Values: Yes/No
Enable caching for API requests and Library Manager
.. _setting_force_verbose: .. _setting_force_verbose:
``force_verbose`` ``force_verbose``
@@ -144,17 +154,21 @@ Examples
1. List all settings and theirs current values 1. List all settings and theirs current values
.. code-block:: bash .. code::
> platformio settings get
$ platformio settings get
Name Value [Default] Description Name Value [Default] Description
------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------
auto_update_libraries Yes Automatically update libraries (Yes/No) auto_update_libraries No Automatically update libraries (Yes/No)
auto_update_platforms Yes Automatically update platforms (Yes/No) auto_update_platforms No Automatically update platforms (Yes/No)
check_libraries_interval 7 Check for the library updates interval (days) check_libraries_interval 7 Check for the library updates interval (days)
check_platformio_interval 3 Check for the new PlatformIO interval (days) check_platformio_interval 3 Check for the new PlatformIO interval (days)
check_platforms_interval 7 Check for the platform updates interval (days) check_platforms_interval 7 Check for the platform updates interval (days)
enable_cache Yes Enable caching for API requests and Library Manager
enable_ssl No Enable SSL for PlatformIO Services
enable_telemetry Yes Telemetry service (Yes/No) enable_telemetry Yes Telemetry service (Yes/No)
force_verbose No Force verbose output when processing environments
2. Show specified setting 2. Show specified setting
@@ -223,8 +237,12 @@ Examples
Name Value [Default] Description Name Value [Default] Description
------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------
auto_update_libraries Yes Automatically update libraries (Yes/No) auto_update_libraries No Automatically update libraries (Yes/No)
auto_update_platforms Yes Automatically update platforms (Yes/No) auto_update_platforms No Automatically update platforms (Yes/No)
check_libraries_interval 7 Check for the library updates interval (days) check_libraries_interval 7 Check for the library updates interval (days)
check_platformio_interval 3 Check for the new PlatformIO interval (days) check_platformio_interval 3 Check for the new PlatformIO interval (days)
check_platforms_interval 7 Check for the platform updates interval (days) check_platforms_interval 7 Check for the platform updates interval (days)
enable_cache Yes Enable caching for API requests and Library Manager
enable_ssl No Enable SSL for PlatformIO Services
enable_telemetry Yes Telemetry service (Yes/No)
force_verbose No Force verbose output when processing environments

22
platformio/app.py
View File

@@ -56,6 +56,10 @@ DEFAULT_SETTINGS = {
"description": "Enable SSL for PlatformIO Services", "description": "Enable SSL for PlatformIO Services",
"value": False "value": False
}, },
"enable_cache": {
"description": "Enable caching for API requests and Library Manager",
"value": True
},
"enable_telemetry": { "enable_telemetry": {
"description": "description":
("Telemetry service <http://docs.platformio.org/en/stable/" ("Telemetry service <http://docs.platformio.org/en/stable/"
@@ -122,16 +126,20 @@ class State(object):
self._lockfile.release() self._lockfile.release()
class LocalCache(object): class ContentCache(object):
def __init__(self, cache_dir=None): def __init__(self, cache_dir=None):
self.cache_dir = None
self.db_path = None
if not get_setting("enable_cache"):
return
self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache") self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache")
if not self.cache_dir: if not self.cache_dir:
os.makedirs(self.cache_dir) os.makedirs(self.cache_dir)
self.db_path = join(self.cache_dir, "db.data") self.db_path = join(self.cache_dir, "db.data")
def __enter__(self): def __enter__(self):
if not isfile(self.db_path): if not self.db_path or not isfile(self.db_path):
return self return self
newlines = [] newlines = []
found = False found = False
@@ -169,24 +177,26 @@ class LocalCache(object):
return h.hexdigest() return h.hexdigest()
def get(self, key): def get(self, key):
if not self.cache_dir:
return None
cache_path = self.get_cache_path(key) cache_path = self.get_cache_path(key)
if not isfile(cache_path): if not isfile(cache_path):
return None return None
with open(cache_path) as fp: with open(cache_path, "rb") as fp:
data = fp.read() data = fp.read()
if data[0] in ("{", "["): if data[0] in ("{", "["):
return json.loads(data) return json.loads(data)
return data return data
def set(self, key, data, valid): def set(self, key, data, valid):
if not data: if not self.cache_dir or not data:
return return
tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
assert valid.endswith(tuple(tdmap.keys())) assert valid.endswith(tuple(tdmap.keys()))
cache_path = self.get_cache_path(key) cache_path = self.get_cache_path(key)
if not isdir(dirname(cache_path)): if not isdir(dirname(cache_path)):
os.makedirs(dirname(cache_path)) os.makedirs(dirname(cache_path))
with open(cache_path, "w") as fp: with open(cache_path, "wb") as fp:
if isinstance(data, dict) or isinstance(data, list): if isinstance(data, dict) or isinstance(data, list):
json.dump(data, fp) json.dump(data, fp)
else: else:
@@ -197,7 +207,7 @@ class LocalCache(object):
return True return True
def clean(self): def clean(self):
if isdir(self.cache_dir): if self.cache_dir and isdir(self.cache_dir):
util.rmtree_(self.cache_dir) util.rmtree_(self.cache_dir)

4
platformio/maintenance.py
View File

@@ -43,8 +43,8 @@ def in_silence(ctx=None):
def clean_cache(): def clean_cache():
with app.LocalCache() as lc: with app.ContentCache() as cc:
lc.clean() cc.clean()
def on_platformio_start(ctx, force, caller): def on_platformio_start(ctx, force, caller):

7
platformio/managers/lib.py
View File

@@ -170,8 +170,11 @@ class LibraryManager(BasePkgManager):
def get_latest_repo_version(self, name, requirements): def get_latest_repo_version(self, name, requirements):
item = self.max_satisfying_repo_version( item = self.max_satisfying_repo_version(
util.get_api_result("/lib/versions/%d" % self._get_pkg_id_by_name( util.get_api_result(
name, requirements)), requirements) "/lib/versions/%d" % self._get_pkg_id_by_name(name,
requirements),
cache_valid="1h"),
requirements)
return item['version'] if item else None return item['version'] if item else None
def _get_pkg_id_by_name(self, def _get_pkg_id_by_name(self,

38
platformio/managers/package.py
View File

@@ -14,15 +14,15 @@
import json import json
import os import os
from os.path import basename, dirname, isdir, isfile, islink, join import shutil
from shutil import copytree from os.path import basename, dirname, getsize, isdir, isfile, islink, join
from tempfile import mkdtemp from tempfile import mkdtemp
import click import click
import requests import requests
import semantic_version import semantic_version
from platformio import exception, telemetry, util from platformio import app, exception, telemetry, util
from platformio.downloader import FileDownloader from platformio.downloader import FileDownloader
from platformio.unpacker import FileUnpacker from platformio.unpacker import FileUnpacker
from platformio.vcsclient import VCSClientFactory from platformio.vcsclient import VCSClientFactory
@@ -190,7 +190,7 @@ class PkgInstallerMixin(object):
self.unpack(url, tmp_dir) self.unpack(url, tmp_dir)
else: else:
util.rmtree_(tmp_dir) util.rmtree_(tmp_dir)
copytree(url, tmp_dir) shutil.copytree(url, tmp_dir)
elif url.startswith(("http://", "https://")): elif url.startswith(("http://", "https://")):
dlpath = self.download(url, tmp_dir, sha1) dlpath = self.download(url, tmp_dir, sha1)
assert isfile(dlpath) assert isfile(dlpath)
@@ -262,6 +262,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
_INSTALLED_CACHE = {} _INSTALLED_CACHE = {}
FILE_CACHE_VALID = "1m" # 1 month
FILE_CACHE_MAX_SIZE = 1024 * 1024
def __init__(self, package_dir, repositories=None): def __init__(self, package_dir, repositories=None):
self.repositories = repositories self.repositories = repositories
self.package_dir = package_dir self.package_dir = package_dir
@@ -273,13 +276,32 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
def manifest_name(self): def manifest_name(self):
raise NotImplementedError() raise NotImplementedError()
@staticmethod def download(self, url, dest_dir, sha1=None):
def download(url, dest_dir, sha1=None): cache_key_fname = app.ContentCache.key_from_args(url, "fname")
cache_key_data = app.ContentCache.key_from_args(url, "data")
if self.FILE_CACHE_VALID:
with app.ContentCache() as cc:
fname = cc.get(cache_key_fname)
cache_path = cc.get_cache_path(cache_key_data)
if fname and isfile(cache_path):
dst_path = join(dest_dir, fname)
shutil.copy(cache_path, dst_path)
return dst_path
fd = FileDownloader(url, dest_dir) fd = FileDownloader(url, dest_dir)
fd.start() fd.start()
if sha1: if sha1:
fd.verify(sha1) fd.verify(sha1)
return fd.get_filepath() dst_path = fd.get_filepath()
if not self.FILE_CACHE_VALID or getsize(
dst_path) > BasePkgManager.FILE_CACHE_MAX_SIZE:
return dst_path
with app.ContentCache() as cc:
cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
return dst_path
@staticmethod @staticmethod
def unpack(source_path, dest_dir): def unpack(source_path, dest_dir):
@@ -580,6 +602,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
class PackageManager(BasePkgManager): class PackageManager(BasePkgManager):
FILE_CACHE_VALID = None # disable package caching
@property @property
def manifest_name(self): def manifest_name(self):
return "package.json" return "package.json"

2
platformio/managers/platform.py
View File

@@ -27,6 +27,8 @@ from platformio.managers.package import BasePkgManager, PackageManager
class PlatformManager(BasePkgManager): class PlatformManager(BasePkgManager):
FILE_CACHE_VALID = None # disable platform caching
def __init__(self, package_dir=None, repositories=None): def __init__(self, package_dir=None, repositories=None):
if not repositories: if not repositories:
repositories = [ repositories = [

12
platformio/util.py
View File

@@ -482,22 +482,22 @@ def _get_api_result(
def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
from platformio.app import LocalCache from platformio.app import ContentCache
total = 0 total = 0
max_retries = 5 max_retries = 5
cache_key = (LocalCache.key_from_args(url, params, data, auth) cache_key = (ContentCache.key_from_args(url, params, data, auth)
if cache_valid else None) if cache_valid else None)
while total < max_retries: while total < max_retries:
try: try:
with LocalCache() as lc: with ContentCache() as cc:
if cache_key: if cache_key:
result = lc.get(cache_key) result = cc.get(cache_key)
if result is not None: if result is not None:
return result return result
result = _get_api_result(url, params, data) result = _get_api_result(url, params, data)
if cache_valid: if cache_valid:
with LocalCache() as lc: with ContentCache() as cc:
lc.set(cache_key, result, cache_valid) cc.set(cache_key, result, cache_valid)
return result return result
except (requests.exceptions.ConnectionError, except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as e: requests.exceptions.Timeout) as e: