platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-29 17:47:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Enable caching by default for API requests and Library Manager

Browse Source
This commit is contained in:
Ivan Kravets
2016-12-05 18:51:25 +02:00
parent 6de8325130
commit 08cda5e7ea
8 changed files with 87 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.rst
View File

@ -44,6 +44,7 @@ PlatformIO 3.0
* Added global ``lib_extra_dirs`` option to ``[platformio]`` section for
`Project Configuration File "platformio.ini" <http://docs.platformio.org/en/stable/projectconf.html>`__
(`issue #842 <https://github.com/platformio/platformio-core/issues/842>`_)
* Enabled caching by default for API requests and Library Manager (see `enable_cache <http://docs.platformio.org/en/latest/userguide/cmd_settings.html#enable-cache>`__ setting)
* Changed a default exit combination for Device Monitor from ``Ctrl+]`` to ``Ctrl+C``
* Improved detecting of ARM mbed media disk for uploading
* Improved Project Generator for CLion IDE when source folder contains nested items

30
docs/userguide/cmd_settings.rst
View File

@ -97,6 +97,16 @@ Check for the platform updates interval.
Enable SSL for PlatformIO Services
.. _enable_cache:
``enable_cache``
^^^^^^^^^^^^^^^^
:Default: Yes
:Values: Yes/No
Enable caching for API requests and Library Manager
.. _setting_force_verbose:
``force_verbose``
@ -144,17 +154,21 @@ Examples
1. List all settings and theirs current values
.. code-block:: bash
.. code::
> platformio settings get
$ platformio settings get
Name Value [Default] Description
------------------------------------------------------------------------------------------
auto_update_libraries Yes Automatically update libraries (Yes/No)
auto_update_platforms Yes Automatically update platforms (Yes/No)
auto_update_libraries No Automatically update libraries (Yes/No)
auto_update_platforms No Automatically update platforms (Yes/No)
check_libraries_interval 7 Check for the library updates interval (days)
check_platformio_interval 3 Check for the new PlatformIO interval (days)
check_platforms_interval 7 Check for the platform updates interval (days)
enable_cache Yes Enable caching for API requests and Library Manager
enable_ssl No Enable SSL for PlatformIO Services
enable_telemetry Yes Telemetry service (Yes/No)
force_verbose No Force verbose output when processing environments
2. Show specified setting
@ -223,8 +237,12 @@ Examples
Name Value [Default] Description
------------------------------------------------------------------------------------------
auto_update_libraries Yes Automatically update libraries (Yes/No)
auto_update_platforms Yes Automatically update platforms (Yes/No)
auto_update_libraries No Automatically update libraries (Yes/No)
auto_update_platforms No Automatically update platforms (Yes/No)
check_libraries_interval 7 Check for the library updates interval (days)
check_platformio_interval 3 Check for the new PlatformIO interval (days)
check_platforms_interval 7 Check for the platform updates interval (days)
enable_cache Yes Enable caching for API requests and Library Manager
enable_ssl No Enable SSL for PlatformIO Services
enable_telemetry Yes Telemetry service (Yes/No)
force_verbose No Force verbose output when processing environments

22
platformio/app.py
View File

@ -56,6 +56,10 @@ DEFAULT_SETTINGS = {
"description": "Enable SSL for PlatformIO Services",
"value": False
},
"enable_cache": {
"description": "Enable caching for API requests and Library Manager",
"value": True
},
"enable_telemetry": {
"description":
("Telemetry service <http://docs.platformio.org/en/stable/"
@ -122,16 +126,20 @@ class State(object):
self._lockfile.release()
class LocalCache(object):
class ContentCache(object):
def __init__(self, cache_dir=None):
self.cache_dir = None
self.db_path = None
if not get_setting("enable_cache"):
return
self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache")
if not self.cache_dir:
os.makedirs(self.cache_dir)
self.db_path = join(self.cache_dir, "db.data")
def __enter__(self):
if not isfile(self.db_path):
if not self.db_path or not isfile(self.db_path):
return self
newlines = []
found = False
@ -169,24 +177,26 @@ class LocalCache(object):
return h.hexdigest()
def get(self, key):
if not self.cache_dir:
return None
cache_path = self.get_cache_path(key)
if not isfile(cache_path):
return None
with open(cache_path) as fp:
with open(cache_path, "rb") as fp:
data = fp.read()
if data[0] in ("{", "["):
return json.loads(data)
return data
def set(self, key, data, valid):
if not data:
if not self.cache_dir or not data:
return
tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
assert valid.endswith(tuple(tdmap.keys()))
cache_path = self.get_cache_path(key)
if not isdir(dirname(cache_path)):
os.makedirs(dirname(cache_path))
with open(cache_path, "w") as fp:
with open(cache_path, "wb") as fp:
if isinstance(data, dict) or isinstance(data, list):
json.dump(data, fp)
else:
@ -197,7 +207,7 @@ class LocalCache(object):
return True
def clean(self):
if isdir(self.cache_dir):
if self.cache_dir and isdir(self.cache_dir):
util.rmtree_(self.cache_dir)

4
platformio/maintenance.py
View File

@ -43,8 +43,8 @@ def in_silence(ctx=None):
def clean_cache():
with app.LocalCache() as lc:
lc.clean()
with app.ContentCache() as cc:
cc.clean()
def on_platformio_start(ctx, force, caller):

7
platformio/managers/lib.py
View File

@ -170,8 +170,11 @@ class LibraryManager(BasePkgManager):
def get_latest_repo_version(self, name, requirements):
item = self.max_satisfying_repo_version(
util.get_api_result("/lib/versions/%d" % self._get_pkg_id_by_name(
name, requirements)), requirements)
util.get_api_result(
"/lib/versions/%d" % self._get_pkg_id_by_name(name,
requirements),
cache_valid="1h"),
requirements)
return item['version'] if item else None
def _get_pkg_id_by_name(self,

38
platformio/managers/package.py
View File

@ -14,15 +14,15 @@
import json
import os
from os.path import basename, dirname, isdir, isfile, islink, join
from shutil import copytree
import shutil
from os.path import basename, dirname, getsize, isdir, isfile, islink, join
from tempfile import mkdtemp
import click
import requests
import semantic_version
from platformio import exception, telemetry, util
from platformio import app, exception, telemetry, util
from platformio.downloader import FileDownloader
from platformio.unpacker import FileUnpacker
from platformio.vcsclient import VCSClientFactory
@ -190,7 +190,7 @@ class PkgInstallerMixin(object):
self.unpack(url, tmp_dir)
else:
util.rmtree_(tmp_dir)
copytree(url, tmp_dir)
shutil.copytree(url, tmp_dir)
elif url.startswith(("http://", "https://")):
dlpath = self.download(url, tmp_dir, sha1)
assert isfile(dlpath)
@ -262,6 +262,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
_INSTALLED_CACHE = {}
FILE_CACHE_VALID = "1m" # 1 month
FILE_CACHE_MAX_SIZE = 1024 * 1024
def __init__(self, package_dir, repositories=None):
self.repositories = repositories
self.package_dir = package_dir
@ -273,13 +276,32 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
def manifest_name(self):
raise NotImplementedError()
@staticmethod
def download(url, dest_dir, sha1=None):
def download(self, url, dest_dir, sha1=None):
cache_key_fname = app.ContentCache.key_from_args(url, "fname")
cache_key_data = app.ContentCache.key_from_args(url, "data")
if self.FILE_CACHE_VALID:
with app.ContentCache() as cc:
fname = cc.get(cache_key_fname)
cache_path = cc.get_cache_path(cache_key_data)
if fname and isfile(cache_path):
dst_path = join(dest_dir, fname)
shutil.copy(cache_path, dst_path)
return dst_path
fd = FileDownloader(url, dest_dir)
fd.start()
if sha1:
fd.verify(sha1)
return fd.get_filepath()
dst_path = fd.get_filepath()
if not self.FILE_CACHE_VALID or getsize(
dst_path) > BasePkgManager.FILE_CACHE_MAX_SIZE:
return dst_path
with app.ContentCache() as cc:
cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
return dst_path
@staticmethod
def unpack(source_path, dest_dir):
@ -580,6 +602,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
class PackageManager(BasePkgManager):
FILE_CACHE_VALID = None # disable package caching
@property
def manifest_name(self):
return "package.json"

2
platformio/managers/platform.py
View File

@ -27,6 +27,8 @@ from platformio.managers.package import BasePkgManager, PackageManager
class PlatformManager(BasePkgManager):
FILE_CACHE_VALID = None # disable platform caching
def __init__(self, package_dir=None, repositories=None):
if not repositories:
repositories = [

12
platformio/util.py
View File

@ -482,22 +482,22 @@ def _get_api_result(
def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
from platformio.app import LocalCache
from platformio.app import ContentCache
total = 0
max_retries = 5
cache_key = (LocalCache.key_from_args(url, params, data, auth)
cache_key = (ContentCache.key_from_args(url, params, data, auth)
if cache_valid else None)
while total < max_retries:
try:
with LocalCache() as lc:
with ContentCache() as cc:
if cache_key:
result = lc.get(cache_key)
result = cc.get(cache_key)
if result is not None:
return result
result = _get_api_result(url, params, data)
if cache_valid:
with LocalCache() as lc:
lc.set(cache_key, result, cache_valid)
with ContentCache() as cc:
cc.set(cache_key, result, cache_valid)
return result
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as e: