From d54327f1a944fa50b3a899adeaf857d2bc389abc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 23 Mar 2018 00:08:07 +0200 Subject: [PATCH] Refactor @memoized decorator with expiration feature; cache installed boards per platform --- platformio/builder/tools/piomisc.py | 2 +- platformio/builder/tools/pioplatform.py | 2 +- platformio/ide/projectgenerator.py | 3 +- platformio/managers/core.py | 2 +- platformio/managers/lib.py | 4 +- platformio/managers/platform.py | 5 ++- platformio/util.py | 52 ++++++++----------------- 7 files changed, 25 insertions(+), 45 deletions(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 69e859f3..bff38d6e 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -199,7 +199,7 @@ def _delete_file(path): pass -@util.memoized +@util.memoized() def _get_compiler_type(env): try: sysenv = environ.copy() diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 6a77bc74..bc5f4663 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -23,7 +23,7 @@ from platformio import exception, util from platformio.managers.platform import PlatformFactory -@util.memoized +@util.memoized() def initPioPlatform(name): return PlatformFactory.newPlatform(name) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index cde75693..f1dcd35e 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -40,7 +40,7 @@ class ProjectGenerator(object): return sorted( [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) - @util.memoized + @util.memoized() def get_project_env(self): data = {} config = util.load_project_config(self.project_dir) @@ -54,7 +54,6 @@ class ProjectGenerator(object): data[k] = v return data - @util.memoized def get_project_build_data(self): data = { "defines": [], diff --git a/platformio/managers/core.py b/platformio/managers/core.py index ff54904e..76a20008 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -23,7 +23,7 @@ from platformio.managers.package import PackageManager CORE_PACKAGES = { "contrib-piohome": ">=0.9.1,<2", "contrib-pysite": ">=0.2.0,<2", - "tool-pioplus": ">=1.1.2,<2", + "tool-pioplus": ">=1.1.3,<2", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.4" } diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 9fa3d02a..bc5677a7 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -398,7 +398,7 @@ class LibraryManager(BasePkgManager): return pkg_dir -@util.memoized +@util.memoized() def get_builtin_libs(storage_names=None): items = [] storage_names = storage_names or [] @@ -417,7 +417,7 @@ def get_builtin_libs(storage_names=None): return items -@util.memoized +@util.memoized() def is_builtin_lib(name): for storage in get_builtin_libs(): if any(l.get("name") == name for l in storage['items']): diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index bffcf16c..b77c0080 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -30,7 +30,7 @@ from platformio.managers.package import BasePkgManager, PackageManager class PlatformManager(BasePkgManager): - FILE_CACHE_VALID = None # disable platform caching + FILE_CACHE_VALID = None # disable platform download caching def __init__(self, package_dir=None, repositories=None): if not repositories: @@ -158,6 +158,7 @@ class PlatformManager(BasePkgManager): self.cache_reset() return True + @util.memoized(expire=5000) def get_installed_boards(self): boards = [] for manifest in self.get_installed(): @@ -169,7 +170,7 @@ class PlatformManager(BasePkgManager): return boards @staticmethod - @util.memoized + @util.memoized() def get_registered_boards(): return util.get_api_result("/boards", cache_valid="7d") diff --git a/platformio/util.py b/platformio/util.py index 8e1c3b7d..10bfed7c 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections -import functools import json import os import platform @@ -113,40 +111,22 @@ class cd(object): class memoized(object): - ''' - Decorator. Caches a function's return value each time it is called. - If called later with the same arguments, the cached value is returned - (not reevaluated). - https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize - ''' - def __init__(self, func): - self.func = func + def __init__(self, expire=0): + self.expire = expire / 1000 # milliseconds self.cache = {} - def __call__(self, *args): - if not isinstance(args, collections.Hashable): - # uncacheable. a list, for instance. - # better to not cache than blow up. - return self.func(*args) - if args in self.cache: - return self.cache[args] - value = self.func(*args) - self.cache[args] = value - return value + def __call__(self, func): - def __repr__(self): - '''Return the function's docstring.''' - return self.func.__doc__ + @wraps(func) + def wrapper(*args, **kwargs): + key = str(args) + str(kwargs) + if (key not in self.cache + or self.cache[key][0] < time.time() - self.expire): + self.cache[key] = (time.time(), func(*args, **kwargs)) + return self.cache[key][1] - def __get__(self, obj, objtype): - '''Support instance methods.''' - fn = functools.partial(self.__call__, obj) - fn.reset = self._reset - return fn - - def _reset(self): - self.cache = {} + return wrapper class throttle(object): @@ -155,15 +135,15 @@ class throttle(object): self.threshhold = threshhold # milliseconds self.last = 0 - def __call__(self, fn): + def __call__(self, func): - @wraps(fn) + @wraps(func) def wrapper(*args, **kwargs): diff = int(round((time.time() - self.last) * 1000)) if diff < self.threshhold: time.sleep((self.threshhold - diff) * 0.001) self.last = time.time() - return fn(*args, **kwargs) + return func(*args, **kwargs) return wrapper @@ -568,7 +548,7 @@ def get_request_defheaders(): return {"User-Agent": "PlatformIO/%s CI/%d %s" % data} -@memoized +@memoized(expire=10000) def _api_request_session(): return requests.Session() @@ -671,7 +651,7 @@ PING_INTERNET_IPS = [ ] -@memoized +@memoized(expire=5000) def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout)