From bcff26d4d7e42d35ab34145ac48d2242d491110b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 6 Jun 2019 00:13:04 +0300 Subject: [PATCH] Refactor using "@util.memoized" --- platformio/commands/home/helpers.py | 6 ++--- platformio/commands/lib.py | 6 ++++- platformio/managers/lib.py | 12 +++++---- platformio/managers/package.py | 41 +++++++++++++---------------- platformio/managers/platform.py | 3 +-- platformio/util.py | 13 ++++++--- 6 files changed, 44 insertions(+), 37 deletions(-) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index a8ff898f..46c1750c 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -43,18 +43,18 @@ class AsyncSession(requests.Session): return defer.ensureDeferred(*args, **kwargs) -@util.memoized(expire=5000) +@util.memoized(expire="60s") def requests_session(): return AsyncSession(n=5) -@util.memoized() +@util.memoized(expire="60s") def get_core_fullpath(): return where_is_program( "platformio" + (".exe" if "windows" in util.get_systype() else "")) -@util.memoized(expire=10000) +@util.memoized(expire="10s") def is_twitter_blocked(): ip = "104.244.42.1" timeout = 2 diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index ad39bf53..c66dc77b 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -147,6 +147,7 @@ def lib_install( # pylint: disable=too-many-arguments force=force) installed_manifests[library] = lm.load_manifest(pkg_dir) elif storage_dir in storage_libdeps: + builtin_lib_storages = None for library in storage_libdeps[storage_dir]: try: pkg_dir = lm.install(library, @@ -155,7 +156,10 @@ def lib_install( # pylint: disable=too-many-arguments force=force) installed_manifests[library] = lm.load_manifest(pkg_dir) except exception.LibNotFound as e: - if not silent or not is_builtin_lib(library): + if builtin_lib_storages is None: + builtin_lib_storages = get_builtin_libs() + if not silent or not is_builtin_lib( + builtin_lib_storages, library): click.secho("Warning! %s" % e, fg="yellow") if not save or not libraries: diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 1c0d04dd..e995d751 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -361,6 +361,7 @@ class LibraryManager(BasePkgManager): if not silent: click.secho("Installing dependencies", fg="yellow") + builtin_lib_storages = None for filters in self.normalize_dependencies(manifest['dependencies']): assert "name" in filters @@ -382,7 +383,10 @@ class LibraryManager(BasePkgManager): try: lib_id = self.search_lib_id(filters, silent, interactive) except exception.LibNotFound as e: - if not silent or is_builtin_lib(filters['name']): + if builtin_lib_storages is None: + builtin_lib_storages = get_builtin_libs() + if not silent or is_builtin_lib(builtin_lib_storages, + filters['name']): click.secho("Warning! %s" % e, fg="yellow") continue @@ -402,7 +406,6 @@ class LibraryManager(BasePkgManager): return pkg_dir -@util.memoized() def get_builtin_libs(storage_names=None): items = [] storage_names = storage_names or [] @@ -421,9 +424,8 @@ def get_builtin_libs(storage_names=None): return items -@util.memoized() -def is_builtin_lib(name): - for storage in get_builtin_libs(): +def is_builtin_lib(storages, name): + for storage in storages or []: if any(l.get("name") == name for l in storage['items']): return True return False diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 617237e9..6eb20188 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -37,8 +37,6 @@ from platformio.vcsclient import VCSClientFactory class PackageRepoIterator(object): - _MANIFEST_CACHE = {} - def __init__(self, package, repositories): assert isinstance(repositories, list) self.package = package @@ -50,27 +48,25 @@ class PackageRepoIterator(object): def __next__(self): return self.next() - def next(self): - manifest = {} - repo = next(self.repositories) - if isinstance(repo, dict): - manifest = repo - elif repo in PackageRepoIterator._MANIFEST_CACHE: - manifest = PackageRepoIterator._MANIFEST_CACHE[repo] - else: - r = None - try: - r = requests.get(repo, headers=util.get_request_defheaders()) - r.raise_for_status() - manifest = r.json() - except: # pylint: disable=bare-except - pass - finally: - if r: - r.close() - PackageRepoIterator._MANIFEST_CACHE[repo] = manifest + @staticmethod + @util.memoized(expire="60s") + def load_manifest(url): + r = None + try: + r = requests.get(url, headers=util.get_request_defheaders()) + r.raise_for_status() + return r.json() + except: # pylint: disable=bare-except + pass + finally: + if r: + r.close() + return None - if self.package in manifest: + def next(self): + repo = next(self.repositories) + manifest = repo if isinstance(repo, dict) else self.load_manifest(repo) + if manifest and self.package in manifest: return manifest[self.package] return next(self) @@ -795,6 +791,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): return True def update(self, package, requirements=None, only_check=False): + self.cache_reset() if isdir(package) and self.get_package_by_dir(package): pkg_dir = package else: diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index d34b9880..506275c5 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -173,7 +173,7 @@ class PlatformManager(BasePkgManager): self.cache_reset() return True - @util.memoized(expire=5000) + @util.memoized(expire="5s") def get_installed_boards(self): boards = [] for manifest in self.get_installed(): @@ -185,7 +185,6 @@ class PlatformManager(BasePkgManager): return boards @staticmethod - @util.memoized() def get_registered_boards(): return util.get_api_result("/boards", cache_valid="7d") diff --git a/platformio/util.py b/platformio/util.py index e07ce391..097cfa8e 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -50,7 +50,12 @@ class cd(object): class memoized(object): def __init__(self, expire=0): - self.expire = expire / 1000 # milliseconds + expire = str(expire) + if expire.isdigit(): + expire = "%ss" % (int(expire) / 1000) + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert expire.endswith(tuple(tdmap)) + self.expire = int(tdmap[expire[-1]] * int(expire[:-1])) self.cache = {} def __call__(self, func): @@ -68,7 +73,7 @@ class memoized(object): return wrapper def _reset(self): - self.cache = {} + self.cache.clear() class throttle(object): @@ -296,7 +301,7 @@ def get_request_defheaders(): return {"User-Agent": "PlatformIO/%s CI/%d %s" % data} -@memoized(expire=10000) +@memoized(expire="60s") def _api_request_session(): return requests.Session() @@ -394,7 +399,7 @@ PING_INTERNET_IPS = [ ] -@memoized(expire=5000) +@memoized(expire="5s") def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout)