Improve cache logic for lookup operations

platformio/pkgmanager.py

@@ -8,28 +8,24 @@ from time import time
 
 import click
 
-from platformio import exception, telemetry
+from platformio import exception, telemetry, util
 from platformio.app import get_state_item, set_state_item
 from platformio.downloader import FileDownloader
 from platformio.unpacker import FileUnpacker
-from platformio.util import get_api_result, get_home_dir, get_systype
 
 
 class PackageManager(object):
 
     def __init__(self):
-        self._package_dir = join(get_home_dir(), "packages")
+        self._package_dir = join(util.get_home_dir(), "packages")
         if not isdir(self._package_dir):
             makedirs(self._package_dir)
         assert isdir(self._package_dir)
 
     @classmethod
+    @util.memoized
     def get_manifest(cls):
-        try:
+        return util.get_api_result("/packages/manifest")
-            return cls._cached_manifest
-        except AttributeError:
-            cls._cached_manifest = get_api_result("/packages/manifest")
-        return cls._cached_manifest
 
     @staticmethod
     def download(url, dest_dir, sha1=None):
@@ -63,7 +59,7 @@ class PackageManager(object):
             raise exception.UnknownPackage(name)
 
         # check system platform
-        systype = get_systype()
+        systype = util.get_systype()
         builds = ([b for b in manifest[name] if b['system'] == "all" or systype
                    in b['system']])
         if not builds:

platformio/platforms/base.py

@@ -132,32 +132,33 @@ class PlatformFactory(object):
         return module
 
     @classmethod
-    def get_platforms(cls, installed=False):
+    @util.memoized
+    def _lookup_platforms(cls):
         platforms = {}
-
+        for d in (util.get_home_dir(), util.get_source_dir()):
-        try:
+            pdir = join(d, "platforms")
-            platforms = cls.get_platforms_cache
+            if not isdir(pdir):
-        except AttributeError:
+                continue
-            for d in (util.get_home_dir(), util.get_source_dir()):
+            for p in listdir(pdir):
-                pdir = join(d, "platforms")
+                if (p in ("__init__.py", "base.py") or not
-                if not isdir(pdir):
+                        p.endswith(".py")):
                     continue
-                for p in listdir(pdir):
+                type_ = p[:-3]
-                    if (p in ("__init__.py", "base.py") or not
+                path = join(pdir, p)
-                            p.endswith(".py")):
+                try:
-                        continue
+                    isplatform = hasattr(
-                    type_ = p[:-3]
+                        cls.load_module(type_, path),
-                    path = join(pdir, p)
+                        cls.get_clsname(type_)
-                    try:
+                    )
-                        isplatform = hasattr(
+                    if isplatform:
-                            cls.load_module(type_, path),
+                        platforms[type_] = path
-                            cls.get_clsname(type_)
+                except exception.UnknownPlatform:
-                        )
+                    pass
-                        if isplatform:
+        return platforms
-                            platforms[type_] = path
+
-                    except exception.UnknownPlatform:
+    @classmethod
-                        pass
+    def get_platforms(cls, installed=False):
-            cls.get_platforms_cache = platforms
+        platforms = cls._lookup_platforms()
 
         if not installed:
             return platforms

platformio/util.py

@@ -1,6 +1,8 @@
 # Copyright (C) Ivan Kravets <me@ikravets.com>
 # See LICENSE for details.
 
+import collections
+import functools
 import json
 import os
 import re
@@ -53,6 +55,39 @@ class AsyncPipe(Thread):
         self.join()
 
 
+class memoized(object):
+    '''
+    Decorator. Caches a function's return value each time it is called.
+    If called later with the same arguments, the cached value is returned
+    (not reevaluated).
+    https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize
+    '''
+
+    def __init__(self, func):
+        self.func = func
+        self.cache = {}
+
+    def __call__(self, *args):
+        if not isinstance(args, collections.Hashable):
+            # uncacheable. a list, for instance.
+            # better to not cache than blow up.
+            return self.func(*args)
+        if args in self.cache:
+            return self.cache[args]
+        else:
+            value = self.func(*args)
+            self.cache[args] = value
+            return value
+
+    def __repr__(self):
+        '''Return the function's docstring.'''
+        return self.func.__doc__
+
+    def __get__(self, obj, objtype):
+        '''Support instance methods.'''
+        return functools.partial(self.__call__, obj)
+
+
 def get_systype():
     data = uname()
     return ("%s_%s" % (data[0], data[4])).lower()
@@ -239,22 +274,24 @@ def get_api_result(path, params=None, data=None):
     return result
 
 
-def get_boards(type_=None):
+@memoized
+def _lookup_boards():
     boards = {}
-    try:
+    bdirs = [join(get_source_dir(), "boards")]
-        boards = get_boards._cache  # pylint: disable=W0212
+    if isdir(join(get_home_dir(), "boards")):
-    except AttributeError:
+        bdirs.append(join(get_home_dir(), "boards"))
-        bdirs = [join(get_source_dir(), "boards")]
-        if isdir(join(get_home_dir(), "boards")):
-            bdirs.append(join(get_home_dir(), "boards"))
 
-        for bdir in bdirs:
+    for bdir in bdirs:
-            for json_file in os.listdir(bdir):
+        for json_file in os.listdir(bdir):
-                if not json_file.endswith(".json"):
+            if not json_file.endswith(".json"):
-                    continue
+                continue
-                with open(join(bdir, json_file)) as f:
+            with open(join(bdir, json_file)) as f:
-                    boards.update(json.load(f))
+                boards.update(json.load(f))
-        get_boards._cache = boards  # pylint: disable=W0212
+    return boards
+
+
+def get_boards(type_=None):
+    boards = _lookup_boards()
 
     if type_ is None:
         return boards
@@ -264,33 +301,34 @@ def get_boards(type_=None):
         return boards[type_]
 
 
-def get_frameworks(type_=None):
+@memoized
+def _lookup_frameworks():
     frameworks = {}
+    frameworks_path = join(
+        get_source_dir(), "builder", "scripts", "frameworks")
 
-    try:
+    frameworks_list = [f[:-3] for f in os.listdir(frameworks_path)
-        frameworks = get_frameworks._cache  # pylint: disable=W0212
+                       if not f.startswith("__") and f.endswith(".py")]
-    except AttributeError:
+    for _type in frameworks_list:
-        frameworks_path = join(
+        script_path = join(frameworks_path, "%s.py" % _type)
-            get_source_dir(), "builder", "scripts", "frameworks")
+        with open(script_path) as f:
+            fcontent = f.read()
+            assert '"""' in fcontent
+            _doc_start = fcontent.index('"""') + 3
+            fdoc = fcontent[
+                _doc_start:fcontent.index('"""', _doc_start)].strip()
+            doclines = [l.strip() for l in fdoc.splitlines() if l.strip()]
+            frameworks[_type] = {
+                "name": doclines[0],
+                "description": " ".join(doclines[1:-1]),
+                "url": doclines[-1],
+                "script": script_path
+            }
+    return frameworks
 
-        frameworks_list = [f[:-3] for f in os.listdir(frameworks_path)
+
-                           if not f.startswith("__") and f.endswith(".py")]
+def get_frameworks(type_=None):
-        for _type in frameworks_list:
+    frameworks = _lookup_frameworks()
-            script_path = join(frameworks_path, "%s.py" % _type)
-            with open(script_path) as f:
-                fcontent = f.read()
-                assert '"""' in fcontent
-                _doc_start = fcontent.index('"""') + 3
-                fdoc = fcontent[
-                    _doc_start:fcontent.index('"""', _doc_start)].strip()
-                doclines = [l.strip() for l in fdoc.splitlines() if l.strip()]
-                frameworks[_type] = {
-                    "name": doclines[0],
-                    "description": " ".join(doclines[1:-1]),
-                    "url": doclines[-1],
-                    "script": script_path
-                }
-        get_frameworks._cache = frameworks  # pylint: disable=W0212
 
     if type_ is None:
         return frameworks