Merge branch 'feature/performance-fixes' into develop

platformio/__main__.py

@@ -119,8 +119,7 @@ An unexpected error occurred. Further steps:
 
 
 def debug_gdb_main():
-    sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]
+    return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:])
-    return main()
 
 
 if __name__ == "__main__":

platformio/app.py

@@ -16,7 +16,6 @@ import codecs
 import hashlib
 import os
 import uuid
-from copy import deepcopy
 from os import environ, getenv, listdir, remove
 from os.path import abspath, dirname, expanduser, isdir, isfile, join
 from time import time
@@ -93,28 +92,26 @@ class State(object):
         self.lock = lock
         if not self.path:
             self.path = join(get_project_core_dir(), "appstate.json")
-        self._state = {}
+        self._storage = {}
-        self._prev_state = {}
         self._lockfile = None
+        self._modified = False
 
     def __enter__(self):
         try:
             self._lock_state_file()
             if isfile(self.path):
-                self._state = util.load_json(self.path)
+                self._storage = util.load_json(self.path)
-                assert isinstance(self._state, dict)
+            assert isinstance(self._storage, dict)
-        except (AssertionError, UnicodeDecodeError,
+        except (AssertionError, ValueError, UnicodeDecodeError,
-                exception.PlatformioException):
+                exception.InvalidJSONFile):
-            self._state = {}
+            self._storage = {}
-        self._prev_state = deepcopy(self._state)
+        return self
-        return self._state
 
     def __exit__(self, type_, value, traceback):
-        new_state = dump_json_to_unicode(self._state)
+        if self._modified:
-        if self._prev_state != new_state:
             try:
                 with open(self.path, "w") as fp:
-                    fp.write(new_state)
+                    fp.write(dump_json_to_unicode(self._storage))
             except IOError:
                 raise exception.HomeDirPermissionsError(get_project_core_dir())
         self._unlock_state_file()
@@ -132,8 +129,31 @@ class State(object):
         if hasattr(self, "_lockfile") and self._lockfile:
             self._lockfile.release()
 
-    def __del__(self):
+    # Dictionary Proxy
-        self._unlock_state_file()
+
+    def as_dict(self):
+        return self._storage
+
+    def get(self, key, default=True):
+        return self._storage.get(key, default)
+
+    def update(self, *args, **kwargs):
+        self._modified = True
+        return self._storage.update(*args, **kwargs)
+
+    def __getitem__(self, key):
+        return self._storage[key]
+
+    def __setitem__(self, key, value):
+        self._modified = True
+        self._storage[key] = value
+
+    def __delitem__(self, key):
+        self._modified = True
+        del self._storage[key]
+
+    def __contains__(self, item):
+        return item in self._storage
 
 
 class ContentCache(object):

platformio/commands/debug/command.py

@@ -123,7 +123,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface,
         if helpers.is_mi_mode(__unprocessed):
             click.echo('~"Preparing firmware for debugging...\\n"')
             output = helpers.GDBBytesIO()
-            with helpers.capture_std_streams(output):
+            with util.capture_std_streams(output):
                 helpers.predebug_project(ctx, project_dir, env_name, preload,
                                          verbose)
             output.close()

platformio/commands/debug/helpers.py

@@ -14,7 +14,6 @@
 
 import sys
 import time
-from contextlib import contextmanager
 from fnmatch import fnmatch
 from hashlib import sha1
 from io import BytesIO
@@ -41,17 +40,6 @@ class GDBBytesIO(BytesIO):  # pylint: disable=too-few-public-methods
         self.STDOUT.flush()
 
 
-@contextmanager
-def capture_std_streams(stdout, stderr=None):
-    _stdout = sys.stdout
-    _stderr = sys.stderr
-    sys.stdout = stdout
-    sys.stderr = stderr or stdout
-    yield
-    sys.stdout = _stdout
-    sys.stderr = _stderr
-
-
 def is_mi_mode(args):
     return "--interpreter" in " ".join(args)
 

platformio/commands/home/rpc/handlers/app.py

@@ -57,7 +57,7 @@ class AppRPC(object):
             ]
 
             state['storage'] = storage
-            return state
+            return state.as_dict()
 
     @staticmethod
     def get_state():
@@ -66,6 +66,6 @@ class AppRPC(object):
     @staticmethod
     def save_state(state):
         with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
-            s.clear()
+            # s.clear()
             s.update(state)
         return True

platformio/commands/home/rpc/handlers/os.py

@@ -14,6 +14,7 @@
 
 from __future__ import absolute_import
 
+import codecs
 import glob
 import os
 import shutil
@@ -67,10 +68,10 @@ class OSRPC(object):
     def request_content(self, uri, data=None, headers=None, cache_valid=None):
         if uri.startswith('http'):
             return self.fetch_content(uri, data, headers, cache_valid)
-        if isfile(uri):
+        if not isfile(uri):
-            with open(uri) as fp:
+            return None
-                return fp.read()
+        with codecs.open(uri, encoding="utf-8") as fp:
-        return None
+            return fp.read()
 
     @staticmethod
     def open_url(url):

platformio/commands/home/rpc/handlers/piocore.py

@@ -16,21 +16,64 @@ from __future__ import absolute_import
 
 import json
 import os
-import re
+import sys
+from io import BytesIO
 
 import jsonrpc  # pylint: disable=import-error
-from twisted.internet import utils  # pylint: disable=import-error
+from twisted.internet import threads  # pylint: disable=import-error
 
-from platformio import __version__
+from platformio import __main__, __version__, util
-from platformio.commands.home import helpers
 from platformio.compat import string_types
 
+try:
+    from thread import get_ident as thread_get_ident
+except ImportError:
+    from threading import get_ident as thread_get_ident
+
+
+class ThreadSafeStdBuffer(object):
+
+    def __init__(self, parent_stream, parent_thread_id):
+        self.parent_stream = parent_stream
+        self.parent_thread_id = parent_thread_id
+        self._buffer = {}
+
+    def write(self, value):
+        thread_id = thread_get_ident()
+        if thread_id == self.parent_thread_id:
+            return self.parent_stream.write(
+                value if isinstance(value, string_types) else value.decode())
+        if thread_id not in self._buffer:
+            self._buffer[thread_id] = BytesIO()
+        return self._buffer[thread_id].write(value)
+
+    def flush(self):
+        return (self.parent_stream.flush()
+                if thread_get_ident() == self.parent_thread_id else None)
+
+    def getvalue_and_close(self, thread_id=None):
+        thread_id = thread_id or thread_get_ident()
+        if thread_id not in self._buffer:
+            return ""
+        result = self._buffer.get(thread_id).getvalue()
+        self._buffer.get(thread_id).close()
+        del self._buffer[thread_id]
+        return result
+
 
 class PIOCoreRPC(object):
 
+    def __init__(self):
+        cur_thread_id = thread_get_ident()
+        PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout,
+                                                       cur_thread_id)
+        PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr,
+                                                       cur_thread_id)
+        sys.stdout = PIOCoreRPC.thread_stdout
+        sys.stderr = PIOCoreRPC.thread_stderr
+
     @staticmethod
     def call(args, options=None):
-        json_output = "--json-output" in args
         try:
             args = [
                 str(arg) if not isinstance(arg, string_types) else arg
@@ -39,13 +82,15 @@ class PIOCoreRPC(object):
         except UnicodeError:
             raise jsonrpc.exceptions.JSONRPCDispatchException(
                 code=4002, message="PIO Core: non-ASCII chars in arguments")
-        d = utils.getProcessOutputAndValue(
+
-            helpers.get_core_fullpath(),
+        def _call_cli():
-            args,
+            with util.cd((options or {}).get("cwd") or os.getcwd()):
-            path=(options or {}).get("cwd"),
+                exit_code = __main__.main(["-c"] + args)
-            env={k: v
+            return (PIOCoreRPC.thread_stdout.getvalue_and_close(),
-                 for k, v in os.environ.items() if "%" not in k})
+                    PIOCoreRPC.thread_stderr.getvalue_and_close(), exit_code)
-        d.addCallback(PIOCoreRPC._call_callback, json_output)
+
+        d = threads.deferToThread(_call_cli)
+        d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args)
         d.addErrback(PIOCoreRPC._call_errback)
         return d
 
@@ -55,15 +100,7 @@ class PIOCoreRPC(object):
         text = ("%s\n\n%s" % (out, err)).strip()
         if code != 0:
             raise Exception(text)
-        if not json_output:
+        return json.loads(out) if json_output else text
-            return text
-        try:
-            return json.loads(out)
-        except ValueError as e:
-            if "sh: " in out:
-                return json.loads(
-                    re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip())
-            raise e
 
     @staticmethod
     def _call_errback(failure):

platformio/commands/home/rpc/server.py

@@ -14,6 +14,7 @@
 
 # pylint: disable=import-error
 
+import click
 import jsonrpc
 from autobahn.twisted.websocket import (WebSocketServerFactory,
                                         WebSocketServerProtocol)
@@ -26,7 +27,7 @@ from platformio.compat import PY2, dump_json_to_unicode, is_bytes
 class JSONRPCServerProtocol(WebSocketServerProtocol):
 
     def onMessage(self, payload, isBinary):  # pylint: disable=unused-argument
-        # print("> %s" % payload)
+        # click.echo("> %s" % payload)
         response = jsonrpc.JSONRPCResponseManager.handle(
             payload, self.factory.dispatcher).data
         # if error
@@ -52,11 +53,12 @@ class JSONRPCServerProtocol(WebSocketServerProtocol):
                                          message=failure.getErrorMessage())
         del response["result"]
         response['error'] = e.error._data  # pylint: disable=protected-access
-        print(response['error'])
         self.sendJSONResponse(response)
 
     def sendJSONResponse(self, response):
-        # print("< %s" % response)
+        # click.echo("< %s" % response)
+        if "error" in response:
+            click.secho("Error: %s" % response['error'], fg="red", err=True)
         response = dump_json_to_unicode(response)
         if not PY2 and not is_bytes(response):
             response = response.encode("utf-8")

platformio/compat.py

@@ -54,9 +54,12 @@ if PY2:
         return data
 
     def dump_json_to_unicode(obj):
+        if isinstance(obj, unicode):
+            return obj
         return json.dumps(obj,
                           encoding=get_filesystem_encoding(),
-                          ensure_ascii=False).encode("utf8")
+                          ensure_ascii=False,
+                          sort_keys=True).encode("utf8")
 
     _magic_check = re.compile('([*?[])')
     _magic_check_bytes = re.compile(b'([*?[])')
@@ -100,4 +103,6 @@ else:
         return data.encode()
 
     def dump_json_to_unicode(obj):
-        return json.dumps(obj, ensure_ascii=False)
+        if isinstance(obj, string_types):
+            return obj
+        return json.dumps(obj, ensure_ascii=False, sort_keys=True)

platformio/util.py

@@ -20,6 +20,7 @@ import socket
 import stat
 import sys
 import time
+from contextlib import contextmanager
 from functools import wraps
 from glob import glob
 from os.path import abspath, basename, dirname, isfile, join
@@ -107,6 +108,17 @@ def singleton(cls):
     return get_instance
 
 
+@contextmanager
+def capture_std_streams(stdout, stderr=None):
+    _stdout = sys.stdout
+    _stderr = sys.stderr
+    sys.stdout = stdout
+    sys.stderr = stderr or stdout
+    yield
+    sys.stdout = _stdout
+    sys.stderr = _stderr
+
+
 def load_json(file_path):
     try:
         with open(file_path, "r") as f: