diff --git a/platformio/__main__.py b/platformio/__main__.py index 8b1ab093..d4664935 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -119,8 +119,7 @@ An unexpected error occurred. Further steps: def debug_gdb_main(): - sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:] - return main() + return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]) if __name__ == "__main__": diff --git a/platformio/app.py b/platformio/app.py index c35c4baf..a6fd6e45 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -16,7 +16,6 @@ import codecs import hashlib import os import uuid -from copy import deepcopy from os import environ, getenv, listdir, remove from os.path import abspath, dirname, expanduser, isdir, isfile, join from time import time @@ -93,28 +92,26 @@ class State(object): self.lock = lock if not self.path: self.path = join(get_project_core_dir(), "appstate.json") - self._state = {} - self._prev_state = {} + self._storage = {} self._lockfile = None + self._modified = False def __enter__(self): try: self._lock_state_file() if isfile(self.path): - self._state = util.load_json(self.path) - assert isinstance(self._state, dict) - except (AssertionError, UnicodeDecodeError, - exception.PlatformioException): - self._state = {} - self._prev_state = deepcopy(self._state) - return self._state + self._storage = util.load_json(self.path) + assert isinstance(self._storage, dict) + except (AssertionError, ValueError, UnicodeDecodeError, + exception.InvalidJSONFile): + self._storage = {} + return self def __exit__(self, type_, value, traceback): - new_state = dump_json_to_unicode(self._state) - if self._prev_state != new_state: + if self._modified: try: with open(self.path, "w") as fp: - fp.write(new_state) + fp.write(dump_json_to_unicode(self._storage)) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() @@ -132,8 +129,31 @@ class State(object): if hasattr(self, "_lockfile") and self._lockfile: self._lockfile.release() - def __del__(self): - self._unlock_state_file() + # Dictionary Proxy + + def as_dict(self): + return self._storage + + def get(self, key, default=True): + return self._storage.get(key, default) + + def update(self, *args, **kwargs): + self._modified = True + return self._storage.update(*args, **kwargs) + + def __getitem__(self, key): + return self._storage[key] + + def __setitem__(self, key, value): + self._modified = True + self._storage[key] = value + + def __delitem__(self, key): + self._modified = True + del self._storage[key] + + def __contains__(self, item): + return item in self._storage class ContentCache(object): diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 7892cfee..8599a962 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -123,7 +123,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, if helpers.is_mi_mode(__unprocessed): click.echo('~"Preparing firmware for debugging...\\n"') output = helpers.GDBBytesIO() - with helpers.capture_std_streams(output): + with util.capture_std_streams(output): helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) output.close() diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index ddc9db97..daaa8d93 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -14,7 +14,6 @@ import sys import time -from contextlib import contextmanager from fnmatch import fnmatch from hashlib import sha1 from io import BytesIO @@ -41,17 +40,6 @@ class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods self.STDOUT.flush() -@contextmanager -def capture_std_streams(stdout, stderr=None): - _stdout = sys.stdout - _stderr = sys.stderr - sys.stdout = stdout - sys.stderr = stderr or stdout - yield - sys.stdout = _stdout - sys.stderr = _stderr - - def is_mi_mode(args): return "--interpreter" in " ".join(args) diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 0f8b8285..1666dc17 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -57,7 +57,7 @@ class AppRPC(object): ] state['storage'] = storage - return state + return state.as_dict() @staticmethod def get_state(): @@ -66,6 +66,6 @@ class AppRPC(object): @staticmethod def save_state(state): with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: - s.clear() + # s.clear() s.update(state) return True diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index afb2e29f..c84f486e 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,6 +14,7 @@ from __future__ import absolute_import +import codecs import glob import os import shutil @@ -67,10 +68,10 @@ class OSRPC(object): def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith('http'): return self.fetch_content(uri, data, headers, cache_valid) - if isfile(uri): - with open(uri) as fp: - return fp.read() - return None + if not isfile(uri): + return None + with codecs.open(uri, encoding="utf-8") as fp: + return fp.read() @staticmethod def open_url(url): diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 198552d1..62dfdd3e 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -16,21 +16,64 @@ from __future__ import absolute_import import json import os -import re +import sys +from io import BytesIO import jsonrpc # pylint: disable=import-error -from twisted.internet import utils # pylint: disable=import-error +from twisted.internet import threads # pylint: disable=import-error -from platformio import __version__ -from platformio.commands.home import helpers +from platformio import __main__, __version__, util from platformio.compat import string_types +try: + from thread import get_ident as thread_get_ident +except ImportError: + from threading import get_ident as thread_get_ident + + +class ThreadSafeStdBuffer(object): + + def __init__(self, parent_stream, parent_thread_id): + self.parent_stream = parent_stream + self.parent_thread_id = parent_thread_id + self._buffer = {} + + def write(self, value): + thread_id = thread_get_ident() + if thread_id == self.parent_thread_id: + return self.parent_stream.write( + value if isinstance(value, string_types) else value.decode()) + if thread_id not in self._buffer: + self._buffer[thread_id] = BytesIO() + return self._buffer[thread_id].write(value) + + def flush(self): + return (self.parent_stream.flush() + if thread_get_ident() == self.parent_thread_id else None) + + def getvalue_and_close(self, thread_id=None): + thread_id = thread_id or thread_get_ident() + if thread_id not in self._buffer: + return "" + result = self._buffer.get(thread_id).getvalue() + self._buffer.get(thread_id).close() + del self._buffer[thread_id] + return result + class PIOCoreRPC(object): + def __init__(self): + cur_thread_id = thread_get_ident() + PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout, + cur_thread_id) + PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr, + cur_thread_id) + sys.stdout = PIOCoreRPC.thread_stdout + sys.stderr = PIOCoreRPC.thread_stderr + @staticmethod def call(args, options=None): - json_output = "--json-output" in args try: args = [ str(arg) if not isinstance(arg, string_types) else arg @@ -39,13 +82,15 @@ class PIOCoreRPC(object): except UnicodeError: raise jsonrpc.exceptions.JSONRPCDispatchException( code=4002, message="PIO Core: non-ASCII chars in arguments") - d = utils.getProcessOutputAndValue( - helpers.get_core_fullpath(), - args, - path=(options or {}).get("cwd"), - env={k: v - for k, v in os.environ.items() if "%" not in k}) - d.addCallback(PIOCoreRPC._call_callback, json_output) + + def _call_cli(): + with util.cd((options or {}).get("cwd") or os.getcwd()): + exit_code = __main__.main(["-c"] + args) + return (PIOCoreRPC.thread_stdout.getvalue_and_close(), + PIOCoreRPC.thread_stderr.getvalue_and_close(), exit_code) + + d = threads.deferToThread(_call_cli) + d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) d.addErrback(PIOCoreRPC._call_errback) return d @@ -55,15 +100,7 @@ class PIOCoreRPC(object): text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) - if not json_output: - return text - try: - return json.loads(out) - except ValueError as e: - if "sh: " in out: - return json.loads( - re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip()) - raise e + return json.loads(out) if json_output else text @staticmethod def _call_errback(failure): diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index f99b09b3..36aa1dff 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -14,6 +14,7 @@ # pylint: disable=import-error +import click import jsonrpc from autobahn.twisted.websocket import (WebSocketServerFactory, WebSocketServerProtocol) @@ -26,7 +27,7 @@ from platformio.compat import PY2, dump_json_to_unicode, is_bytes class JSONRPCServerProtocol(WebSocketServerProtocol): def onMessage(self, payload, isBinary): # pylint: disable=unused-argument - # print("> %s" % payload) + # click.echo("> %s" % payload) response = jsonrpc.JSONRPCResponseManager.handle( payload, self.factory.dispatcher).data # if error @@ -52,11 +53,12 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): message=failure.getErrorMessage()) del response["result"] response['error'] = e.error._data # pylint: disable=protected-access - print(response['error']) self.sendJSONResponse(response) def sendJSONResponse(self, response): - # print("< %s" % response) + # click.echo("< %s" % response) + if "error" in response: + click.secho("Error: %s" % response['error'], fg="red", err=True) response = dump_json_to_unicode(response) if not PY2 and not is_bytes(response): response = response.encode("utf-8") diff --git a/platformio/compat.py b/platformio/compat.py index 4aad4ea2..8b082b4b 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -54,9 +54,12 @@ if PY2: return data def dump_json_to_unicode(obj): + if isinstance(obj, unicode): + return obj return json.dumps(obj, encoding=get_filesystem_encoding(), - ensure_ascii=False).encode("utf8") + ensure_ascii=False, + sort_keys=True).encode("utf8") _magic_check = re.compile('([*?[])') _magic_check_bytes = re.compile(b'([*?[])') @@ -100,4 +103,6 @@ else: return data.encode() def dump_json_to_unicode(obj): - return json.dumps(obj, ensure_ascii=False) + if isinstance(obj, string_types): + return obj + return json.dumps(obj, ensure_ascii=False, sort_keys=True) diff --git a/platformio/util.py b/platformio/util.py index b93b4d8a..377160b1 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -20,6 +20,7 @@ import socket import stat import sys import time +from contextlib import contextmanager from functools import wraps from glob import glob from os.path import abspath, basename, dirname, isfile, join @@ -107,6 +108,17 @@ def singleton(cls): return get_instance +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + def load_json(file_path): try: with open(file_path, "r") as f: