From dfca7f0b6877e9f3fe06920e9276339a778c8fd6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Jul 2019 15:55:42 +0300 Subject: [PATCH 1/4] Speedup PIO Home via internal calling of PIO Core CLI --- platformio/__main__.py | 3 +- platformio/commands/debug/command.py | 2 +- platformio/commands/debug/helpers.py | 12 -------- platformio/commands/home/rpc/handlers/os.py | 9 +++--- .../commands/home/rpc/handlers/piocore.py | 28 +++++++++++-------- platformio/commands/platform.py | 2 +- platformio/compat.py | 4 +++ platformio/util.py | 12 ++++++++ 8 files changed, 41 insertions(+), 31 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index 8b1ab093..d4664935 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -119,8 +119,7 @@ An unexpected error occurred. Further steps: def debug_gdb_main(): - sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:] - return main() + return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]) if __name__ == "__main__": diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 7892cfee..8599a962 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -123,7 +123,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, if helpers.is_mi_mode(__unprocessed): click.echo('~"Preparing firmware for debugging...\\n"') output = helpers.GDBBytesIO() - with helpers.capture_std_streams(output): + with util.capture_std_streams(output): helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) output.close() diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index ddc9db97..daaa8d93 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -14,7 +14,6 @@ import sys import time -from contextlib import contextmanager from fnmatch import fnmatch from hashlib import sha1 from io import BytesIO @@ -41,17 +40,6 @@ class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods self.STDOUT.flush() -@contextmanager -def capture_std_streams(stdout, stderr=None): - _stdout = sys.stdout - _stderr = sys.stderr - sys.stdout = stdout - sys.stderr = stderr or stdout - yield - sys.stdout = _stdout - sys.stderr = _stderr - - def is_mi_mode(args): return "--interpreter" in " ".join(args) diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index afb2e29f..c84f486e 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,6 +14,7 @@ from __future__ import absolute_import +import codecs import glob import os import shutil @@ -67,10 +68,10 @@ class OSRPC(object): def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith('http'): return self.fetch_content(uri, data, headers, cache_valid) - if isfile(uri): - with open(uri) as fp: - return fp.read() - return None + if not isfile(uri): + return None + with codecs.open(uri, encoding="utf-8") as fp: + return fp.read() @staticmethod def open_url(url): diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 198552d1..19d7d370 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,12 +17,12 @@ from __future__ import absolute_import import json import os import re +from io import BytesIO import jsonrpc # pylint: disable=import-error -from twisted.internet import utils # pylint: disable=import-error +from twisted.internet import threads # pylint: disable=import-error -from platformio import __version__ -from platformio.commands.home import helpers +from platformio import __main__, __version__, util from platformio.compat import string_types @@ -30,7 +30,6 @@ class PIOCoreRPC(object): @staticmethod def call(args, options=None): - json_output = "--json-output" in args try: args = [ str(arg) if not isinstance(arg, string_types) else arg @@ -39,13 +38,20 @@ class PIOCoreRPC(object): except UnicodeError: raise jsonrpc.exceptions.JSONRPCDispatchException( code=4002, message="PIO Core: non-ASCII chars in arguments") - d = utils.getProcessOutputAndValue( - helpers.get_core_fullpath(), - args, - path=(options or {}).get("cwd"), - env={k: v - for k, v in os.environ.items() if "%" not in k}) - d.addCallback(PIOCoreRPC._call_callback, json_output) + + def _call_cli(): + outbuff = BytesIO() + errbuff = BytesIO() + with util.capture_std_streams(outbuff, errbuff): + with util.cd((options or {}).get("cwd") or os.getcwd()): + exit_code = __main__.main(["-c"] + args) + result = (outbuff.getvalue(), errbuff.getvalue(), exit_code) + outbuff.close() + errbuff.close() + return result + + d = threads.deferToThread(_call_cli) + d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) d.addErrback(PIOCoreRPC._call_errback) return d diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 5b35c4df..26f86666 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -182,7 +182,7 @@ def platform_frameworks(query, json_output): for framework in util.get_api_result("/frameworks", cache_valid="7d"): if query == "all": query = "" - search_data = dump_json_to_unicode(framework) + search_data = framework if query and query.lower() not in search_data.lower(): continue framework['homepage'] = ("https://platformio.org/frameworks/" + diff --git a/platformio/compat.py b/platformio/compat.py index 4aad4ea2..686518a8 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -54,6 +54,8 @@ if PY2: return data def dump_json_to_unicode(obj): + if isinstance(obj, unicode): + return obj return json.dumps(obj, encoding=get_filesystem_encoding(), ensure_ascii=False).encode("utf8") @@ -100,4 +102,6 @@ else: return data.encode() def dump_json_to_unicode(obj): + if isinstance(obj, string_types): + return obj return json.dumps(obj, ensure_ascii=False) diff --git a/platformio/util.py b/platformio/util.py index b93b4d8a..377160b1 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -20,6 +20,7 @@ import socket import stat import sys import time +from contextlib import contextmanager from functools import wraps from glob import glob from os.path import abspath, basename, dirname, isfile, join @@ -107,6 +108,17 @@ def singleton(cls): return get_instance +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + def load_json(file_path): try: with open(file_path, "r") as f: From bf77d70d8295e2c00ac46adc6ee38c91ad708a1d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Jul 2019 20:39:52 +0300 Subject: [PATCH 2/4] Thread safe internal PIO Core calling for PIO Home --- .../commands/home/rpc/handlers/piocore.py | 64 +++++++++++++------ platformio/commands/home/rpc/server.py | 7 +- platformio/commands/platform.py | 2 +- 3 files changed, 50 insertions(+), 23 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 19d7d370..4370b5a9 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -16,7 +16,8 @@ from __future__ import absolute_import import json import os -import re +import sys +import thread from io import BytesIO import jsonrpc # pylint: disable=import-error @@ -26,8 +27,46 @@ from platformio import __main__, __version__, util from platformio.compat import string_types +class ThreadSafeStdBuffer(object): + + def __init__(self, parent_stream, parent_thread_id): + self.parent_stream = parent_stream + self.parent_thread_id = parent_thread_id + self._buffer = {} + + def write(self, value): + thread_id = thread.get_ident() + if thread_id == self.parent_thread_id: + return self.parent_stream.write(value) + if thread_id not in self._buffer: + self._buffer[thread_id] = BytesIO() + return self._buffer[thread_id].write(value) + + def flush(self): + return (self.parent_stream.flush() + if thread.get_ident() == self.parent_thread_id else None) + + def getvalue_and_close(self, thread_id=None): + thread_id = thread_id or thread.get_ident() + if thread_id not in self._buffer: + return "" + result = self._buffer.get(thread_id).getvalue() + self._buffer.get(thread_id).close() + del self._buffer[thread_id] + return result + + class PIOCoreRPC(object): + def __init__(self): + cur_thread_id = thread.get_ident() + PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout, + cur_thread_id) + PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr, + cur_thread_id) + sys.stdout = PIOCoreRPC.thread_stdout + sys.stderr = PIOCoreRPC.thread_stderr + @staticmethod def call(args, options=None): try: @@ -40,15 +79,10 @@ class PIOCoreRPC(object): code=4002, message="PIO Core: non-ASCII chars in arguments") def _call_cli(): - outbuff = BytesIO() - errbuff = BytesIO() - with util.capture_std_streams(outbuff, errbuff): - with util.cd((options or {}).get("cwd") or os.getcwd()): - exit_code = __main__.main(["-c"] + args) - result = (outbuff.getvalue(), errbuff.getvalue(), exit_code) - outbuff.close() - errbuff.close() - return result + with util.cd((options or {}).get("cwd") or os.getcwd()): + exit_code = __main__.main(["-c"] + args) + return (PIOCoreRPC.thread_stdout.getvalue_and_close(), + PIOCoreRPC.thread_stderr.getvalue_and_close(), exit_code) d = threads.deferToThread(_call_cli) d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) @@ -61,15 +95,7 @@ class PIOCoreRPC(object): text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) - if not json_output: - return text - try: - return json.loads(out) - except ValueError as e: - if "sh: " in out: - return json.loads( - re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip()) - raise e + return json.loads(out) if json_output else text @staticmethod def _call_errback(failure): diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index f99b09b3..b77e1b94 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -14,6 +14,7 @@ # pylint: disable=import-error +import click import jsonrpc from autobahn.twisted.websocket import (WebSocketServerFactory, WebSocketServerProtocol) @@ -26,7 +27,7 @@ from platformio.compat import PY2, dump_json_to_unicode, is_bytes class JSONRPCServerProtocol(WebSocketServerProtocol): def onMessage(self, payload, isBinary): # pylint: disable=unused-argument - # print("> %s" % payload) + # click.echo("> %s" % payload) response = jsonrpc.JSONRPCResponseManager.handle( payload, self.factory.dispatcher).data # if error @@ -52,11 +53,11 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): message=failure.getErrorMessage()) del response["result"] response['error'] = e.error._data # pylint: disable=protected-access - print(response['error']) + click.secho(str(response['error']), fg="red", err=True) self.sendJSONResponse(response) def sendJSONResponse(self, response): - # print("< %s" % response) + # click.echo("< %s" % response) response = dump_json_to_unicode(response) if not PY2 and not is_bytes(response): response = response.encode("utf-8") diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 26f86666..5b35c4df 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -182,7 +182,7 @@ def platform_frameworks(query, json_output): for framework in util.get_api_result("/frameworks", cache_valid="7d"): if query == "all": query = "" - search_data = framework + search_data = dump_json_to_unicode(framework) if query and query.lower() not in search_data.lower(): continue framework['homepage'] = ("https://platformio.org/frameworks/" + From 6d9de80f121d80ff008221ad9577095551763cc1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Jul 2019 20:42:23 +0300 Subject: [PATCH 3/4] Better comparison for app state changes --- platformio/app.py | 19 ++++++++++--------- platformio/compat.py | 5 +++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index c35c4baf..12dcbb41 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,9 +14,9 @@ import codecs import hashlib +import json import os import uuid -from copy import deepcopy from os import environ, getenv, listdir, remove from os.path import abspath, dirname, expanduser, isdir, isfile, join from time import time @@ -94,27 +94,28 @@ class State(object): if not self.path: self.path = join(get_project_core_dir(), "appstate.json") self._state = {} - self._prev_state = {} + self._prev_state_raw = "" self._lockfile = None def __enter__(self): try: self._lock_state_file() if isfile(self.path): - self._state = util.load_json(self.path) + with open(self.path) as fp: + self._prev_state_raw = fp.read().strip() + self._state = json.loads(self._prev_state_raw) assert isinstance(self._state, dict) - except (AssertionError, UnicodeDecodeError, - exception.PlatformioException): + except (AssertionError, ValueError, UnicodeDecodeError): self._state = {} - self._prev_state = deepcopy(self._state) + self._prev_state_raw = "" return self._state def __exit__(self, type_, value, traceback): - new_state = dump_json_to_unicode(self._state) - if self._prev_state != new_state: + new_state_raw = dump_json_to_unicode(self._state) + if self._prev_state_raw != new_state_raw: try: with open(self.path, "w") as fp: - fp.write(new_state) + fp.write(new_state_raw) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() diff --git a/platformio/compat.py b/platformio/compat.py index 686518a8..8b082b4b 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -58,7 +58,8 @@ if PY2: return obj return json.dumps(obj, encoding=get_filesystem_encoding(), - ensure_ascii=False).encode("utf8") + ensure_ascii=False, + sort_keys=True).encode("utf8") _magic_check = re.compile('([*?[])') _magic_check_bytes = re.compile(b'([*?[])') @@ -104,4 +105,4 @@ else: def dump_json_to_unicode(obj): if isinstance(obj, string_types): return obj - return json.dumps(obj, ensure_ascii=False) + return json.dumps(obj, ensure_ascii=False, sort_keys=True) From d2c86ab71c4bb1ecae9a710e092358cc1e467ee0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 00:41:47 +0300 Subject: [PATCH 4/4] Refactor state to a proxied dictionary --- platformio/app.py | 51 +++++++++++++------ platformio/commands/home/rpc/handlers/app.py | 4 +- .../commands/home/rpc/handlers/piocore.py | 17 ++++--- platformio/commands/home/rpc/server.py | 3 +- 4 files changed, 50 insertions(+), 25 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 12dcbb41..a6fd6e45 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,7 +14,6 @@ import codecs import hashlib -import json import os import uuid from os import environ, getenv, listdir, remove @@ -93,29 +92,26 @@ class State(object): self.lock = lock if not self.path: self.path = join(get_project_core_dir(), "appstate.json") - self._state = {} - self._prev_state_raw = "" + self._storage = {} self._lockfile = None + self._modified = False def __enter__(self): try: self._lock_state_file() if isfile(self.path): - with open(self.path) as fp: - self._prev_state_raw = fp.read().strip() - self._state = json.loads(self._prev_state_raw) - assert isinstance(self._state, dict) - except (AssertionError, ValueError, UnicodeDecodeError): - self._state = {} - self._prev_state_raw = "" - return self._state + self._storage = util.load_json(self.path) + assert isinstance(self._storage, dict) + except (AssertionError, ValueError, UnicodeDecodeError, + exception.InvalidJSONFile): + self._storage = {} + return self def __exit__(self, type_, value, traceback): - new_state_raw = dump_json_to_unicode(self._state) - if self._prev_state_raw != new_state_raw: + if self._modified: try: with open(self.path, "w") as fp: - fp.write(new_state_raw) + fp.write(dump_json_to_unicode(self._storage)) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() @@ -133,8 +129,31 @@ class State(object): if hasattr(self, "_lockfile") and self._lockfile: self._lockfile.release() - def __del__(self): - self._unlock_state_file() + # Dictionary Proxy + + def as_dict(self): + return self._storage + + def get(self, key, default=True): + return self._storage.get(key, default) + + def update(self, *args, **kwargs): + self._modified = True + return self._storage.update(*args, **kwargs) + + def __getitem__(self, key): + return self._storage[key] + + def __setitem__(self, key, value): + self._modified = True + self._storage[key] = value + + def __delitem__(self, key): + self._modified = True + del self._storage[key] + + def __contains__(self, item): + return item in self._storage class ContentCache(object): diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 0f8b8285..1666dc17 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -57,7 +57,7 @@ class AppRPC(object): ] state['storage'] = storage - return state + return state.as_dict() @staticmethod def get_state(): @@ -66,6 +66,6 @@ class AppRPC(object): @staticmethod def save_state(state): with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: - s.clear() + # s.clear() s.update(state) return True diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 4370b5a9..62dfdd3e 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import json import os import sys -import thread from io import BytesIO import jsonrpc # pylint: disable=import-error @@ -26,6 +25,11 @@ from twisted.internet import threads # pylint: disable=import-error from platformio import __main__, __version__, util from platformio.compat import string_types +try: + from thread import get_ident as thread_get_ident +except ImportError: + from threading import get_ident as thread_get_ident + class ThreadSafeStdBuffer(object): @@ -35,19 +39,20 @@ class ThreadSafeStdBuffer(object): self._buffer = {} def write(self, value): - thread_id = thread.get_ident() + thread_id = thread_get_ident() if thread_id == self.parent_thread_id: - return self.parent_stream.write(value) + return self.parent_stream.write( + value if isinstance(value, string_types) else value.decode()) if thread_id not in self._buffer: self._buffer[thread_id] = BytesIO() return self._buffer[thread_id].write(value) def flush(self): return (self.parent_stream.flush() - if thread.get_ident() == self.parent_thread_id else None) + if thread_get_ident() == self.parent_thread_id else None) def getvalue_and_close(self, thread_id=None): - thread_id = thread_id or thread.get_ident() + thread_id = thread_id or thread_get_ident() if thread_id not in self._buffer: return "" result = self._buffer.get(thread_id).getvalue() @@ -59,7 +64,7 @@ class ThreadSafeStdBuffer(object): class PIOCoreRPC(object): def __init__(self): - cur_thread_id = thread.get_ident() + cur_thread_id = thread_get_ident() PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout, cur_thread_id) PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr, diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index b77e1b94..36aa1dff 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -53,11 +53,12 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): message=failure.getErrorMessage()) del response["result"] response['error'] = e.error._data # pylint: disable=protected-access - click.secho(str(response['error']), fg="red", err=True) self.sendJSONResponse(response) def sendJSONResponse(self, response): # click.echo("< %s" % response) + if "error" in response: + click.secho("Error: %s" % response['error'], fg="red", err=True) response = dump_json_to_unicode(response) if not PY2 and not is_bytes(response): response = response.encode("utf-8")