platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-29 17:47:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/performance-fixes' into develop

Browse Source
This commit is contained in:
Ivan Kravets
2019-07-02 00:42:04 +03:00
parent 8525fd6ae8 d2c86ab71c
commit 5afaa6d0ee
10 changed files with 126 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
platformio/__main__.py
View File

@ -119,8 +119,7 @@ An unexpected error occurred. Further steps:
def debug_gdb_main():
sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]
return main()
return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:])
if __name__ == "__main__":

50
platformio/app.py
View File

@ -16,7 +16,6 @@ import codecs
import hashlib
import os
import uuid
from copy import deepcopy
from os import environ, getenv, listdir, remove
from os.path import abspath, dirname, expanduser, isdir, isfile, join
from time import time
@ -93,28 +92,26 @@ class State(object):
self.lock = lock
if not self.path:
self.path = join(get_project_core_dir(), "appstate.json")
self._state = {}
self._prev_state = {}
self._storage = {}
self._lockfile = None
self._modified = False
def __enter__(self):
try:
self._lock_state_file()
if isfile(self.path):
self._state = util.load_json(self.path)
assert isinstance(self._state, dict)
except (AssertionError, UnicodeDecodeError,
exception.PlatformioException):
self._state = {}
self._prev_state = deepcopy(self._state)
return self._state
self._storage = util.load_json(self.path)
assert isinstance(self._storage, dict)
except (AssertionError, ValueError, UnicodeDecodeError,
exception.InvalidJSONFile):
self._storage = {}
return self
def __exit__(self, type_, value, traceback):
new_state = dump_json_to_unicode(self._state)
if self._prev_state != new_state:
if self._modified:
try:
with open(self.path, "w") as fp:
fp.write(new_state)
fp.write(dump_json_to_unicode(self._storage))
except IOError:
raise exception.HomeDirPermissionsError(get_project_core_dir())
self._unlock_state_file()
@ -132,8 +129,31 @@ class State(object):
if hasattr(self, "_lockfile") and self._lockfile:
self._lockfile.release()
def __del__(self):
self._unlock_state_file()
# Dictionary Proxy
def as_dict(self):
return self._storage
def get(self, key, default=True):
return self._storage.get(key, default)
def update(self, *args, **kwargs):
self._modified = True
return self._storage.update(*args, **kwargs)
def __getitem__(self, key):
return self._storage[key]
def __setitem__(self, key, value):
self._modified = True
self._storage[key] = value
def __delitem__(self, key):
self._modified = True
del self._storage[key]
def __contains__(self, item):
return item in self._storage
class ContentCache(object):

2
platformio/commands/debug/command.py
View File

@ -123,7 +123,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface,
if helpers.is_mi_mode(__unprocessed):
click.echo('~"Preparing firmware for debugging...\\n"')
output = helpers.GDBBytesIO()
with helpers.capture_std_streams(output):
with util.capture_std_streams(output):
helpers.predebug_project(ctx, project_dir, env_name, preload,
verbose)
output.close()

12
platformio/commands/debug/helpers.py
View File

@ -14,7 +14,6 @@
import sys
import time
from contextlib import contextmanager
from fnmatch import fnmatch
from hashlib import sha1
from io import BytesIO
@ -41,17 +40,6 @@ class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods
self.STDOUT.flush()
@contextmanager
def capture_std_streams(stdout, stderr=None):
_stdout = sys.stdout
_stderr = sys.stderr
sys.stdout = stdout
sys.stderr = stderr or stdout
yield
sys.stdout = _stdout
sys.stderr = _stderr
def is_mi_mode(args):
return "--interpreter" in " ".join(args)

4
platformio/commands/home/rpc/handlers/app.py
View File

@ -57,7 +57,7 @@ class AppRPC(object):
]
state['storage'] = storage
return state
return state.as_dict()
@staticmethod
def get_state():
@ -66,6 +66,6 @@ class AppRPC(object):
@staticmethod
def save_state(state):
with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
s.clear()
# s.clear()
s.update(state)
return True

9
platformio/commands/home/rpc/handlers/os.py
View File

@ -14,6 +14,7 @@
from __future__ import absolute_import
import codecs
import glob
import os
import shutil
@ -67,10 +68,10 @@ class OSRPC(object):
def request_content(self, uri, data=None, headers=None, cache_valid=None):
if uri.startswith('http'):
return self.fetch_content(uri, data, headers, cache_valid)
if isfile(uri):
with open(uri) as fp:
return fp.read()
return None
if not isfile(uri):
return None
with codecs.open(uri, encoding="utf-8") as fp:
return fp.read()
@staticmethod
def open_url(url):

79
platformio/commands/home/rpc/handlers/piocore.py
View File

@ -16,21 +16,64 @@ from __future__ import absolute_import
import json
import os
import re
import sys
from io import BytesIO
import jsonrpc # pylint: disable=import-error
from twisted.internet import utils # pylint: disable=import-error
from twisted.internet import threads # pylint: disable=import-error
from platformio import __version__
from platformio.commands.home import helpers
from platformio import __main__, __version__, util
from platformio.compat import string_types
try:
from thread import get_ident as thread_get_ident
except ImportError:
from threading import get_ident as thread_get_ident
class ThreadSafeStdBuffer(object):
def __init__(self, parent_stream, parent_thread_id):
self.parent_stream = parent_stream
self.parent_thread_id = parent_thread_id
self._buffer = {}
def write(self, value):
thread_id = thread_get_ident()
if thread_id == self.parent_thread_id:
return self.parent_stream.write(
value if isinstance(value, string_types) else value.decode())
if thread_id not in self._buffer:
self._buffer[thread_id] = BytesIO()
return self._buffer[thread_id].write(value)
def flush(self):
return (self.parent_stream.flush()
if thread_get_ident() == self.parent_thread_id else None)
def getvalue_and_close(self, thread_id=None):
thread_id = thread_id or thread_get_ident()
if thread_id not in self._buffer:
return ""
result = self._buffer.get(thread_id).getvalue()
self._buffer.get(thread_id).close()
del self._buffer[thread_id]
return result
class PIOCoreRPC(object):
def __init__(self):
cur_thread_id = thread_get_ident()
PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout,
cur_thread_id)
PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr,
cur_thread_id)
sys.stdout = PIOCoreRPC.thread_stdout
sys.stderr = PIOCoreRPC.thread_stderr
@staticmethod
def call(args, options=None):
json_output = "--json-output" in args
try:
args = [
str(arg) if not isinstance(arg, string_types) else arg
@ -39,13 +82,15 @@ class PIOCoreRPC(object):
except UnicodeError:
raise jsonrpc.exceptions.JSONRPCDispatchException(
code=4002, message="PIO Core: non-ASCII chars in arguments")
d = utils.getProcessOutputAndValue(
helpers.get_core_fullpath(),
args,
path=(options or {}).get("cwd"),
env={k: v
for k, v in os.environ.items() if "%" not in k})
d.addCallback(PIOCoreRPC._call_callback, json_output)
def _call_cli():
with util.cd((options or {}).get("cwd") or os.getcwd()):
exit_code = __main__.main(["-c"] + args)
return (PIOCoreRPC.thread_stdout.getvalue_and_close(),
PIOCoreRPC.thread_stderr.getvalue_and_close(), exit_code)
d = threads.deferToThread(_call_cli)
d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args)
d.addErrback(PIOCoreRPC._call_errback)
return d
@ -55,15 +100,7 @@ class PIOCoreRPC(object):
text = ("%s\n\n%s" % (out, err)).strip()
if code != 0:
raise Exception(text)
if not json_output:
return text
try:
return json.loads(out)
except ValueError as e:
if "sh: " in out:
return json.loads(
re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip())
raise e
return json.loads(out) if json_output else text
@staticmethod
def _call_errback(failure):

8
platformio/commands/home/rpc/server.py
View File

@ -14,6 +14,7 @@
# pylint: disable=import-error
import click
import jsonrpc
from autobahn.twisted.websocket import (WebSocketServerFactory,
WebSocketServerProtocol)
@ -26,7 +27,7 @@ from platformio.compat import PY2, dump_json_to_unicode, is_bytes
class JSONRPCServerProtocol(WebSocketServerProtocol):
def onMessage(self, payload, isBinary): # pylint: disable=unused-argument
# print("> %s" % payload)
# click.echo("> %s" % payload)
response = jsonrpc.JSONRPCResponseManager.handle(
payload, self.factory.dispatcher).data
# if error
@ -52,11 +53,12 @@ class JSONRPCServerProtocol(WebSocketServerProtocol):
message=failure.getErrorMessage())
del response["result"]
response['error'] = e.error._data # pylint: disable=protected-access
print(response['error'])
self.sendJSONResponse(response)
def sendJSONResponse(self, response):
# print("< %s" % response)
# click.echo("< %s" % response)
if "error" in response:
click.secho("Error: %s" % response['error'], fg="red", err=True)
response = dump_json_to_unicode(response)
if not PY2 and not is_bytes(response):
response = response.encode("utf-8")

9
platformio/compat.py
View File

@ -54,9 +54,12 @@ if PY2:
return data
def dump_json_to_unicode(obj):
if isinstance(obj, unicode):
return obj
return json.dumps(obj,
encoding=get_filesystem_encoding(),
ensure_ascii=False).encode("utf8")
ensure_ascii=False,
sort_keys=True).encode("utf8")
_magic_check = re.compile('([*?[])')
_magic_check_bytes = re.compile(b'([*?[])')
@ -100,4 +103,6 @@ else:
return data.encode()
def dump_json_to_unicode(obj):
return json.dumps(obj, ensure_ascii=False)
if isinstance(obj, string_types):
return obj
return json.dumps(obj, ensure_ascii=False, sort_keys=True)

12
platformio/util.py
View File

@ -20,6 +20,7 @@ import socket
import stat
import sys
import time
from contextlib import contextmanager
from functools import wraps
from glob import glob
from os.path import abspath, basename, dirname, isfile, join
@ -107,6 +108,17 @@ def singleton(cls):
return get_instance
@contextmanager
def capture_std_streams(stdout, stderr=None):
_stdout = sys.stdout
_stderr = sys.stderr
sys.stdout = stdout
sys.stderr = stderr or stdout
yield
sys.stdout = _stdout
sys.stderr = _stderr
def load_json(file_path):
try:
with open(file_path, "r") as f: