feedc0de/platformio-core
1
0
Fork 0
forked from platformio/platformio-core
Code Pull Requests Activity

Fix numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars // Resolve #143, Resolve #1342, Resolve #1959, Resolve #2100

Browse Source
This commit is contained in:
Ivan Kravets
2019-06-05 17:57:22 +03:00
parent 84ce7db3e3
commit 394d272324
14 changed files with 103 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
platformio/commands/home/rpc/handlers/app.py
View File

@@ -14,11 +14,9 @@
from __future__ import absolute_import
import json
from os.path import expanduser, isfile, join
from os.path import expanduser, join
from platformio import __version__, app, exception, util
from platformio.compat import path_to_unicode
from platformio import __version__, app, util
from platformio.project.helpers import (get_project_core_dir,
is_platformio_project)
@@ -29,57 +27,45 @@ class AppRPC(object):
@staticmethod
def load_state():
state = None
try:
if isfile(AppRPC.APPSTATE_PATH):
state = util.load_json(AppRPC.APPSTATE_PATH)
except exception.PlatformioException:
pass
if not isinstance(state, dict):
state = {}
storage = state.get("storage", {})
with app.State(AppRPC.APPSTATE_PATH, lock=True) as state:
storage = state.get("storage", {})
# base data
caller_id = app.get_session_var("caller_id")
storage['cid'] = app.get_cid()
storage['coreVersion'] = __version__
storage['coreSystype'] = util.get_systype()
storage['coreCaller'] = (str(caller_id).lower() if caller_id else None)
storage['coreSettings'] = {
name: {
"description": data['description'],
"default_value": data['value'],
"value": app.get_setting(name)
# base data
caller_id = app.get_session_var("caller_id")
storage['cid'] = app.get_cid()
storage['coreVersion'] = __version__
storage['coreSystype'] = util.get_systype()
storage['coreCaller'] = (str(caller_id).lower()
if caller_id else None)
storage['coreSettings'] = {
name: {
"description": data['description'],
"default_value": data['value'],
"value": app.get_setting(name)
}
for name, data in app.DEFAULT_SETTINGS.items()
}
for name, data in app.DEFAULT_SETTINGS.items()
}
# encode to UTF-8
for key in storage['coreSettings']:
if not key.endswith("dir"):
continue
storage['coreSettings'][key]['default_value'] = path_to_unicode(
storage['coreSettings'][key]['default_value'])
storage['coreSettings'][key]['value'] = path_to_unicode(
storage['coreSettings'][key]['value'])
storage['homeDir'] = path_to_unicode(expanduser("~"))
storage['projectsDir'] = storage['coreSettings']['projects_dir'][
'value']
storage['homeDir'] = expanduser("~")
storage['projectsDir'] = storage['coreSettings']['projects_dir'][
'value']
# skip non-existing recent projects
storage['recentProjects'] = [
p for p in storage.get("recentProjects", [])
if is_platformio_project(p)
]
# skip non-existing recent projects
storage['recentProjects'] = [
p for p in storage.get("recentProjects", [])
if is_platformio_project(p)
]
state['storage'] = storage
return state
state['storage'] = storage
return state
@staticmethod
def get_state():
return AppRPC.load_state()
def save_state(self, state):
with open(self.APPSTATE_PATH, "w") as fp:
json.dump(state, fp)
@staticmethod
def save_state(state):
with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
s.clear()
s.update(state)
return True

12
platformio/commands/home/rpc/handlers/misc.py
View File

@@ -27,7 +27,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC
class MiscRPC(object):
def load_latest_tweets(self, username):
cache_key = "piohome_latest_tweets_%s" % username
cache_key = "piohome_latest_tweets_" + str(username)
cache_valid = "7d"
with app.ContentCache() as cc:
cache_data = cc.get(cache_key)
@@ -60,13 +60,11 @@ class MiscRPC(object):
"include_new_items_bar=true") % username
if helpers.is_twitter_blocked():
api_url = self._get_proxed_uri(api_url)
html_or_json = yield OSRPC.fetch_content(
content = yield OSRPC.fetch_content(
api_url, headers=self._get_twitter_headers(username))
# issue with PIO Core < 3.5.3 and ContentCache
if not isinstance(html_or_json, dict):
html_or_json = json.loads(html_or_json)
assert "items_html" in html_or_json
soup = BeautifulSoup(html_or_json['items_html'], "html.parser")
content = json.loads(content)
assert "items_html" in content
soup = BeautifulSoup(content['items_html'], "html.parser")
tweet_nodes = soup.find_all("div",
attrs={
"class": "tweet",

4
platformio/commands/home/rpc/handlers/os.py
View File

@@ -25,7 +25,7 @@ from twisted.internet import defer # pylint: disable=import-error
from platformio import app, util
from platformio.commands.home import helpers
from platformio.compat import PY2, get_filesystem_encoding, path_to_unicode
from platformio.compat import PY2, get_filesystem_encoding
class OSRPC(object):
@@ -150,6 +150,6 @@ class OSRPC(object):
items = []
for item in util.get_logical_devices():
if item['name']:
item['name'] = path_to_unicode(item['name'])
item['name'] = item['name']
items.append(item)
return items

12
platformio/commands/home/rpc/handlers/piocore.py
View File

@@ -23,7 +23,7 @@ from twisted.internet import utils # pylint: disable=import-error
from platformio import __version__
from platformio.commands.home import helpers
from platformio.compat import get_filesystem_encoding, string_types
from platformio.compat import string_types
class PIOCoreRPC(object):
@@ -33,8 +33,8 @@ class PIOCoreRPC(object):
json_output = "--json-output" in args
try:
args = [
arg.encode(get_filesystem_encoding()) if isinstance(
arg, string_types) else str(arg) for arg in args
str(arg) if not isinstance(arg, string_types) else arg
for arg in args
]
except UnicodeError:
raise jsonrpc.exceptions.JSONRPCDispatchException(
@@ -51,18 +51,12 @@ class PIOCoreRPC(object):
@staticmethod
def _call_callback(result, json_output=False):
result = list(result)
assert len(result) == 3
for i in (0, 1):
result[i] = result[i].decode(get_filesystem_encoding()).strip()
out, err, code = result
text = ("%s\n\n%s" % (out, err)).strip()
if code != 0:
raise Exception(text)
if not json_output:
return text
try:
return json.loads(out)
except ValueError as e:

15
platformio/commands/home/rpc/handlers/project.py
View File

@@ -25,7 +25,7 @@ import jsonrpc # pylint: disable=import-error
from platformio import exception, util
from platformio.commands.home.rpc.handlers.app import AppRPC
from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
from platformio.compat import get_filesystem_encoding
from platformio.compat import PY2, get_filesystem_encoding
from platformio.ide.projectgenerator import ProjectGenerator
from platformio.managers.platform import PlatformManager
from platformio.project.config import ProjectConfig
@@ -172,6 +172,10 @@ class ProjectRPC(object):
return project_dir
def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
board = str(board)
if arduino_project_dir and PY2:
arduino_project_dir = arduino_project_dir.encode(
get_filesystem_encoding())
# don't import PIO Project
if is_platformio_project(arduino_project_dir):
return arduino_project_dir
@@ -188,7 +192,7 @@ class ProjectRPC(object):
message="Not an Arduino project: %s" % arduino_project_dir)
state = AppRPC.load_state()
project_dir = join(state['storage']['projectsDir'].decode("utf-8"),
project_dir = join(state['storage']['projectsDir'],
time.strftime("%y%m%d-%H%M%S-") + board)
if not isdir(project_dir):
os.makedirs(project_dir)
@@ -213,8 +217,7 @@ class ProjectRPC(object):
src_dir = get_project_src_dir()
if isdir(src_dir):
util.rmtree_(src_dir)
shutil.copytree(
arduino_project_dir.encode(get_filesystem_encoding()), src_dir)
shutil.copytree(arduino_project_dir, src_dir)
return project_dir
@staticmethod
@@ -255,12 +258,14 @@ class ProjectRPC(object):
@staticmethod
def import_pio(project_dir):
if project_dir and PY2:
project_dir = project_dir.encode(get_filesystem_encoding())
if not project_dir or not is_platformio_project(project_dir):
raise jsonrpc.exceptions.JSONRPCDispatchException(
code=4001,
message="Not an PlatformIO project: %s" % project_dir)
new_project_dir = join(
AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"),
AppRPC.load_state()['storage']['projectsDir'],
time.strftime("%y%m%d-%H%M%S-") + basename(project_dir))
shutil.copytree(project_dir, new_project_dir)

9
platformio/commands/home/rpc/server.py
View File

@@ -14,14 +14,14 @@
# pylint: disable=import-error
import json
import jsonrpc
from autobahn.twisted.websocket import (WebSocketServerFactory,
WebSocketServerProtocol)
from jsonrpc.exceptions import JSONRPCDispatchException
from twisted.internet import defer
from platformio.compat import PY2, dump_json_to_unicode, is_bytes
class JSONRPCServerProtocol(WebSocketServerProtocol):
@@ -57,7 +57,10 @@ class JSONRPCServerProtocol(WebSocketServerProtocol):
def sendJSONResponse(self, response):
# print("< %s" % response)
self.sendMessage(json.dumps(response).encode("utf8"))
response = dump_json_to_unicode(response)
if not PY2 and not is_bytes(response):
response = response.encode("utf-8")
self.sendMessage(response)
class JSONRPCServerFactory(WebSocketServerFactory):