platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-29 17:47:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars // Resolve #143, Resolve #1342, Resolve #1959, Resolve #2100

Browse Source
This commit is contained in:
Ivan Kravets
2019-06-05 17:57:22 +03:00
parent 84ce7db3e3
commit 394d272324
14 changed files with 103 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
HISTORY.rst
View File

@ -43,7 +43,7 @@ PlatformIO 4.0
- Added support for the latest Python "Click" package (CLI) (`issue #349 <https://github.com/platformio/platformio-core/issues/349>`_)
- Added options to override default locations used by PlatformIO Core (`core_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#core-dir>`__, `globallib_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#globallib-dir>`__, `platforms_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#platforms-dir>`__, `packages_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#packages-dir>`__, `cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#cache-dir>`__) (`issue #1615 <https://github.com/platformio/platformio-core/issues/1615>`_)
- Removed line-buffering from `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__ command which was leading to omitting progress bar from upload tools (`issue #856 <https://github.com/platformio/platformio-core/issues/856>`_)
- Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #2100 <https://github.com/platformio/platformio-core/issues/2100>`_)
- Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 <https://github.com/platformio/platformio-core/issues/143>`_, `issue #1342 <https://github.com/platformio/platformio-core/issues/1342>`_, `issue #1959 <https://github.com/platformio/platformio-core/issues/1959>`_, `issue #2100 <https://github.com/platformio/platformio-core/issues/2100>`_)
* **Integration**

16
platformio/app.py
View File

@ -14,7 +14,6 @@
import codecs
import hashlib
import json
import os
import uuid
from copy import deepcopy
@ -25,7 +24,8 @@ from time import time
import requests
from platformio import exception, lockfile, util
from platformio.compat import WINDOWS, hashlib_encode_data
from platformio.compat import (WINDOWS, dump_json_to_unicode,
hashlib_encode_data)
from platformio.proc import is_ci
from platformio.project.helpers import (get_project_cache_dir,
get_project_core_dir)
@ -102,16 +102,19 @@ class State(object):
self._lock_state_file()
if isfile(self.path):
self._state = util.load_json(self.path)
except exception.PlatformioException:
assert isinstance(self._state, dict)
except (AssertionError, UnicodeDecodeError,
exception.PlatformioException):
self._state = {}
self._prev_state = deepcopy(self._state)
return self._state
def __exit__(self, type_, value, traceback):
if self._prev_state != self._state:
new_state = dump_json_to_unicode(self._state)
if self._prev_state != new_state:
try:
with codecs.open(self.path, "w", encoding="utf8") as fp:
json.dump(self._state, fp)
with open(self.path, "w") as fp:
fp.write(new_state)
except IOError:
raise exception.HomeDirPermissionsError(get_project_core_dir())
self._unlock_state_file()
@ -167,6 +170,7 @@ class ContentCache(object):
return True
def get_cache_path(self, key):
key = str(key)
assert len(key) > 3
return join(self.cache_dir, key[-2:], key)

3
platformio/commands/boards.py
View File

@ -17,6 +17,7 @@ import json
import click
from platformio import util
from platformio.compat import dump_json_to_unicode
from platformio.managers.platform import PlatformManager
@ -82,4 +83,4 @@ def _print_boards_json(query, installed=False):
if query.lower() not in search_data.lower():
continue
result.append(board)
click.echo(json.dumps(result))
click.echo(dump_json_to_unicode(result))

5
platformio/commands/device.py
View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import sys
from fnmatch import fnmatch
from os import getcwd
@ -22,6 +21,7 @@ import click
from serial.tools import miniterm
from platformio import exception, util
from platformio.compat import dump_json_to_unicode
from platformio.project.config import ProjectConfig
@ -50,7 +50,8 @@ def device_list( # pylint: disable=too-many-branches
single_key = list(data)[0] if len(list(data)) == 1 else None
if json_output:
return click.echo(json.dumps(data[single_key] if single_key else data))
return click.echo(
dump_json_to_unicode(data[single_key] if single_key else data))
titles = {
"serial": "Serial Ports",

80
platformio/commands/home/rpc/handlers/app.py
View File

@ -14,11 +14,9 @@
from __future__ import absolute_import
import json
from os.path import expanduser, isfile, join
from os.path import expanduser, join
from platformio import __version__, app, exception, util
from platformio.compat import path_to_unicode
from platformio import __version__, app, util
from platformio.project.helpers import (get_project_core_dir,
is_platformio_project)
@ -29,57 +27,45 @@ class AppRPC(object):
@staticmethod
def load_state():
state = None
try:
if isfile(AppRPC.APPSTATE_PATH):
state = util.load_json(AppRPC.APPSTATE_PATH)
except exception.PlatformioException:
pass
if not isinstance(state, dict):
state = {}
storage = state.get("storage", {})
with app.State(AppRPC.APPSTATE_PATH, lock=True) as state:
storage = state.get("storage", {})
# base data
caller_id = app.get_session_var("caller_id")
storage['cid'] = app.get_cid()
storage['coreVersion'] = __version__
storage['coreSystype'] = util.get_systype()
storage['coreCaller'] = (str(caller_id).lower() if caller_id else None)
storage['coreSettings'] = {
name: {
"description": data['description'],
"default_value": data['value'],
"value": app.get_setting(name)
# base data
caller_id = app.get_session_var("caller_id")
storage['cid'] = app.get_cid()
storage['coreVersion'] = __version__
storage['coreSystype'] = util.get_systype()
storage['coreCaller'] = (str(caller_id).lower()
if caller_id else None)
storage['coreSettings'] = {
name: {
"description": data['description'],
"default_value": data['value'],
"value": app.get_setting(name)
}
for name, data in app.DEFAULT_SETTINGS.items()
}
for name, data in app.DEFAULT_SETTINGS.items()
}
# encode to UTF-8
for key in storage['coreSettings']:
if not key.endswith("dir"):
continue
storage['coreSettings'][key]['default_value'] = path_to_unicode(
storage['coreSettings'][key]['default_value'])
storage['coreSettings'][key]['value'] = path_to_unicode(
storage['coreSettings'][key]['value'])
storage['homeDir'] = path_to_unicode(expanduser("~"))
storage['projectsDir'] = storage['coreSettings']['projects_dir'][
'value']
storage['homeDir'] = expanduser("~")
storage['projectsDir'] = storage['coreSettings']['projects_dir'][
'value']
# skip non-existing recent projects
storage['recentProjects'] = [
p for p in storage.get("recentProjects", [])
if is_platformio_project(p)
]
# skip non-existing recent projects
storage['recentProjects'] = [
p for p in storage.get("recentProjects", [])
if is_platformio_project(p)
]
state['storage'] = storage
return state
state['storage'] = storage
return state
@staticmethod
def get_state():
return AppRPC.load_state()
def save_state(self, state):
with open(self.APPSTATE_PATH, "w") as fp:
json.dump(state, fp)
@staticmethod
def save_state(state):
with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
s.clear()
s.update(state)
return True

12
platformio/commands/home/rpc/handlers/misc.py
View File

@ -27,7 +27,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC
class MiscRPC(object):
def load_latest_tweets(self, username):
cache_key = "piohome_latest_tweets_%s" % username
cache_key = "piohome_latest_tweets_" + str(username)
cache_valid = "7d"
with app.ContentCache() as cc:
cache_data = cc.get(cache_key)
@ -60,13 +60,11 @@ class MiscRPC(object):
"include_new_items_bar=true") % username
if helpers.is_twitter_blocked():
api_url = self._get_proxed_uri(api_url)
html_or_json = yield OSRPC.fetch_content(
content = yield OSRPC.fetch_content(
api_url, headers=self._get_twitter_headers(username))
# issue with PIO Core < 3.5.3 and ContentCache
if not isinstance(html_or_json, dict):
html_or_json = json.loads(html_or_json)
assert "items_html" in html_or_json
soup = BeautifulSoup(html_or_json['items_html'], "html.parser")
content = json.loads(content)
assert "items_html" in content
soup = BeautifulSoup(content['items_html'], "html.parser")
tweet_nodes = soup.find_all("div",
attrs={
"class": "tweet",

4
platformio/commands/home/rpc/handlers/os.py
View File

@ -25,7 +25,7 @@ from twisted.internet import defer # pylint: disable=import-error
from platformio import app, util
from platformio.commands.home import helpers
from platformio.compat import PY2, get_filesystem_encoding, path_to_unicode
from platformio.compat import PY2, get_filesystem_encoding
class OSRPC(object):
@ -150,6 +150,6 @@ class OSRPC(object):
items = []
for item in util.get_logical_devices():
if item['name']:
item['name'] = path_to_unicode(item['name'])
item['name'] = item['name']
items.append(item)
return items

12
platformio/commands/home/rpc/handlers/piocore.py
View File

@ -23,7 +23,7 @@ from twisted.internet import utils # pylint: disable=import-error
from platformio import __version__
from platformio.commands.home import helpers
from platformio.compat import get_filesystem_encoding, string_types
from platformio.compat import string_types
class PIOCoreRPC(object):
@ -33,8 +33,8 @@ class PIOCoreRPC(object):
json_output = "--json-output" in args
try:
args = [
arg.encode(get_filesystem_encoding()) if isinstance(
arg, string_types) else str(arg) for arg in args
str(arg) if not isinstance(arg, string_types) else arg
for arg in args
]
except UnicodeError:
raise jsonrpc.exceptions.JSONRPCDispatchException(
@ -51,18 +51,12 @@ class PIOCoreRPC(object):
@staticmethod
def _call_callback(result, json_output=False):
result = list(result)
assert len(result) == 3
for i in (0, 1):
result[i] = result[i].decode(get_filesystem_encoding()).strip()
out, err, code = result
text = ("%s\n\n%s" % (out, err)).strip()
if code != 0:
raise Exception(text)
if not json_output:
return text
try:
return json.loads(out)
except ValueError as e:

15
platformio/commands/home/rpc/handlers/project.py
View File

@ -25,7 +25,7 @@ import jsonrpc # pylint: disable=import-error
from platformio import exception, util
from platformio.commands.home.rpc.handlers.app import AppRPC
from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
from platformio.compat import get_filesystem_encoding
from platformio.compat import PY2, get_filesystem_encoding
from platformio.ide.projectgenerator import ProjectGenerator
from platformio.managers.platform import PlatformManager
from platformio.project.config import ProjectConfig
@ -172,6 +172,10 @@ class ProjectRPC(object):
return project_dir
def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
board = str(board)
if arduino_project_dir and PY2:
arduino_project_dir = arduino_project_dir.encode(
get_filesystem_encoding())
# don't import PIO Project
if is_platformio_project(arduino_project_dir):
return arduino_project_dir
@ -188,7 +192,7 @@ class ProjectRPC(object):
message="Not an Arduino project: %s" % arduino_project_dir)
state = AppRPC.load_state()
project_dir = join(state['storage']['projectsDir'].decode("utf-8"),
project_dir = join(state['storage']['projectsDir'],
time.strftime("%y%m%d-%H%M%S-") + board)
if not isdir(project_dir):
os.makedirs(project_dir)
@ -213,8 +217,7 @@ class ProjectRPC(object):
src_dir = get_project_src_dir()
if isdir(src_dir):
util.rmtree_(src_dir)
shutil.copytree(
arduino_project_dir.encode(get_filesystem_encoding()), src_dir)
shutil.copytree(arduino_project_dir, src_dir)
return project_dir
@staticmethod
@ -255,12 +258,14 @@ class ProjectRPC(object):
@staticmethod
def import_pio(project_dir):
if project_dir and PY2:
project_dir = project_dir.encode(get_filesystem_encoding())
if not project_dir or not is_platformio_project(project_dir):
raise jsonrpc.exceptions.JSONRPCDispatchException(
code=4001,
message="Not an PlatformIO project: %s" % project_dir)
new_project_dir = join(
AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"),
AppRPC.load_state()['storage']['projectsDir'],
time.strftime("%y%m%d-%H%M%S-") + basename(project_dir))
shutil.copytree(project_dir, new_project_dir)

9
platformio/commands/home/rpc/server.py
View File

@ -14,14 +14,14 @@
# pylint: disable=import-error
import json
import jsonrpc
from autobahn.twisted.websocket import (WebSocketServerFactory,
WebSocketServerProtocol)
from jsonrpc.exceptions import JSONRPCDispatchException
from twisted.internet import defer
from platformio.compat import PY2, dump_json_to_unicode, is_bytes
class JSONRPCServerProtocol(WebSocketServerProtocol):
@ -57,7 +57,10 @@ class JSONRPCServerProtocol(WebSocketServerProtocol):
def sendJSONResponse(self, response):
# print("< %s" % response)
self.sendMessage(json.dumps(response).encode("utf8"))
response = dump_json_to_unicode(response)
if not PY2 and not is_bytes(response):
response = response.encode("utf-8")
self.sendMessage(response)
class JSONRPCServerFactory(WebSocketServerFactory):

18
platformio/commands/lib.py
View File

@ -14,7 +14,6 @@
# pylint: disable=too-many-branches, too-many-locals
import json
import time
from os.path import isdir, join
@ -22,6 +21,7 @@ import click
import semantic_version
from platformio import exception, util
from platformio.compat import dump_json_to_unicode
from platformio.managers.lib import (LibraryManager, get_builtin_libs,
is_builtin_lib)
from platformio.proc import is_ci
@ -247,8 +247,8 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output):
if json_output:
return click.echo(
json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) ==
1 else json_result))
dump_json_to_unicode(json_result[storage_dirs[0]]
if len(storage_dirs) == 1 else json_result))
return True
@ -274,8 +274,8 @@ def lib_list(ctx, json_output):
if json_output:
return click.echo(
json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) ==
1 else json_result))
dump_json_to_unicode(json_result[storage_dirs[0]]
if len(storage_dirs) == 1 else json_result))
return True
@ -309,7 +309,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
cache_valid="1d")
if json_output:
click.echo(json.dumps(result))
click.echo(dump_json_to_unicode(result))
return
if result['total'] == 0:
@ -361,7 +361,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
def lib_builtin(storage, json_output):
items = get_builtin_libs(storage)
if json_output:
return click.echo(json.dumps(items))
return click.echo(dump_json_to_unicode(items))
for storage_ in items:
if not storage_['items']:
@ -390,7 +390,7 @@ def lib_show(library, json_output):
interactive=not json_output)
lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
if json_output:
return click.echo(json.dumps(lib))
return click.echo(dump_json_to_unicode(lib))
click.secho(lib['name'], fg="cyan")
click.echo("=" * len(lib['name']))
@ -478,7 +478,7 @@ def lib_stats(json_output):
result = util.get_api_result("/lib/stats", cache_valid="1h")
if json_output:
return click.echo(json.dumps(result))
return click.echo(dump_json_to_unicode(result))
printitem_tpl = "{name:<33} {url}"
printitemdate_tpl = "{name:<33} {date:23} {url}"

16
platformio/commands/platform.py
View File

@ -12,13 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from os.path import dirname, isdir
import click
from platformio import app, exception, util
from platformio.commands.boards import print_boards
from platformio.compat import dump_json_to_unicode
from platformio.managers.platform import PlatformFactory, PlatformManager
@ -156,7 +156,7 @@ def platform_search(query, json_output):
for platform in _get_registry_platforms():
if query == "all":
query = ""
search_data = json.dumps(platform)
search_data = dump_json_to_unicode(platform)
if query and query.lower() not in search_data.lower():
continue
platforms.append(
@ -165,7 +165,7 @@ def platform_search(query, json_output):
expose_packages=False))
if json_output:
click.echo(json.dumps(platforms))
click.echo(dump_json_to_unicode(platforms))
else:
_print_platforms(platforms)
@ -178,7 +178,7 @@ def platform_frameworks(query, json_output):
for framework in util.get_api_result("/frameworks", cache_valid="7d"):
if query == "all":
query = ""
search_data = json.dumps(framework)
search_data = dump_json_to_unicode(framework)
if query and query.lower() not in search_data.lower():
continue
framework['homepage'] = ("https://platformio.org/frameworks/" +
@ -191,7 +191,7 @@ def platform_frameworks(query, json_output):
frameworks = sorted(frameworks, key=lambda manifest: manifest['name'])
if json_output:
click.echo(json.dumps(frameworks))
click.echo(dump_json_to_unicode(frameworks))
else:
_print_platforms(frameworks)
@ -209,7 +209,7 @@ def platform_list(json_output):
platforms = sorted(platforms, key=lambda manifest: manifest['name'])
if json_output:
click.echo(json.dumps(platforms))
click.echo(dump_json_to_unicode(platforms))
else:
_print_platforms(platforms)
@ -222,7 +222,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches
if not data:
raise exception.UnknownPlatform(platform)
if json_output:
return click.echo(json.dumps(data))
return click.echo(dump_json_to_unicode(data))
click.echo("{name} ~ {title}".format(name=click.style(data['name'],
fg="cyan"),
@ -361,7 +361,7 @@ def platform_update( # pylint: disable=too-many-locals
if latest:
data['versionLatest'] = latest
result.append(data)
return click.echo(json.dumps(result))
return click.echo(dump_json_to_unicode(result))
# cleanup cached board and platform lists
app.clean_cache()

2
platformio/commands/run/processor.py
View File

@ -37,7 +37,7 @@ class EnvironmentProcessor(object):
self.cmd_ctx = cmd_ctx
self.name = name
self.config = config
self.targets = targets
self.targets = [str(t) for t in targets]
self.upload_port = upload_port
self.silent = silent
self.verbose = verbose

11
platformio/managers/platform.py
View File

@ -24,7 +24,7 @@ import click
import semantic_version
from platformio import __version__, app, exception, util
from platformio.compat import hashlib_encode_data, is_bytes
from platformio.compat import PY2, hashlib_encode_data, is_bytes
from platformio.managers.core import get_core_package_dir
from platformio.managers.package import BasePkgManager, PackageManager
from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv,
@ -693,6 +693,15 @@ class PlatformBoardConfig(object):
value = self._manifest
for k in path.split("."):
value = value[k]
# pylint: disable=undefined-variable
if PY2 and isinstance(value, unicode):
# cast to plain string from unicode for PY2, resolves issue in
# dev/platform when BoardConfig.get() is used in pair with
# os.path.join(file_encoding, unicode_encoding)
try:
value = value.encode("utf-8")
except UnicodeEncodeError:
pass
return value
except KeyError:
if default is not None: