diff --git a/HISTORY.rst b/HISTORY.rst index 621b6e52..810f89c5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,7 @@ PlatformIO 4.0 - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) + - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #2100 `_) * **Integration** diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 1fd76b24..70f71ad6 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import sys from os import environ from os.path import join from time import time @@ -29,7 +27,7 @@ from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error from platformio import util -from platformio.compat import PY2, path_to_unicode +from platformio.compat import PY2, dump_json_to_unicode from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project import helpers as project_helpers @@ -151,17 +149,8 @@ if "envdump" in COMMAND_LINE_TARGETS: env.Exit(0) if "idedata" in COMMAND_LINE_TARGETS: - try: - Import("projenv") - print("\n%s\n" % path_to_unicode( - json.dumps( - env.DumpIDEData(projenv), # pylint: disable=undefined-variable - ensure_ascii=False))) - env.Exit(0) - except UnicodeDecodeError: - sys.stderr.write( - "\nUnicodeDecodeError: Non-ASCII characters found in build " - "environment\n" - "See explanation in FAQ > Troubleshooting > Building\n" - "https://docs.platformio.org/page/faq.html\n\n") - env.Exit(1) + Import("projenv") + print("\n%s\n" % dump_json_to_unicode( + env.DumpIDEData(projenv) # pylint: disable=undefined-variable + )) + env.Exit(0) diff --git a/platformio/compat.py b/platformio/compat.py index 975400fe..53ba736a 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -14,6 +14,7 @@ # pylint: disable=unused-import +import json import os import re import sys @@ -50,6 +51,11 @@ if PY2: data = str(data) return data + def dump_json_to_unicode(obj): + return json.dumps(obj, + encoding=get_filesystem_encoding(), + ensure_ascii=False).encode("utf8") + _magic_check = re.compile('([*?[])') _magic_check_bytes = re.compile(b'([*?[])') @@ -90,3 +96,6 @@ else: if not isinstance(data, string_types): data = str(data) return data.encode() + + def dump_json_to_unicode(obj): + return json.dumps(obj, ensure_ascii=False) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index e420802a..eb3ce6ec 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import codecs import json import os import re @@ -23,7 +24,7 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run -from platformio.compat import PY2, WINDOWS, get_file_contents +from platformio.compat import WINDOWS, get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig from platformio.project.helpers import (get_project_lib_dir, @@ -35,7 +36,7 @@ class ProjectGenerator(object): def __init__(self, project_dir, ide, env_name): self.project_dir = project_dir - self.ide = ide + self.ide = str(ide) self.env_name = env_name self._tplvars = {} @@ -121,8 +122,7 @@ class ProjectGenerator(object): file_name = basename(tpl_path)[:-4] contents = self._render_tpl(tpl_path) - self._merge_contents(join(dst_dir, file_name), - contents.encode("utf8") if PY2 else contents) + self._merge_contents(join(dst_dir, file_name), contents) def _render_tpl(self, tpl_path): return bottle.template(get_file_contents(tpl_path), **self._tplvars) @@ -131,8 +131,8 @@ class ProjectGenerator(object): def _merge_contents(dst_path, contents): if basename(dst_path) == ".gitignore" and isfile(dst_path): return - with open(dst_path, "w") as f: - f.write(contents) + with codecs.open(dst_path, "w", encoding="utf8") as fp: + fp.write(contents) def _gather_tplvars(self): self._tplvars.update(self.get_project_env()) diff --git a/platformio/proc.py b/platformio/proc.py index 7808fa42..c90590e3 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -19,7 +19,7 @@ from os.path import isdir, isfile, join, normpath from threading import Thread from platformio import exception -from platformio.compat import WINDOWS, get_filesystem_encoding, string_types +from platformio.compat import WINDOWS, string_types class AsyncPipeBase(object): @@ -124,7 +124,10 @@ def exec_command(*args, **kwargs): for k, v in result.items(): if isinstance(result[k], bytes): - result[k] = result[k].decode(get_filesystem_encoding()) + try: + result[k] = result[k].decode(sys.getdefaultencoding()) + except UnicodeDecodeError: + result[k] = result[k].decode("latin-1") if v and isinstance(v, string_types): result[k] = result[k].strip()