Fix numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars // Resolve #2100

HISTORY.rst

@@ -42,6 +42,7 @@ PlatformIO 4.0
   - Added support for the latest Python "Click" package (CLI) (`issue #349 <https://github.com/platformio/platformio-core/issues/349>`_)
   - Added options to override default locations used by PlatformIO Core (`core_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#core-dir>`__, `globallib_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#globallib-dir>`__, `platforms_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#platforms-dir>`__, `packages_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#packages-dir>`__, `cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#cache-dir>`__) (`issue #1615 <https://github.com/platformio/platformio-core/issues/1615>`_)
   - Removed line-buffering from `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__ command which was leading to omitting progress bar from upload tools (`issue #856 <https://github.com/platformio/platformio-core/issues/856>`_)
+  - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #2100 <https://github.com/platformio/platformio-core/issues/2100>`_)
 
 * **Integration**
 

platformio/builder/main.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-import sys
 from os import environ
 from os.path import join
 from time import time
@@ -29,7 +27,7 @@ from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
 from platformio import util
-from platformio.compat import PY2, path_to_unicode
+from platformio.compat import PY2, dump_json_to_unicode
 from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
 from platformio.project import helpers as project_helpers
@@ -151,17 +149,8 @@ if "envdump" in COMMAND_LINE_TARGETS:
     env.Exit(0)
 
 if "idedata" in COMMAND_LINE_TARGETS:
-    try:
+    Import("projenv")
-        Import("projenv")
+    print("\n%s\n" % dump_json_to_unicode(
-        print("\n%s\n" % path_to_unicode(
+        env.DumpIDEData(projenv)  # pylint: disable=undefined-variable
-            json.dumps(
+    ))
-                env.DumpIDEData(projenv),  # pylint: disable=undefined-variable
+    env.Exit(0)
-                ensure_ascii=False)))
-        env.Exit(0)
-    except UnicodeDecodeError:
-        sys.stderr.write(
-            "\nUnicodeDecodeError: Non-ASCII characters found in build "
-            "environment\n"
-            "See explanation in FAQ > Troubleshooting > Building\n"
-            "https://docs.platformio.org/page/faq.html\n\n")
-        env.Exit(1)

platformio/compat.py

@@ -14,6 +14,7 @@
 
 # pylint: disable=unused-import
 
+import json
 import os
 import re
 import sys
@@ -50,6 +51,11 @@ if PY2:
             data = str(data)
         return data
 
+    def dump_json_to_unicode(obj):
+        return json.dumps(obj,
+                          encoding=get_filesystem_encoding(),
+                          ensure_ascii=False).encode("utf8")
+
     _magic_check = re.compile('([*?[])')
     _magic_check_bytes = re.compile(b'([*?[])')
 
@@ -90,3 +96,6 @@ else:
         if not isinstance(data, string_types):
             data = str(data)
         return data.encode()
+
+    def dump_json_to_unicode(obj):
+        return json.dumps(obj, ensure_ascii=False)

platformio/ide/projectgenerator.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import json
 import os
 import re
@@ -23,7 +24,7 @@ from click.testing import CliRunner
 
 from platformio import exception, util
 from platformio.commands.run import cli as cmd_run
-from platformio.compat import PY2, WINDOWS, get_file_contents
+from platformio.compat import WINDOWS, get_file_contents
 from platformio.proc import where_is_program
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (get_project_lib_dir,
@@ -35,7 +36,7 @@ class ProjectGenerator(object):
 
     def __init__(self, project_dir, ide, env_name):
         self.project_dir = project_dir
-        self.ide = ide
+        self.ide = str(ide)
         self.env_name = env_name
 
         self._tplvars = {}
@@ -121,8 +122,7 @@ class ProjectGenerator(object):
 
             file_name = basename(tpl_path)[:-4]
             contents = self._render_tpl(tpl_path)
-            self._merge_contents(join(dst_dir, file_name),
+            self._merge_contents(join(dst_dir, file_name), contents)
-                                 contents.encode("utf8") if PY2 else contents)
 
     def _render_tpl(self, tpl_path):
         return bottle.template(get_file_contents(tpl_path), **self._tplvars)
@@ -131,8 +131,8 @@ class ProjectGenerator(object):
     def _merge_contents(dst_path, contents):
         if basename(dst_path) == ".gitignore" and isfile(dst_path):
             return
-        with open(dst_path, "w") as f:
+        with codecs.open(dst_path, "w", encoding="utf8") as fp:
-            f.write(contents)
+            fp.write(contents)
 
     def _gather_tplvars(self):
         self._tplvars.update(self.get_project_env())

platformio/proc.py

@@ -19,7 +19,7 @@ from os.path import isdir, isfile, join, normpath
 from threading import Thread
 
 from platformio import exception
-from platformio.compat import WINDOWS, get_filesystem_encoding, string_types
+from platformio.compat import WINDOWS, string_types
 
 
 class AsyncPipeBase(object):
@@ -124,7 +124,10 @@ def exec_command(*args, **kwargs):
 
     for k, v in result.items():
         if isinstance(result[k], bytes):
-            result[k] = result[k].decode(get_filesystem_encoding())
+            try:
+                result[k] = result[k].decode(sys.getdefaultencoding())
+            except UnicodeDecodeError:
+                result[k] = result[k].decode("latin-1")
         if v and isinstance(v, string_types):
             result[k] = result[k].strip()