Fix numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars // Resolve #143, Resolve #1342, Resolve #1959, Resolve #2100

HISTORY.rst

@@ -43,7 +43,7 @@ PlatformIO 4.0
   - Added support for the latest Python "Click" package (CLI) (`issue #349 <https://github.com/platformio/platformio-core/issues/349>`_)
   - Added options to override default locations used by PlatformIO Core (`core_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#core-dir>`__, `globallib_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#globallib-dir>`__, `platforms_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#platforms-dir>`__, `packages_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#packages-dir>`__, `cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#cache-dir>`__) (`issue #1615 <https://github.com/platformio/platformio-core/issues/1615>`_)
   - Removed line-buffering from `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__ command which was leading to omitting progress bar from upload tools (`issue #856 <https://github.com/platformio/platformio-core/issues/856>`_)
-  - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #2100 <https://github.com/platformio/platformio-core/issues/2100>`_)
+  - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 <https://github.com/platformio/platformio-core/issues/143>`_, `issue #1342 <https://github.com/platformio/platformio-core/issues/1342>`_, `issue #1959 <https://github.com/platformio/platformio-core/issues/1959>`_, `issue #2100 <https://github.com/platformio/platformio-core/issues/2100>`_)
 
 * **Integration**
 

platformio/app.py

@@ -14,7 +14,6 @@
 
 import codecs
 import hashlib
-import json
 import os
 import uuid
 from copy import deepcopy
@@ -25,7 +24,8 @@ from time import time
 import requests
 
 from platformio import exception, lockfile, util
-from platformio.compat import WINDOWS, hashlib_encode_data
+from platformio.compat import (WINDOWS, dump_json_to_unicode,
+                               hashlib_encode_data)
 from platformio.proc import is_ci
 from platformio.project.helpers import (get_project_cache_dir,
                                         get_project_core_dir)
@@ -102,16 +102,19 @@ class State(object):
             self._lock_state_file()
             if isfile(self.path):
                 self._state = util.load_json(self.path)
-        except exception.PlatformioException:
+                assert isinstance(self._state, dict)
+        except (AssertionError, UnicodeDecodeError,
+                exception.PlatformioException):
             self._state = {}
         self._prev_state = deepcopy(self._state)
         return self._state
 
     def __exit__(self, type_, value, traceback):
-        if self._prev_state != self._state:
+        new_state = dump_json_to_unicode(self._state)
+        if self._prev_state != new_state:
             try:
-                with codecs.open(self.path, "w", encoding="utf8") as fp:
+                with open(self.path, "w") as fp:
-                    json.dump(self._state, fp)
+                    fp.write(new_state)
             except IOError:
                 raise exception.HomeDirPermissionsError(get_project_core_dir())
         self._unlock_state_file()
@@ -167,6 +170,7 @@ class ContentCache(object):
         return True
 
     def get_cache_path(self, key):
+        key = str(key)
         assert len(key) > 3
         return join(self.cache_dir, key[-2:], key)
 

platformio/commands/boards.py

@@ -17,6 +17,7 @@ import json
 import click
 
 from platformio import util
+from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformManager
 
 
@@ -82,4 +83,4 @@ def _print_boards_json(query, installed=False):
             if query.lower() not in search_data.lower():
                 continue
         result.append(board)
-    click.echo(json.dumps(result))
+    click.echo(dump_json_to_unicode(result))

platformio/commands/device.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import sys
 from fnmatch import fnmatch
 from os import getcwd
@@ -22,6 +21,7 @@ import click
 from serial.tools import miniterm
 
 from platformio import exception, util
+from platformio.compat import dump_json_to_unicode
 from platformio.project.config import ProjectConfig
 
 
@@ -50,7 +50,8 @@ def device_list(  # pylint: disable=too-many-branches
     single_key = list(data)[0] if len(list(data)) == 1 else None
 
     if json_output:
-        return click.echo(json.dumps(data[single_key] if single_key else data))
+        return click.echo(
+            dump_json_to_unicode(data[single_key] if single_key else data))
 
     titles = {
         "serial": "Serial Ports",

platformio/commands/home/rpc/handlers/app.py

@@ -14,11 +14,9 @@
 
 from __future__ import absolute_import
 
-import json
+from os.path import expanduser, join
-from os.path import expanduser, isfile, join
 
-from platformio import __version__, app, exception, util
+from platformio import __version__, app, util
-from platformio.compat import path_to_unicode
 from platformio.project.helpers import (get_project_core_dir,
                                         is_platformio_project)
 
@@ -29,57 +27,45 @@ class AppRPC(object):
 
     @staticmethod
     def load_state():
-        state = None
+        with app.State(AppRPC.APPSTATE_PATH, lock=True) as state:
-        try:
+            storage = state.get("storage", {})
-            if isfile(AppRPC.APPSTATE_PATH):
-                state = util.load_json(AppRPC.APPSTATE_PATH)
-        except exception.PlatformioException:
-            pass
-        if not isinstance(state, dict):
-            state = {}
-        storage = state.get("storage", {})
 
-        # base data
+            # base data
-        caller_id = app.get_session_var("caller_id")
+            caller_id = app.get_session_var("caller_id")
-        storage['cid'] = app.get_cid()
+            storage['cid'] = app.get_cid()
-        storage['coreVersion'] = __version__
+            storage['coreVersion'] = __version__
-        storage['coreSystype'] = util.get_systype()
+            storage['coreSystype'] = util.get_systype()
-        storage['coreCaller'] = (str(caller_id).lower() if caller_id else None)
+            storage['coreCaller'] = (str(caller_id).lower()
-        storage['coreSettings'] = {
+                                     if caller_id else None)
-            name: {
+            storage['coreSettings'] = {
-                "description": data['description'],
+                name: {
-                "default_value": data['value'],
+                    "description": data['description'],
-                "value": app.get_setting(name)
+                    "default_value": data['value'],
+                    "value": app.get_setting(name)
+                }
+                for name, data in app.DEFAULT_SETTINGS.items()
             }
-            for name, data in app.DEFAULT_SETTINGS.items()
-        }
 
-        # encode to UTF-8
+            storage['homeDir'] = expanduser("~")
-        for key in storage['coreSettings']:
+            storage['projectsDir'] = storage['coreSettings']['projects_dir'][
-            if not key.endswith("dir"):
+                'value']
-                continue
-            storage['coreSettings'][key]['default_value'] = path_to_unicode(
-                storage['coreSettings'][key]['default_value'])
-            storage['coreSettings'][key]['value'] = path_to_unicode(
-                storage['coreSettings'][key]['value'])
-        storage['homeDir'] = path_to_unicode(expanduser("~"))
-        storage['projectsDir'] = storage['coreSettings']['projects_dir'][
-            'value']
 
-        # skip non-existing recent projects
+            # skip non-existing recent projects
-        storage['recentProjects'] = [
+            storage['recentProjects'] = [
-            p for p in storage.get("recentProjects", [])
+                p for p in storage.get("recentProjects", [])
-            if is_platformio_project(p)
+                if is_platformio_project(p)
-        ]
+            ]
 
-        state['storage'] = storage
+            state['storage'] = storage
-        return state
+            return state
 
     @staticmethod
     def get_state():
         return AppRPC.load_state()
 
-    def save_state(self, state):
+    @staticmethod
-        with open(self.APPSTATE_PATH, "w") as fp:
+    def save_state(state):
-            json.dump(state, fp)
+        with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
+            s.clear()
+            s.update(state)
         return True

platformio/commands/home/rpc/handlers/misc.py

@@ -27,7 +27,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC
 class MiscRPC(object):
 
     def load_latest_tweets(self, username):
-        cache_key = "piohome_latest_tweets_%s" % username
+        cache_key = "piohome_latest_tweets_" + str(username)
         cache_valid = "7d"
         with app.ContentCache() as cc:
             cache_data = cc.get(cache_key)
@@ -60,13 +60,11 @@ class MiscRPC(object):
                    "include_new_items_bar=true") % username
         if helpers.is_twitter_blocked():
             api_url = self._get_proxed_uri(api_url)
-        html_or_json = yield OSRPC.fetch_content(
+        content = yield OSRPC.fetch_content(
             api_url, headers=self._get_twitter_headers(username))
-        # issue with PIO Core < 3.5.3 and ContentCache
+        content = json.loads(content)
-        if not isinstance(html_or_json, dict):
+        assert "items_html" in content
-            html_or_json = json.loads(html_or_json)
+        soup = BeautifulSoup(content['items_html'], "html.parser")
-        assert "items_html" in html_or_json
-        soup = BeautifulSoup(html_or_json['items_html'], "html.parser")
         tweet_nodes = soup.find_all("div",
                                     attrs={
                                         "class": "tweet",

platformio/commands/home/rpc/handlers/os.py

@@ -25,7 +25,7 @@ from twisted.internet import defer  # pylint: disable=import-error
 
 from platformio import app, util
 from platformio.commands.home import helpers
-from platformio.compat import PY2, get_filesystem_encoding, path_to_unicode
+from platformio.compat import PY2, get_filesystem_encoding
 
 
 class OSRPC(object):
@@ -150,6 +150,6 @@ class OSRPC(object):
         items = []
         for item in util.get_logical_devices():
             if item['name']:
-                item['name'] = path_to_unicode(item['name'])
+                item['name'] = item['name']
             items.append(item)
         return items

platformio/commands/home/rpc/handlers/piocore.py

@@ -23,7 +23,7 @@ from twisted.internet import utils  # pylint: disable=import-error
 
 from platformio import __version__
 from platformio.commands.home import helpers
-from platformio.compat import get_filesystem_encoding, string_types
+from platformio.compat import string_types
 
 
 class PIOCoreRPC(object):
@@ -33,8 +33,8 @@ class PIOCoreRPC(object):
         json_output = "--json-output" in args
         try:
             args = [
-                arg.encode(get_filesystem_encoding()) if isinstance(
+                str(arg) if not isinstance(arg, string_types) else arg
-                    arg, string_types) else str(arg) for arg in args
+                for arg in args
             ]
         except UnicodeError:
             raise jsonrpc.exceptions.JSONRPCDispatchException(
@@ -51,18 +51,12 @@ class PIOCoreRPC(object):
 
     @staticmethod
     def _call_callback(result, json_output=False):
-        result = list(result)
-        assert len(result) == 3
-        for i in (0, 1):
-            result[i] = result[i].decode(get_filesystem_encoding()).strip()
         out, err, code = result
         text = ("%s\n\n%s" % (out, err)).strip()
         if code != 0:
             raise Exception(text)
-
         if not json_output:
             return text
-
         try:
             return json.loads(out)
         except ValueError as e:

platformio/commands/home/rpc/handlers/project.py

@@ -25,7 +25,7 @@ import jsonrpc  # pylint: disable=import-error
 from platformio import exception, util
 from platformio.commands.home.rpc.handlers.app import AppRPC
 from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
-from platformio.compat import get_filesystem_encoding
+from platformio.compat import PY2, get_filesystem_encoding
 from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
 from platformio.project.config import ProjectConfig
@@ -172,6 +172,10 @@ class ProjectRPC(object):
         return project_dir
 
     def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
+        board = str(board)
+        if arduino_project_dir and PY2:
+            arduino_project_dir = arduino_project_dir.encode(
+                get_filesystem_encoding())
         # don't import PIO Project
         if is_platformio_project(arduino_project_dir):
             return arduino_project_dir
@@ -188,7 +192,7 @@ class ProjectRPC(object):
                 message="Not an Arduino project: %s" % arduino_project_dir)
 
         state = AppRPC.load_state()
-        project_dir = join(state['storage']['projectsDir'].decode("utf-8"),
+        project_dir = join(state['storage']['projectsDir'],
                            time.strftime("%y%m%d-%H%M%S-") + board)
         if not isdir(project_dir):
             os.makedirs(project_dir)
@@ -213,8 +217,7 @@ class ProjectRPC(object):
             src_dir = get_project_src_dir()
             if isdir(src_dir):
                 util.rmtree_(src_dir)
-            shutil.copytree(
+            shutil.copytree(arduino_project_dir, src_dir)
-                arduino_project_dir.encode(get_filesystem_encoding()), src_dir)
         return project_dir
 
     @staticmethod
@@ -255,12 +258,14 @@ class ProjectRPC(object):
 
     @staticmethod
     def import_pio(project_dir):
+        if project_dir and PY2:
+            project_dir = project_dir.encode(get_filesystem_encoding())
         if not project_dir or not is_platformio_project(project_dir):
             raise jsonrpc.exceptions.JSONRPCDispatchException(
                 code=4001,
                 message="Not an PlatformIO project: %s" % project_dir)
         new_project_dir = join(
-            AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"),
+            AppRPC.load_state()['storage']['projectsDir'],
             time.strftime("%y%m%d-%H%M%S-") + basename(project_dir))
         shutil.copytree(project_dir, new_project_dir)
 

platformio/commands/home/rpc/server.py

@@ -14,14 +14,14 @@
 
 # pylint: disable=import-error
 
-import json
-
 import jsonrpc
 from autobahn.twisted.websocket import (WebSocketServerFactory,
                                         WebSocketServerProtocol)
 from jsonrpc.exceptions import JSONRPCDispatchException
 from twisted.internet import defer
 
+from platformio.compat import PY2, dump_json_to_unicode, is_bytes
+
 
 class JSONRPCServerProtocol(WebSocketServerProtocol):
 
@@ -57,7 +57,10 @@ class JSONRPCServerProtocol(WebSocketServerProtocol):
 
     def sendJSONResponse(self, response):
         # print("< %s" % response)
-        self.sendMessage(json.dumps(response).encode("utf8"))
+        response = dump_json_to_unicode(response)
+        if not PY2 and not is_bytes(response):
+            response = response.encode("utf-8")
+        self.sendMessage(response)
 
 
 class JSONRPCServerFactory(WebSocketServerFactory):

platformio/commands/lib.py

@@ -14,7 +14,6 @@
 
 # pylint: disable=too-many-branches, too-many-locals
 
-import json
 import time
 from os.path import isdir, join
 
@@ -22,6 +21,7 @@ import click
 import semantic_version
 
 from platformio import exception, util
+from platformio.compat import dump_json_to_unicode
 from platformio.managers.lib import (LibraryManager, get_builtin_libs,
                                      is_builtin_lib)
 from platformio.proc import is_ci
@@ -247,8 +247,8 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output):
 
     if json_output:
         return click.echo(
-            json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) ==
+            dump_json_to_unicode(json_result[storage_dirs[0]]
-                       1 else json_result))
+                                 if len(storage_dirs) == 1 else json_result))
 
     return True
 
@@ -274,8 +274,8 @@ def lib_list(ctx, json_output):
 
     if json_output:
         return click.echo(
-            json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) ==
+            dump_json_to_unicode(json_result[storage_dirs[0]]
-                       1 else json_result))
+                                 if len(storage_dirs) == 1 else json_result))
 
     return True
 
@@ -309,7 +309,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
                                  cache_valid="1d")
 
     if json_output:
-        click.echo(json.dumps(result))
+        click.echo(dump_json_to_unicode(result))
         return
 
     if result['total'] == 0:
@@ -361,7 +361,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
 def lib_builtin(storage, json_output):
     items = get_builtin_libs(storage)
     if json_output:
-        return click.echo(json.dumps(items))
+        return click.echo(dump_json_to_unicode(items))
 
     for storage_ in items:
         if not storage_['items']:
@@ -390,7 +390,7 @@ def lib_show(library, json_output):
                               interactive=not json_output)
     lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:
-        return click.echo(json.dumps(lib))
+        return click.echo(dump_json_to_unicode(lib))
 
     click.secho(lib['name'], fg="cyan")
     click.echo("=" * len(lib['name']))
@@ -478,7 +478,7 @@ def lib_stats(json_output):
     result = util.get_api_result("/lib/stats", cache_valid="1h")
 
     if json_output:
-        return click.echo(json.dumps(result))
+        return click.echo(dump_json_to_unicode(result))
 
     printitem_tpl = "{name:<33} {url}"
     printitemdate_tpl = "{name:<33} {date:23} {url}"

platformio/commands/platform.py

@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 from os.path import dirname, isdir
 
 import click
 
 from platformio import app, exception, util
 from platformio.commands.boards import print_boards
+from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformFactory, PlatformManager
 
 
@@ -156,7 +156,7 @@ def platform_search(query, json_output):
     for platform in _get_registry_platforms():
         if query == "all":
             query = ""
-        search_data = json.dumps(platform)
+        search_data = dump_json_to_unicode(platform)
         if query and query.lower() not in search_data.lower():
             continue
         platforms.append(
@@ -165,7 +165,7 @@ def platform_search(query, json_output):
                                         expose_packages=False))
 
     if json_output:
-        click.echo(json.dumps(platforms))
+        click.echo(dump_json_to_unicode(platforms))
     else:
         _print_platforms(platforms)
 
@@ -178,7 +178,7 @@ def platform_frameworks(query, json_output):
     for framework in util.get_api_result("/frameworks", cache_valid="7d"):
         if query == "all":
             query = ""
-        search_data = json.dumps(framework)
+        search_data = dump_json_to_unicode(framework)
         if query and query.lower() not in search_data.lower():
             continue
         framework['homepage'] = ("https://platformio.org/frameworks/" +
@@ -191,7 +191,7 @@ def platform_frameworks(query, json_output):
 
     frameworks = sorted(frameworks, key=lambda manifest: manifest['name'])
     if json_output:
-        click.echo(json.dumps(frameworks))
+        click.echo(dump_json_to_unicode(frameworks))
     else:
         _print_platforms(frameworks)
 
@@ -209,7 +209,7 @@ def platform_list(json_output):
 
     platforms = sorted(platforms, key=lambda manifest: manifest['name'])
     if json_output:
-        click.echo(json.dumps(platforms))
+        click.echo(dump_json_to_unicode(platforms))
     else:
         _print_platforms(platforms)
 
@@ -222,7 +222,7 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     if not data:
         raise exception.UnknownPlatform(platform)
     if json_output:
-        return click.echo(json.dumps(data))
+        return click.echo(dump_json_to_unicode(data))
 
     click.echo("{name} ~ {title}".format(name=click.style(data['name'],
                                                           fg="cyan"),
@@ -361,7 +361,7 @@ def platform_update(  # pylint: disable=too-many-locals
             if latest:
                 data['versionLatest'] = latest
             result.append(data)
-        return click.echo(json.dumps(result))
+        return click.echo(dump_json_to_unicode(result))
 
     # cleanup cached board and platform lists
     app.clean_cache()

platformio/commands/run/processor.py

@@ -37,7 +37,7 @@ class EnvironmentProcessor(object):
         self.cmd_ctx = cmd_ctx
         self.name = name
         self.config = config
-        self.targets = targets
+        self.targets = [str(t) for t in targets]
         self.upload_port = upload_port
         self.silent = silent
         self.verbose = verbose

platformio/managers/platform.py

@@ -24,7 +24,7 @@ import click
 import semantic_version
 
 from platformio import __version__, app, exception, util
-from platformio.compat import hashlib_encode_data, is_bytes
+from platformio.compat import PY2, hashlib_encode_data, is_bytes
 from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
 from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv,
@@ -693,6 +693,15 @@ class PlatformBoardConfig(object):
             value = self._manifest
             for k in path.split("."):
                 value = value[k]
+            # pylint: disable=undefined-variable
+            if PY2 and isinstance(value, unicode):
+                # cast to plain string from unicode for PY2, resolves issue in
+                # dev/platform when BoardConfig.get() is used in pair with
+                # os.path.join(file_encoding, unicode_encoding)
+                try:
+                    value = value.encode("utf-8")
+                except UnicodeEncodeError:
+                    pass
             return value
         except KeyError:
             if default is not None: