diff --git a/HISTORY.rst b/HISTORY.rst index c1122952..d896f7e3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -43,7 +43,7 @@ PlatformIO 4.0 - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) - - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #2100 `_) + - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 `_, `issue #1342 `_, `issue #1959 `_, `issue #2100 `_) * **Integration** diff --git a/platformio/app.py b/platformio/app.py index 49f24625..c35c4baf 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,7 +14,6 @@ import codecs import hashlib -import json import os import uuid from copy import deepcopy @@ -25,7 +24,8 @@ from time import time import requests from platformio import exception, lockfile, util -from platformio.compat import WINDOWS, hashlib_encode_data +from platformio.compat import (WINDOWS, dump_json_to_unicode, + hashlib_encode_data) from platformio.proc import is_ci from platformio.project.helpers import (get_project_cache_dir, get_project_core_dir) @@ -102,16 +102,19 @@ class State(object): self._lock_state_file() if isfile(self.path): self._state = util.load_json(self.path) - except exception.PlatformioException: + assert isinstance(self._state, dict) + except (AssertionError, UnicodeDecodeError, + exception.PlatformioException): self._state = {} self._prev_state = deepcopy(self._state) return self._state def __exit__(self, type_, value, traceback): - if self._prev_state != self._state: + new_state = dump_json_to_unicode(self._state) + if self._prev_state != new_state: try: - with codecs.open(self.path, "w", encoding="utf8") as fp: - json.dump(self._state, fp) + with open(self.path, "w") as fp: + fp.write(new_state) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() @@ -167,6 +170,7 @@ class ContentCache(object): return True def get_cache_path(self, key): + key = str(key) assert len(key) > 3 return join(self.cache_dir, key[-2:], key) diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index b48be4df..6aff1681 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -17,6 +17,7 @@ import json import click from platformio import util +from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformManager @@ -82,4 +83,4 @@ def _print_boards_json(query, installed=False): if query.lower() not in search_data.lower(): continue result.append(board) - click.echo(json.dumps(result)) + click.echo(dump_json_to_unicode(result)) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 7876fae7..d5ffe030 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import sys from fnmatch import fnmatch from os import getcwd @@ -22,6 +21,7 @@ import click from serial.tools import miniterm from platformio import exception, util +from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig @@ -50,7 +50,8 @@ def device_list( # pylint: disable=too-many-branches single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: - return click.echo(json.dumps(data[single_key] if single_key else data)) + return click.echo( + dump_json_to_unicode(data[single_key] if single_key else data)) titles = { "serial": "Serial Ports", diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index e356fa06..0f8b8285 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -14,11 +14,9 @@ from __future__ import absolute_import -import json -from os.path import expanduser, isfile, join +from os.path import expanduser, join -from platformio import __version__, app, exception, util -from platformio.compat import path_to_unicode +from platformio import __version__, app, util from platformio.project.helpers import (get_project_core_dir, is_platformio_project) @@ -29,57 +27,45 @@ class AppRPC(object): @staticmethod def load_state(): - state = None - try: - if isfile(AppRPC.APPSTATE_PATH): - state = util.load_json(AppRPC.APPSTATE_PATH) - except exception.PlatformioException: - pass - if not isinstance(state, dict): - state = {} - storage = state.get("storage", {}) + with app.State(AppRPC.APPSTATE_PATH, lock=True) as state: + storage = state.get("storage", {}) - # base data - caller_id = app.get_session_var("caller_id") - storage['cid'] = app.get_cid() - storage['coreVersion'] = __version__ - storage['coreSystype'] = util.get_systype() - storage['coreCaller'] = (str(caller_id).lower() if caller_id else None) - storage['coreSettings'] = { - name: { - "description": data['description'], - "default_value": data['value'], - "value": app.get_setting(name) + # base data + caller_id = app.get_session_var("caller_id") + storage['cid'] = app.get_cid() + storage['coreVersion'] = __version__ + storage['coreSystype'] = util.get_systype() + storage['coreCaller'] = (str(caller_id).lower() + if caller_id else None) + storage['coreSettings'] = { + name: { + "description": data['description'], + "default_value": data['value'], + "value": app.get_setting(name) + } + for name, data in app.DEFAULT_SETTINGS.items() } - for name, data in app.DEFAULT_SETTINGS.items() - } - # encode to UTF-8 - for key in storage['coreSettings']: - if not key.endswith("dir"): - continue - storage['coreSettings'][key]['default_value'] = path_to_unicode( - storage['coreSettings'][key]['default_value']) - storage['coreSettings'][key]['value'] = path_to_unicode( - storage['coreSettings'][key]['value']) - storage['homeDir'] = path_to_unicode(expanduser("~")) - storage['projectsDir'] = storage['coreSettings']['projects_dir'][ - 'value'] + storage['homeDir'] = expanduser("~") + storage['projectsDir'] = storage['coreSettings']['projects_dir'][ + 'value'] - # skip non-existing recent projects - storage['recentProjects'] = [ - p for p in storage.get("recentProjects", []) - if is_platformio_project(p) - ] + # skip non-existing recent projects + storage['recentProjects'] = [ + p for p in storage.get("recentProjects", []) + if is_platformio_project(p) + ] - state['storage'] = storage - return state + state['storage'] = storage + return state @staticmethod def get_state(): return AppRPC.load_state() - def save_state(self, state): - with open(self.APPSTATE_PATH, "w") as fp: - json.dump(state, fp) + @staticmethod + def save_state(state): + with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: + s.clear() + s.update(state) return True diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index 8de0e195..77df7bc8 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -27,7 +27,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): def load_latest_tweets(self, username): - cache_key = "piohome_latest_tweets_%s" % username + cache_key = "piohome_latest_tweets_" + str(username) cache_valid = "7d" with app.ContentCache() as cc: cache_data = cc.get(cache_key) @@ -60,13 +60,11 @@ class MiscRPC(object): "include_new_items_bar=true") % username if helpers.is_twitter_blocked(): api_url = self._get_proxed_uri(api_url) - html_or_json = yield OSRPC.fetch_content( + content = yield OSRPC.fetch_content( api_url, headers=self._get_twitter_headers(username)) - # issue with PIO Core < 3.5.3 and ContentCache - if not isinstance(html_or_json, dict): - html_or_json = json.loads(html_or_json) - assert "items_html" in html_or_json - soup = BeautifulSoup(html_or_json['items_html'], "html.parser") + content = json.loads(content) + assert "items_html" in content + soup = BeautifulSoup(content['items_html'], "html.parser") tweet_nodes = soup.find_all("div", attrs={ "class": "tweet", diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index c394d935..d26a0c00 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -25,7 +25,7 @@ from twisted.internet import defer # pylint: disable=import-error from platformio import app, util from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding, path_to_unicode +from platformio.compat import PY2, get_filesystem_encoding class OSRPC(object): @@ -150,6 +150,6 @@ class OSRPC(object): items = [] for item in util.get_logical_devices(): if item['name']: - item['name'] = path_to_unicode(item['name']) + item['name'] = item['name'] items.append(item) return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index b651b498..198552d1 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -23,7 +23,7 @@ from twisted.internet import utils # pylint: disable=import-error from platformio import __version__ from platformio.commands.home import helpers -from platformio.compat import get_filesystem_encoding, string_types +from platformio.compat import string_types class PIOCoreRPC(object): @@ -33,8 +33,8 @@ class PIOCoreRPC(object): json_output = "--json-output" in args try: args = [ - arg.encode(get_filesystem_encoding()) if isinstance( - arg, string_types) else str(arg) for arg in args + str(arg) if not isinstance(arg, string_types) else arg + for arg in args ] except UnicodeError: raise jsonrpc.exceptions.JSONRPCDispatchException( @@ -51,18 +51,12 @@ class PIOCoreRPC(object): @staticmethod def _call_callback(result, json_output=False): - result = list(result) - assert len(result) == 3 - for i in (0, 1): - result[i] = result[i].decode(get_filesystem_encoding()).strip() out, err, code = result text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) - if not json_output: return text - try: return json.loads(out) except ValueError as e: diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 6e029aca..bb301ea9 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -25,7 +25,7 @@ import jsonrpc # pylint: disable=import-error from platformio import exception, util from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC -from platformio.compat import get_filesystem_encoding +from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig @@ -172,6 +172,10 @@ class ProjectRPC(object): return project_dir def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + board = str(board) + if arduino_project_dir and PY2: + arduino_project_dir = arduino_project_dir.encode( + get_filesystem_encoding()) # don't import PIO Project if is_platformio_project(arduino_project_dir): return arduino_project_dir @@ -188,7 +192,7 @@ class ProjectRPC(object): message="Not an Arduino project: %s" % arduino_project_dir) state = AppRPC.load_state() - project_dir = join(state['storage']['projectsDir'].decode("utf-8"), + project_dir = join(state['storage']['projectsDir'], time.strftime("%y%m%d-%H%M%S-") + board) if not isdir(project_dir): os.makedirs(project_dir) @@ -213,8 +217,7 @@ class ProjectRPC(object): src_dir = get_project_src_dir() if isdir(src_dir): util.rmtree_(src_dir) - shutil.copytree( - arduino_project_dir.encode(get_filesystem_encoding()), src_dir) + shutil.copytree(arduino_project_dir, src_dir) return project_dir @staticmethod @@ -255,12 +258,14 @@ class ProjectRPC(object): @staticmethod def import_pio(project_dir): + if project_dir and PY2: + project_dir = project_dir.encode(get_filesystem_encoding()) if not project_dir or not is_platformio_project(project_dir): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir) new_project_dir = join( - AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"), + AppRPC.load_state()['storage']['projectsDir'], time.strftime("%y%m%d-%H%M%S-") + basename(project_dir)) shutil.copytree(project_dir, new_project_dir) diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index d62faa3d..f99b09b3 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -14,14 +14,14 @@ # pylint: disable=import-error -import json - import jsonrpc from autobahn.twisted.websocket import (WebSocketServerFactory, WebSocketServerProtocol) from jsonrpc.exceptions import JSONRPCDispatchException from twisted.internet import defer +from platformio.compat import PY2, dump_json_to_unicode, is_bytes + class JSONRPCServerProtocol(WebSocketServerProtocol): @@ -57,7 +57,10 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): def sendJSONResponse(self, response): # print("< %s" % response) - self.sendMessage(json.dumps(response).encode("utf8")) + response = dump_json_to_unicode(response) + if not PY2 and not is_bytes(response): + response = response.encode("utf-8") + self.sendMessage(response) class JSONRPCServerFactory(WebSocketServerFactory): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 23094f43..ad39bf53 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -14,7 +14,6 @@ # pylint: disable=too-many-branches, too-many-locals -import json import time from os.path import isdir, join @@ -22,6 +21,7 @@ import click import semantic_version from platformio import exception, util +from platformio.compat import dump_json_to_unicode from platformio.managers.lib import (LibraryManager, get_builtin_libs, is_builtin_lib) from platformio.proc import is_ci @@ -247,8 +247,8 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): if json_output: return click.echo( - json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) == - 1 else json_result)) + dump_json_to_unicode(json_result[storage_dirs[0]] + if len(storage_dirs) == 1 else json_result)) return True @@ -274,8 +274,8 @@ def lib_list(ctx, json_output): if json_output: return click.echo( - json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) == - 1 else json_result)) + dump_json_to_unicode(json_result[storage_dirs[0]] + if len(storage_dirs) == 1 else json_result)) return True @@ -309,7 +309,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): cache_valid="1d") if json_output: - click.echo(json.dumps(result)) + click.echo(dump_json_to_unicode(result)) return if result['total'] == 0: @@ -361,7 +361,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): def lib_builtin(storage, json_output): items = get_builtin_libs(storage) if json_output: - return click.echo(json.dumps(items)) + return click.echo(dump_json_to_unicode(items)) for storage_ in items: if not storage_['items']: @@ -390,7 +390,7 @@ def lib_show(library, json_output): interactive=not json_output) lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: - return click.echo(json.dumps(lib)) + return click.echo(dump_json_to_unicode(lib)) click.secho(lib['name'], fg="cyan") click.echo("=" * len(lib['name'])) @@ -478,7 +478,7 @@ def lib_stats(json_output): result = util.get_api_result("/lib/stats", cache_valid="1h") if json_output: - return click.echo(json.dumps(result)) + return click.echo(dump_json_to_unicode(result)) printitem_tpl = "{name:<33} {url}" printitemdate_tpl = "{name:<33} {date:23} {url}" diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 9f56f8d6..49a61812 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json from os.path import dirname, isdir import click from platformio import app, exception, util from platformio.commands.boards import print_boards +from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory, PlatformManager @@ -156,7 +156,7 @@ def platform_search(query, json_output): for platform in _get_registry_platforms(): if query == "all": query = "" - search_data = json.dumps(platform) + search_data = dump_json_to_unicode(platform) if query and query.lower() not in search_data.lower(): continue platforms.append( @@ -165,7 +165,7 @@ def platform_search(query, json_output): expose_packages=False)) if json_output: - click.echo(json.dumps(platforms)) + click.echo(dump_json_to_unicode(platforms)) else: _print_platforms(platforms) @@ -178,7 +178,7 @@ def platform_frameworks(query, json_output): for framework in util.get_api_result("/frameworks", cache_valid="7d"): if query == "all": query = "" - search_data = json.dumps(framework) + search_data = dump_json_to_unicode(framework) if query and query.lower() not in search_data.lower(): continue framework['homepage'] = ("https://platformio.org/frameworks/" + @@ -191,7 +191,7 @@ def platform_frameworks(query, json_output): frameworks = sorted(frameworks, key=lambda manifest: manifest['name']) if json_output: - click.echo(json.dumps(frameworks)) + click.echo(dump_json_to_unicode(frameworks)) else: _print_platforms(frameworks) @@ -209,7 +209,7 @@ def platform_list(json_output): platforms = sorted(platforms, key=lambda manifest: manifest['name']) if json_output: - click.echo(json.dumps(platforms)) + click.echo(dump_json_to_unicode(platforms)) else: _print_platforms(platforms) @@ -222,7 +222,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if not data: raise exception.UnknownPlatform(platform) if json_output: - return click.echo(json.dumps(data)) + return click.echo(dump_json_to_unicode(data)) click.echo("{name} ~ {title}".format(name=click.style(data['name'], fg="cyan"), @@ -361,7 +361,7 @@ def platform_update( # pylint: disable=too-many-locals if latest: data['versionLatest'] = latest result.append(data) - return click.echo(json.dumps(result)) + return click.echo(dump_json_to_unicode(result)) # cleanup cached board and platform lists app.clean_cache() diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 66c07296..2fbba4be 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -37,7 +37,7 @@ class EnvironmentProcessor(object): self.cmd_ctx = cmd_ctx self.name = name self.config = config - self.targets = targets + self.targets = [str(t) for t in targets] self.upload_port = upload_port self.silent = silent self.verbose = verbose diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index b2e467d4..d34b9880 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -24,7 +24,7 @@ import click import semantic_version from platformio import __version__, app, exception, util -from platformio.compat import hashlib_encode_data, is_bytes +from platformio.compat import PY2, hashlib_encode_data, is_bytes from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, @@ -693,6 +693,15 @@ class PlatformBoardConfig(object): value = self._manifest for k in path.split("."): value = value[k] + # pylint: disable=undefined-variable + if PY2 and isinstance(value, unicode): + # cast to plain string from unicode for PY2, resolves issue in + # dev/platform when BoardConfig.get() is used in pair with + # os.path.join(file_encoding, unicode_encoding) + try: + value = value.encode("utf-8") + except UnicodeEncodeError: + pass return value except KeyError: if default is not None: