From 3a27fbc883cdcd12c241dafcf9d512c214c10813 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 5 Mar 2020 23:52:46 +0200 Subject: [PATCH] Fixed an issue when Python 2 does not keep encoding when converting .INO file // Resolve #3393 --- HISTORY.rst | 1 + platformio/builder/tools/piomisc.py | 48 ++++++++++++++++++++++++----- platformio/fs.py | 38 +++++------------------ 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 16145dfa..ea04061a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ PlatformIO Core 4 * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server * Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) +* Fixed an issue when Python 2 does not keep encoding when converting .INO file (`issue #3393 `_) 4.2.1 (2020-02-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index ba019ec3..2df5303d 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -15,17 +15,19 @@ from __future__ import absolute_import import atexit +import io import re import sys from os import environ, remove, walk from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp +import click from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import fs, util -from platformio.compat import glob_escape +from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command @@ -48,6 +50,40 @@ class InoToCPPConverter(object): def __init__(self, env): self.env = env self._main_ino = None + self._safe_encoding = None + + def read_safe_contents(self, path): + last_exc = None + for encoding in ( + "utf-8", + None, + get_filesystem_encoding(), + get_locale_encoding(), + "latin-1", + ): + try: + with io.open(path, encoding=encoding) as fp: + contents = fp.read() + self._safe_encoding = encoding + return contents + except UnicodeDecodeError as e: + last_exc = e + click.secho( + "Unicode decode error has occurred, please remove invalid " + "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" + % path, + fg="yellow", + err=True, + ) + if last_exc: + raise last_exc + return None + + def write_safe_contents(self, path, contents): + with io.open( + path, "w", encoding=self._safe_encoding, errors="backslashreplace" + ) as fp: + return fp.write(contents) def is_main_node(self, contents): return self.DETECTMAIN_RE.search(contents) @@ -62,7 +98,7 @@ class InoToCPPConverter(object): assert nodes lines = [] for node in nodes: - contents = fs.get_file_contents(node.get_path()) + contents = self.read_safe_contents(node.get_path()) _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] if self.is_main_node(contents): lines = _lines + lines @@ -78,16 +114,14 @@ class InoToCPPConverter(object): def process(self, contents): out_file = self._main_ino + ".cpp" assert self._gcc_preprocess(contents, out_file) - contents = fs.get_file_contents(out_file) + contents = self.read_safe_contents(out_file) contents = self._join_multiline_strings(contents) - fs.write_file_contents( - out_file, self.append_prototypes(contents), errors="backslashreplace" - ) + self.write_safe_contents(out_file, self.append_prototypes(contents)) return out_file def _gcc_preprocess(self, contents, out_file): tmp_path = mkstemp()[1] - fs.write_file_contents(tmp_path, contents, errors="backslashreplace") + self.write_safe_contents(tmp_path, contents) self.env.Execute( self.env.VerboseAction( '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( diff --git a/platformio/fs.py b/platformio/fs.py index ed0102cd..575a14e5 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import io import json import os import re @@ -49,30 +48,6 @@ def get_source_dir(): return os.path.dirname(curpath) -def get_file_contents(path, encoding=None): - try: - with io.open(path, encoding=encoding) as fp: - return fp.read() - except UnicodeDecodeError: - click.secho( - "Unicode decode error has occurred, please remove invalid " - "(non-ASCII or non-UTF8) characters from %s file" % path, - fg="yellow", - err=True, - ) - with io.open(path, encoding="latin-1") as fp: - return fp.read() - - -def write_file_contents(path, contents, errors=None): - try: - with open(path, "w") as fp: - return fp.write(contents) - except UnicodeEncodeError: - with io.open(path, "w", encoding="latin-1", errors=errors) as fp: - return fp.write(contents) - - def load_json(file_path): try: with open(file_path, "r") as f: @@ -102,11 +77,14 @@ def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel def _rules_to_set(rules_path): - return set( - l.strip() - for l in get_file_contents(rules_path).split("\n") - if l.strip() and not l.startswith("#") - ) + result = set() + with open(rules_path) as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + result.add(line) + return result if "linux" not in get_systype(): return None