From 943c6bc59cd8f1e3b40b15e71d741014d775977b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 19 Apr 2022 11:36:05 +0300 Subject: [PATCH] Move INO converter to a separate tool --- platformio/builder/tools/pioino.py | 254 ++++++++++++++++++++++++++++ platformio/builder/tools/piomisc.py | 238 +------------------------- 2 files changed, 256 insertions(+), 236 deletions(-) create mode 100644 platformio/builder/tools/pioino.py diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py new file mode 100644 index 00000000..7a95da9b --- /dev/null +++ b/platformio/builder/tools/pioino.py @@ -0,0 +1,254 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import atexit +import glob +import io +import os +import re +import tempfile + +import click + +from platformio.compat import get_filesystem_encoding, get_locale_encoding + + +class InoToCPPConverter(object): + + PROTOTYPE_RE = re.compile( + r"""^( + (?:template\<.*\>\s*)? # template + ([a-z_\d\&]+\*?\s+){1,2} # return type + ([a-z_\d]+\s*) # name of prototype + \([a-z_,\.\*\&\[\]\s\d]*\) # arguments + )\s*(\{|;) # must end with `{` or `;` + """, + re.X | re.M | re.I, + ) + DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) + PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)" + + def __init__(self, env): + self.env = env + self._main_ino = None + self._safe_encoding = None + + def read_safe_contents(self, path): + error_reported = False + for encoding in ( + "utf-8", + None, + get_filesystem_encoding(), + get_locale_encoding(), + "latin-1", + ): + try: + with io.open(path, encoding=encoding) as fp: + contents = fp.read() + self._safe_encoding = encoding + return contents + except UnicodeDecodeError: + if not error_reported: + error_reported = True + click.secho( + "Unicode decode error has occurred, please remove invalid " + "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" + % path, + fg="yellow", + err=True, + ) + return "" + + def write_safe_contents(self, path, contents): + with io.open( + path, "w", encoding=self._safe_encoding, errors="backslashreplace" + ) as fp: + return fp.write(contents) + + def is_main_node(self, contents): + return self.DETECTMAIN_RE.search(contents) + + def convert(self, nodes): + contents = self.merge(nodes) + if not contents: + return None + return self.process(contents) + + def merge(self, nodes): + assert nodes + lines = [] + for node in nodes: + contents = self.read_safe_contents(node.get_path()) + _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] + if self.is_main_node(contents): + lines = _lines + lines + self._main_ino = node.get_path() + else: + lines.extend(_lines) + + if not self._main_ino: + self._main_ino = nodes[0].get_path() + + return "\n".join(["#include "] + lines) if lines else None + + def process(self, contents): + out_file = self._main_ino + ".cpp" + assert self._gcc_preprocess(contents, out_file) + contents = self.read_safe_contents(out_file) + contents = self._join_multiline_strings(contents) + self.write_safe_contents(out_file, self.append_prototypes(contents)) + return out_file + + def _gcc_preprocess(self, contents, out_file): + tmp_path = tempfile.mkstemp()[1] + self.write_safe_contents(tmp_path, contents) + self.env.Execute( + self.env.VerboseAction( + '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( + out_file, tmp_path + ), + "Converting " + os.path.basename(out_file[:-4]), + ) + ) + atexit.register(_delete_file, tmp_path) + return os.path.isfile(out_file) + + def _join_multiline_strings(self, contents): + if "\\\n" not in contents: + return contents + newlines = [] + linenum = 0 + stropen = False + for line in contents.split("\n"): + _linenum = self._parse_preproc_line_num(line) + if _linenum is not None: + linenum = _linenum + else: + linenum += 1 + + if line.endswith("\\"): + if line.startswith('"'): + stropen = True + newlines.append(line[:-1]) + continue + if stropen: + newlines[len(newlines) - 1] += line[:-1] + continue + elif stropen and line.endswith(('",', '";')): + newlines[len(newlines) - 1] += line + stropen = False + newlines.append( + '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/")) + ) + continue + + newlines.append(line) + + return "\n".join(newlines) + + @staticmethod + def _parse_preproc_line_num(line): + if not line.startswith("#"): + return None + tokens = line.split(" ", 3) + if len(tokens) > 2 and tokens[1].isdigit(): + return int(tokens[1]) + return None + + def _parse_prototypes(self, contents): + prototypes = [] + reserved_keywords = set(["if", "else", "while"]) + for match in self.PROTOTYPE_RE.finditer(contents): + if ( + set([match.group(2).strip(), match.group(3).strip()]) + & reserved_keywords + ): + continue + prototypes.append(match) + return prototypes + + def _get_total_lines(self, contents): + total = 0 + if contents.endswith("\n"): + contents = contents[:-1] + for line in contents.split("\n")[::-1]: + linenum = self._parse_preproc_line_num(line) + if linenum is not None: + return total + linenum + total += 1 + return total + + def append_prototypes(self, contents): + prototypes = self._parse_prototypes(contents) or [] + + # skip already declared prototypes + declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";") + prototypes = [m for m in prototypes if m.group(1).strip() not in declared] + + if not prototypes: + return contents + + prototype_names = set(m.group(3).strip() for m in prototypes) + split_pos = prototypes[0].start() + match_ptrs = re.search( + self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), + contents[:split_pos], + re.M, + ) + if match_ptrs: + split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1 + + result = [] + result.append(contents[:split_pos].strip()) + result.append("%s;" % ";\n".join([m.group(1) for m in prototypes])) + result.append( + '#line %d "%s"' + % ( + self._get_total_lines(contents[:split_pos]), + self._main_ino.replace("\\", "/"), + ) + ) + result.append(contents[split_pos:].strip()) + return "\n".join(result) + + +def ConvertInoToCpp(env): + src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR")) + ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( + os.path.join(src_dir, "*.pde") + ) + if not ino_nodes: + return + c = InoToCPPConverter(env) + out_file = c.convert(ino_nodes) + + atexit.register(_delete_file, out_file) + + +def _delete_file(path): + try: + if os.path.isfile(path): + os.remove(path) + except: # pylint: disable=bare-except + pass + + +def generate(env): + env.AddMethod(ConvertInoToCpp) + + +def exists(_): + return True diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index ae3a22ba..07434ca7 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -14,244 +14,15 @@ from __future__ import absolute_import -import atexit -import glob -import io import os -import re import sys -import tempfile - -import click from platformio import fs, util -from platformio.compat import get_filesystem_encoding, get_locale_encoding -from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command -class InoToCPPConverter(object): - - PROTOTYPE_RE = re.compile( - r"""^( - (?:template\<.*\>\s*)? # template - ([a-z_\d\&]+\*?\s+){1,2} # return type - ([a-z_\d]+\s*) # name of prototype - \([a-z_,\.\*\&\[\]\s\d]*\) # arguments - )\s*(\{|;) # must end with `{` or `;` - """, - re.X | re.M | re.I, - ) - DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) - PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)" - - def __init__(self, env): - self.env = env - self._main_ino = None - self._safe_encoding = None - - def read_safe_contents(self, path): - error_reported = False - for encoding in ( - "utf-8", - None, - get_filesystem_encoding(), - get_locale_encoding(), - "latin-1", - ): - try: - with io.open(path, encoding=encoding) as fp: - contents = fp.read() - self._safe_encoding = encoding - return contents - except UnicodeDecodeError: - if not error_reported: - error_reported = True - click.secho( - "Unicode decode error has occurred, please remove invalid " - "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" - % path, - fg="yellow", - err=True, - ) - return "" - - def write_safe_contents(self, path, contents): - with io.open( - path, "w", encoding=self._safe_encoding, errors="backslashreplace" - ) as fp: - return fp.write(contents) - - def is_main_node(self, contents): - return self.DETECTMAIN_RE.search(contents) - - def convert(self, nodes): - contents = self.merge(nodes) - if not contents: - return None - return self.process(contents) - - def merge(self, nodes): - assert nodes - lines = [] - for node in nodes: - contents = self.read_safe_contents(node.get_path()) - _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] - if self.is_main_node(contents): - lines = _lines + lines - self._main_ino = node.get_path() - else: - lines.extend(_lines) - - if not self._main_ino: - self._main_ino = nodes[0].get_path() - - return "\n".join(["#include "] + lines) if lines else None - - def process(self, contents): - out_file = self._main_ino + ".cpp" - assert self._gcc_preprocess(contents, out_file) - contents = self.read_safe_contents(out_file) - contents = self._join_multiline_strings(contents) - self.write_safe_contents(out_file, self.append_prototypes(contents)) - return out_file - - def _gcc_preprocess(self, contents, out_file): - tmp_path = tempfile.mkstemp()[1] - self.write_safe_contents(tmp_path, contents) - self.env.Execute( - self.env.VerboseAction( - '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( - out_file, tmp_path - ), - "Converting " + os.path.basename(out_file[:-4]), - ) - ) - atexit.register(_delete_file, tmp_path) - return os.path.isfile(out_file) - - def _join_multiline_strings(self, contents): - if "\\\n" not in contents: - return contents - newlines = [] - linenum = 0 - stropen = False - for line in contents.split("\n"): - _linenum = self._parse_preproc_line_num(line) - if _linenum is not None: - linenum = _linenum - else: - linenum += 1 - - if line.endswith("\\"): - if line.startswith('"'): - stropen = True - newlines.append(line[:-1]) - continue - if stropen: - newlines[len(newlines) - 1] += line[:-1] - continue - elif stropen and line.endswith(('",', '";')): - newlines[len(newlines) - 1] += line - stropen = False - newlines.append( - '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/")) - ) - continue - - newlines.append(line) - - return "\n".join(newlines) - - @staticmethod - def _parse_preproc_line_num(line): - if not line.startswith("#"): - return None - tokens = line.split(" ", 3) - if len(tokens) > 2 and tokens[1].isdigit(): - return int(tokens[1]) - return None - - def _parse_prototypes(self, contents): - prototypes = [] - reserved_keywords = set(["if", "else", "while"]) - for match in self.PROTOTYPE_RE.finditer(contents): - if ( - set([match.group(2).strip(), match.group(3).strip()]) - & reserved_keywords - ): - continue - prototypes.append(match) - return prototypes - - def _get_total_lines(self, contents): - total = 0 - if contents.endswith("\n"): - contents = contents[:-1] - for line in contents.split("\n")[::-1]: - linenum = self._parse_preproc_line_num(line) - if linenum is not None: - return total + linenum - total += 1 - return total - - def append_prototypes(self, contents): - prototypes = self._parse_prototypes(contents) or [] - - # skip already declared prototypes - declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";") - prototypes = [m for m in prototypes if m.group(1).strip() not in declared] - - if not prototypes: - return contents - - prototype_names = set(m.group(3).strip() for m in prototypes) - split_pos = prototypes[0].start() - match_ptrs = re.search( - self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), - contents[:split_pos], - re.M, - ) - if match_ptrs: - split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1 - - result = [] - result.append(contents[:split_pos].strip()) - result.append("%s;" % ";\n".join([m.group(1) for m in prototypes])) - result.append( - '#line %d "%s"' - % ( - self._get_total_lines(contents[:split_pos]), - self._main_ino.replace("\\", "/"), - ) - ) - result.append(contents[split_pos:].strip()) - return "\n".join(result) - - -def ConvertInoToCpp(env): - src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR")) - ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( - os.path.join(src_dir, "*.pde") - ) - if not ino_nodes: - return - c = InoToCPPConverter(env) - out_file = c.convert(ino_nodes) - - atexit.register(_delete_file, out_file) - - -def _delete_file(path): - try: - if os.path.isfile(path): - os.remove(path) - except: # pylint: disable=bare-except - pass - - @util.memoized() -def _get_compiler_type(env): +def GetCompilerType(env): if env.subst("$CC").endswith("-gcc"): return "gcc" try: @@ -270,10 +41,6 @@ def _get_compiler_type(env): return None -def GetCompilerType(env): - return _get_compiler_type(env) - - def GetActualLDScript(env): def _lookup_in_ldpath(script): for d in env.get("LIBPATH", []): @@ -319,7 +86,7 @@ def GetActualLDScript(env): env.Exit(1) -def ConfigureDebugFlags(env): +def ConfigureDebugTarget(env): def _cleanup_debug_flags(scope): if scope not in env: return @@ -384,7 +151,6 @@ def exists(_): def generate(env): - env.AddMethod(ConvertInoToCpp) env.AddMethod(GetCompilerType) env.AddMethod(GetActualLDScript) env.AddMethod(ConfigureDebugFlags)