From 8a379d2db26ff0deb37be83e82d1abe72e5439f8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 31 Aug 2016 00:16:23 +0300 Subject: [PATCH] Refactor INO to CPP converter --- HISTORY.rst | 1 + docs/platforms/creating_platform.rst | 5 +- platformio/__init__.py | 2 +- platformio/builder/tools/piolib.py | 30 +++-- platformio/builder/tools/piomisc.py | 156 +++++++++++++------------ platformio/builder/tools/platformio.py | 8 +- platformio/commands/platform.py | 2 +- platformio/managers/package.py | 6 +- tests/ino2cpp/basic/basic.ino | 18 +++ tests/ino2cpp/multifiles/bar.ino | 3 + tests/ino2cpp/multifiles/foo.pde | 13 +++ tests/test_ino2cpp.py | 41 +++++++ 12 files changed, 179 insertions(+), 106 deletions(-) create mode 100644 tests/ino2cpp/basic/basic.ino create mode 100644 tests/ino2cpp/multifiles/bar.ino create mode 100644 tests/ino2cpp/multifiles/foo.pde create mode 100644 tests/test_ino2cpp.py diff --git a/HISTORY.rst b/HISTORY.rst index 6be89824..901c1ba9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -73,6 +73,7 @@ PlatformIO 3.0 * Print human-readable information when processing environments without ``-v, --verbose`` option (`issue #721 `_) +* Improved INO to CPP converter * Added ``license`` field to `library.json `__ (`issue #522 `_) * Warn about unknown options in project configuration file ``platformio.ini`` diff --git a/docs/platforms/creating_platform.rst b/docs/platforms/creating_platform.rst index 90f14517..29ceec6f 100644 --- a/docs/platforms/creating_platform.rst +++ b/docs/platforms/creating_platform.rst @@ -198,10 +198,7 @@ Manifest File ``platform.json`` "description": "My custom development platform", "url": "http://example.com", "homepage": "http://platformio.org/platforms/myplatform", - "license": { - "type": "Apache-2.0", - "url": "http://opensource.org/licenses/apache2.0.php" - }, + "license": "Apache-2.0", "engines": { "platformio": "~3.0.0", "scons": ">=2.3.0,<2.6.0" diff --git a/platformio/__init__.py b/platformio/__init__.py index 7ee7955e..29a27397 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 0, "0a13") +VERSION = (3, 0, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 04b2d40c..47f0351c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -134,8 +134,8 @@ class LibBuilderBase(object): # pylint: disable=too-many-instance-attributes def build_dir(self): return join("$BUILD_DIR", "lib", basename(self.path)) - def get_inc_dirs(self, use_build_dir=False): - return [self.build_dir if use_build_dir else self.src_dir] + def get_inc_dirs(self): + return [self.src_dir] @property def build_flags(self): @@ -290,8 +290,7 @@ class LibBuilderBase(object): # pylint: disable=too-many-instance-attributes self.env.AppendUnique(**{key: lb.env.get(key)}) if not self._built_node: - self.env.AppendUnique(CPPPATH=self.get_inc_dirs( - use_build_dir=True)) + self.env.AppendUnique(CPPPATH=self.get_inc_dirs()) if self.lib_archive: self._built_node = self.env.BuildLibrary( self.build_dir, self.src_dir, self.src_filter) @@ -345,12 +344,11 @@ class ArduinoLibBuilder(LibBuilderBase): manifest[key.strip()] = value.strip() return manifest - def get_inc_dirs(self, use_build_dir=False): - inc_dirs = LibBuilderBase.get_inc_dirs(self, use_build_dir) + def get_inc_dirs(self): + inc_dirs = LibBuilderBase.get_inc_dirs(self) if not isdir(join(self.path, "utility")): return inc_dirs - inc_dirs.append( - join(self.build_dir if use_build_dir else self.path, "utility")) + inc_dirs.append(join(self.path, "utility")) return inc_dirs @property @@ -382,8 +380,8 @@ class MbedLibBuilder(LibBuilderBase): return join(self.path, "source") return LibBuilderBase.src_dir.fget(self) - def get_inc_dirs(self, use_build_dir=False): - inc_dirs = LibBuilderBase.get_inc_dirs(self, use_build_dir) + def get_inc_dirs(self): + inc_dirs = LibBuilderBase.get_inc_dirs(self) if self.path not in inc_dirs: inc_dirs.append(self.path) for p in self._manifest.get("extraIncludes", []): @@ -451,12 +449,11 @@ class PlatformIOLibBuilder(LibBuilderBase): ilist = [i.strip() for i in ilist.split(",")] return item.lower() in [i.lower() for i in ilist] - def get_inc_dirs(self, use_build_dir=False): - inc_dirs = LibBuilderBase.get_inc_dirs(self, use_build_dir) + def get_inc_dirs(self): + inc_dirs = LibBuilderBase.get_inc_dirs(self) for path in self.env['CPPPATH']: if path not in self.envorigin['CPPPATH']: - inc_dirs.append( - path if use_build_dir else self.env.subst(path)) + inc_dirs.append(self.env.subst(path)) return inc_dirs @@ -499,8 +496,9 @@ def GetLibBuilders(env): try: lb = LibBuilderFactory.new(env, join(libs_dir, item)) except ValueError: - sys.stderr.write("Skip library with broken manifest: %s\n" % - join(libs_dir, item)) + if verbose: + sys.stderr.write("Skip library with broken manifest: %s\n" + % join(libs_dir, item)) continue if _check_lib_builder(lb): items += (lb, ) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 78ca1e26..e5b73741 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -19,7 +19,8 @@ import re import sys from glob import glob from os import environ, remove -from os.path import isfile, join +from os.path import basename, isfile, join +from tempfile import mkstemp from SCons.Action import Action from SCons.Script import ARGUMENTS @@ -38,106 +39,118 @@ class InoToCPPConverter(object): DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)" - def __init__(self, nodes): - self.nodes = nodes + def __init__(self, env): + self.env = env + self._main_ino = None def is_main_node(self, contents): return self.DETECTMAIN_RE.search(contents) - def _parse_prototypes(self, file_path, contents): + def convert(self, nodes): + contents = self.merge(nodes) + if not contents: + return + return self.process(contents) + + def merge(self, nodes): + lines = [] + for node in nodes: + contents = node.get_text_contents() + _lines = [ + '# 1 "%s"' % node.get_path().replace("\\", "/"), contents + ] + if self.is_main_node(contents): + lines = _lines + lines + self._main_ino = node.get_path() + else: + lines.extend(_lines) + + return "\n".join(["#include "] + lines) if lines else None + + def process(self, contents): + out_file = self._main_ino + ".cpp" + assert self._gcc_preprocess(contents, out_file) + with open(out_file) as fp: + contents = fp.read() + with open(out_file, "w") as fp: + fp.write(self.append_prototypes(contents)) + return out_file + + def _gcc_preprocess(self, contents, out_file): + tmp_path = mkstemp()[1] + with open(tmp_path, "w") as fp: + fp.write(contents) + fp.close() + self.env.Execute( + self.env.VerboseAction( + '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( + out_file, tmp_path), "Converting " + basename( + out_file[:-4]))) + remove(tmp_path) + return isfile(out_file) + + def _parse_prototypes(self, contents): prototypes = [] reserved_keywords = set(["if", "else", "while"]) for match in self.PROTOTYPE_RE.finditer(contents): if (set([match.group(2).strip(), match.group(3).strip()]) & reserved_keywords): continue - prototypes.append({"path": file_path, "match": match}) + prototypes.append(match) return prototypes - def append_prototypes(self, file_path, contents, prototypes): - result = [] + @staticmethod + def _get_total_lines(contents): + total = 0 + for line in contents.split("\n")[::-1]: + if line.startswith("#"): + tokens = line.split(" ", 3) + if len(tokens) > 2 and tokens[1].isdigit(): + return int(tokens[1]) + total + total += 1 + return total + + def append_prototypes(self, contents): + prototypes = self._parse_prototypes(contents) if not prototypes: - return result - - prototype_names = set( - [p['match'].group(3).strip() for p in prototypes]) - split_pos = prototypes[0]['match'].start() - for item in prototypes: - if item['path'] == file_path: - split_pos = item['match'].start() - break + return contents + prototype_names = set([m.group(3).strip() for m in prototypes]) + split_pos = prototypes[0].start() match_ptrs = re.search(self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), contents[:split_pos], re.M) if match_ptrs: split_pos = contents.rfind("\n", 0, match_ptrs.start()) + result = [] result.append(contents[:split_pos].strip()) - result.append("%s;" % - ";\n".join([p['match'].group(1) for p in prototypes])) + result.append("%s;" % ";\n".join([m.group(1) for m in prototypes])) result.append('#line %d "%s"' % - (contents.count("\n", 0, split_pos) + 2, - file_path.replace("\\", "/"))) + (self._get_total_lines(contents[:split_pos]), + self._main_ino.replace("\\", "/"))) result.append(contents[split_pos:].strip()) - - return result - - def convert(self): - prototypes = [] - data = [] - for node in self.nodes: - ino_contents = node.get_text_contents() - prototypes += self._parse_prototypes(node.get_path(), ino_contents) - - item = (node.get_path(), ino_contents) - if self.is_main_node(ino_contents): - data = [item] + data - else: - data.append(item) - - if not data: - return None - - result = ["#include "] - is_first = True - for file_path, contents in data: - result.append('#line 1 "%s"' % file_path.replace("\\", "/")) - - if is_first and prototypes: - result += self.append_prototypes(file_path, contents, - prototypes) - else: - result.append(contents) - is_first = False - return "\n".join(result) def ConvertInoToCpp(env): - def delete_tmpcpp_file(file_): + def _delete_file(path): try: - remove(file_) + if isfile(path): + remove(path) except: # pylint: disable=bare-except - if isfile(file_): - print("Warning: Could not remove temporary file '%s'. " - "Please remove it manually." % file_) + if path and isfile(path): + sys.stderr.write( + "Warning: Could not remove temporary file '%s'. " + "Please remove it manually.\n" % path) ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) + env.Glob(join("$PROJECTSRC_DIR", "*.pde"))) + c = InoToCPPConverter(env) + out_file = c.convert(ino_nodes) - c = InoToCPPConverter(ino_nodes) - data = c.convert() - - if not data: - return - - tmpcpp_file = join(env.subst("$PROJECTSRC_DIR"), "tmp_ino_to.cpp") - with open(tmpcpp_file, "w") as f: - f.write(data) - - atexit.register(delete_tmpcpp_file, tmpcpp_file) + atexit.register(_delete_file, out_file) def DumpIDEData(env): @@ -146,14 +159,7 @@ def DumpIDEData(env): includes = [] for item in env_.get("CPPPATH", []): - invardir = False - for vardiritem in env_.get("VARIANT_DIRS", []): - if item == vardiritem[0]: - includes.append(env_.subst(vardiritem[1])) - invardir = True - break - if not invardir: - includes.append(env_.subst(item)) + includes.append(env_.subst(item)) # installed libs for lb in env.GetLibBuilders(): diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 9634ef1e..3f56eebd 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -198,11 +198,6 @@ def MatchSourceFiles(env, src_dir, src_filter=None): return sorted(list(matches)) -def VariantDirWrap(env, variant_dir, src_dir, duplicate=False): - DefaultEnvironment().Append(VARIANT_DIRS=[(variant_dir, src_dir)]) - env.VariantDir(variant_dir, src_dir, duplicate) - - def CollectBuildFiles(env, variant_dir, src_dir, @@ -222,7 +217,7 @@ def CollectBuildFiles(env, if _var_dir not in variants: variants.append(_var_dir) - env.VariantDirWrap(_var_dir, _src_dir, duplicate) + env.VariantDir(_var_dir, _src_dir, duplicate) if env.IsFileWithExt(item, SRC_BUILD_EXT): sources.append(env.File(join(_var_dir, basename(item)))) @@ -283,7 +278,6 @@ def generate(env): env.AddMethod(ProcessUnFlags) env.AddMethod(IsFileWithExt) env.AddMethod(MatchSourceFiles) - env.AddMethod(VariantDirWrap) env.AddMethod(CollectBuildFiles) env.AddMethod(BuildFrameworks) env.AddMethod(BuildLibrary) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 7934ebe9..7f98b257 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -165,7 +165,7 @@ def platform_show(platform): if p.homepage: click.echo("Home: %s" % p.homepage) if p.license: - click.echo("License: %s" % p.license.get("type")) + click.echo("License: %s" % p.license) if p.frameworks: click.echo("Frameworks: %s" % ", ".join(p.frameworks.keys())) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 649b747f..39626f76 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -27,7 +27,6 @@ from platformio.downloader import FileDownloader from platformio.unpacker import FileUnpacker from platformio.vcsclient import VCSClientFactory - # pylint: disable=too-many-arguments @@ -504,7 +503,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): label=manifest['name']) return True - def update(self, name, requirements=None, only_check=False): + def update(self, # pylint: disable=too-many-return-statements + name, + requirements=None, + only_check=False): name, requirements, url = self.parse_pkg_name(name, requirements) package_dir = self.get_package_dir(name, requirements, url) if not package_dir: diff --git a/tests/ino2cpp/basic/basic.ino b/tests/ino2cpp/basic/basic.ino new file mode 100644 index 00000000..47b48694 --- /dev/null +++ b/tests/ino2cpp/basic/basic.ino @@ -0,0 +1,18 @@ + +struct MyItem { + byte foo[50]; + int bar; +}; + +void setup() { + struct MyItem item1; + myFunction(&item1); +} + +void loop() { + +} + +void myFunction(struct MyItem *item) { + +} diff --git a/tests/ino2cpp/multifiles/bar.ino b/tests/ino2cpp/multifiles/bar.ino new file mode 100644 index 00000000..5eac56d9 --- /dev/null +++ b/tests/ino2cpp/multifiles/bar.ino @@ -0,0 +1,3 @@ +void barFunc () { // my comment + +} \ No newline at end of file diff --git a/tests/ino2cpp/multifiles/foo.pde b/tests/ino2cpp/multifiles/foo.pde new file mode 100644 index 00000000..4abeeffc --- /dev/null +++ b/tests/ino2cpp/multifiles/foo.pde @@ -0,0 +1,13 @@ + +void setup() { + barFunc(); + fooFunc(); +} + +void loop() { + +} + +char* fooFunc() { + +} \ No newline at end of file diff --git a/tests/test_ino2cpp.py b/tests/test_ino2cpp.py new file mode 100644 index 00000000..0a6efc77 --- /dev/null +++ b/tests/test_ino2cpp.py @@ -0,0 +1,41 @@ +# Copyright 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import listdir +from os.path import dirname, isdir, join, normpath + +import pytest + +from platformio import util + + +def pytest_generate_tests(metafunc): + if "piotest_dir" not in metafunc.fixturenames: + return + test_dir = normpath(join(dirname(__file__), "ino2cpp")) + test_dirs = [] + for name in listdir(test_dir): + if isdir(join(test_dir, name)): + test_dirs.append(join(test_dir, name)) + test_dirs.sort() + metafunc.parametrize("piotest_dir", test_dirs) + + +@pytest.mark.examples +def test_ci(platformio_setup, piotest_dir): + result = util.exec_command( + ["platformio", "--force", "ci", piotest_dir, "-b", "uno"] + ) + if result['returncode'] != 0: + pytest.fail(result)