Refactor INO to CPP converter

HISTORY.rst

@@ -73,6 +73,7 @@ PlatformIO 3.0
 * Print human-readable information when processing environments without
   ``-v, --verbose`` option
   (`issue #721 <https://github.com/platformio/platformio/issues/721>`_)
+* Improved INO to CPP converter
 * Added ``license`` field to `library.json <http://docs.platformio.org/en/latest/librarymanager/config.html>`__
   (`issue #522 <https://github.com/platformio/platformio/issues/522>`_)
 * Warn about unknown options in project configuration file ``platformio.ini``

docs/platforms/creating_platform.rst

@@ -198,10 +198,7 @@ Manifest File ``platform.json``
       "description": "My custom development platform",
       "url": "http://example.com",
       "homepage": "http://platformio.org/platforms/myplatform",
-      "license": {
+      "license": "Apache-2.0",
-        "type": "Apache-2.0",
-        "url": "http://opensource.org/licenses/apache2.0.php"
-      },
       "engines": {
         "platformio": "~3.0.0",
         "scons": ">=2.3.0,<2.6.0"

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 0, "0a13")
+VERSION = (3, 0, "0b1")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/builder/tools/piolib.py

@@ -134,8 +134,8 @@ class LibBuilderBase(object):  # pylint: disable=too-many-instance-attributes
     def build_dir(self):
         return join("$BUILD_DIR", "lib", basename(self.path))
 
-    def get_inc_dirs(self, use_build_dir=False):
+    def get_inc_dirs(self):
-        return [self.build_dir if use_build_dir else self.src_dir]
+        return [self.src_dir]
 
     @property
     def build_flags(self):
@@ -290,8 +290,7 @@ class LibBuilderBase(object):  # pylint: disable=too-many-instance-attributes
                 self.env.AppendUnique(**{key: lb.env.get(key)})
 
         if not self._built_node:
-            self.env.AppendUnique(CPPPATH=self.get_inc_dirs(
+            self.env.AppendUnique(CPPPATH=self.get_inc_dirs())
-                use_build_dir=True))
             if self.lib_archive:
                 self._built_node = self.env.BuildLibrary(
                     self.build_dir, self.src_dir, self.src_filter)
@@ -345,12 +344,11 @@ class ArduinoLibBuilder(LibBuilderBase):
                 manifest[key.strip()] = value.strip()
         return manifest
 
-    def get_inc_dirs(self, use_build_dir=False):
+    def get_inc_dirs(self):
-        inc_dirs = LibBuilderBase.get_inc_dirs(self, use_build_dir)
+        inc_dirs = LibBuilderBase.get_inc_dirs(self)
         if not isdir(join(self.path, "utility")):
             return inc_dirs
-        inc_dirs.append(
+        inc_dirs.append(join(self.path, "utility"))
-            join(self.build_dir if use_build_dir else self.path, "utility"))
         return inc_dirs
 
     @property
@@ -382,8 +380,8 @@ class MbedLibBuilder(LibBuilderBase):
             return join(self.path, "source")
         return LibBuilderBase.src_dir.fget(self)
 
-    def get_inc_dirs(self, use_build_dir=False):
+    def get_inc_dirs(self):
-        inc_dirs = LibBuilderBase.get_inc_dirs(self, use_build_dir)
+        inc_dirs = LibBuilderBase.get_inc_dirs(self)
         if self.path not in inc_dirs:
             inc_dirs.append(self.path)
         for p in self._manifest.get("extraIncludes", []):
@@ -451,12 +449,11 @@ class PlatformIOLibBuilder(LibBuilderBase):
             ilist = [i.strip() for i in ilist.split(",")]
         return item.lower() in [i.lower() for i in ilist]
 
-    def get_inc_dirs(self, use_build_dir=False):
+    def get_inc_dirs(self):
-        inc_dirs = LibBuilderBase.get_inc_dirs(self, use_build_dir)
+        inc_dirs = LibBuilderBase.get_inc_dirs(self)
         for path in self.env['CPPPATH']:
             if path not in self.envorigin['CPPPATH']:
-                inc_dirs.append(
+                inc_dirs.append(self.env.subst(path))
-                    path if use_build_dir else self.env.subst(path))
         return inc_dirs
 
 
@@ -499,8 +496,9 @@ def GetLibBuilders(env):
             try:
                 lb = LibBuilderFactory.new(env, join(libs_dir, item))
             except ValueError:
-                sys.stderr.write("Skip library with broken manifest: %s\n" %
+                if verbose:
-                                 join(libs_dir, item))
+                    sys.stderr.write("Skip library with broken manifest: %s\n"
+                                     % join(libs_dir, item))
                 continue
             if _check_lib_builder(lb):
                 items += (lb, )

platformio/builder/tools/piomisc.py

@@ -19,7 +19,8 @@ import re
 import sys
 from glob import glob
 from os import environ, remove
-from os.path import isfile, join
+from os.path import basename, isfile, join
+from tempfile import mkstemp
 
 from SCons.Action import Action
 from SCons.Script import ARGUMENTS
@@ -38,106 +39,118 @@ class InoToCPPConverter(object):
     DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
     PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
 
-    def __init__(self, nodes):
+    def __init__(self, env):
-        self.nodes = nodes
+        self.env = env
+        self._main_ino = None
 
     def is_main_node(self, contents):
         return self.DETECTMAIN_RE.search(contents)
 
-    def _parse_prototypes(self, file_path, contents):
+    def convert(self, nodes):
+        contents = self.merge(nodes)
+        if not contents:
+            return
+        return self.process(contents)
+
+    def merge(self, nodes):
+        lines = []
+        for node in nodes:
+            contents = node.get_text_contents()
+            _lines = [
+                '# 1 "%s"' % node.get_path().replace("\\", "/"), contents
+            ]
+            if self.is_main_node(contents):
+                lines = _lines + lines
+                self._main_ino = node.get_path()
+            else:
+                lines.extend(_lines)
+
+        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
+
+    def process(self, contents):
+        out_file = self._main_ino + ".cpp"
+        assert self._gcc_preprocess(contents, out_file)
+        with open(out_file) as fp:
+            contents = fp.read()
+        with open(out_file, "w") as fp:
+            fp.write(self.append_prototypes(contents))
+        return out_file
+
+    def _gcc_preprocess(self, contents, out_file):
+        tmp_path = mkstemp()[1]
+        with open(tmp_path, "w") as fp:
+            fp.write(contents)
+            fp.close()
+        self.env.Execute(
+            self.env.VerboseAction(
+                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
+                    out_file, tmp_path), "Converting " + basename(
+                        out_file[:-4])))
+        remove(tmp_path)
+        return isfile(out_file)
+
+    def _parse_prototypes(self, contents):
         prototypes = []
         reserved_keywords = set(["if", "else", "while"])
         for match in self.PROTOTYPE_RE.finditer(contents):
             if (set([match.group(2).strip(), match.group(3).strip()]) &
                     reserved_keywords):
                 continue
-            prototypes.append({"path": file_path, "match": match})
+            prototypes.append(match)
         return prototypes
 
-    def append_prototypes(self, file_path, contents, prototypes):
+    @staticmethod
-        result = []
+    def _get_total_lines(contents):
+        total = 0
+        for line in contents.split("\n")[::-1]:
+            if line.startswith("#"):
+                tokens = line.split(" ", 3)
+                if len(tokens) > 2 and tokens[1].isdigit():
+                    return int(tokens[1]) + total
+            total += 1
+        return total
+
+    def append_prototypes(self, contents):
+        prototypes = self._parse_prototypes(contents)
         if not prototypes:
-            return result
+            return contents
-
-        prototype_names = set(
-            [p['match'].group(3).strip() for p in prototypes])
-        split_pos = prototypes[0]['match'].start()
-        for item in prototypes:
-            if item['path'] == file_path:
-                split_pos = item['match'].start()
-                break
 
+        prototype_names = set([m.group(3).strip() for m in prototypes])
+        split_pos = prototypes[0].start()
         match_ptrs = re.search(self.PROTOPTRS_TPLRE %
                                ("|".join(prototype_names)),
                                contents[:split_pos], re.M)
         if match_ptrs:
             split_pos = contents.rfind("\n", 0, match_ptrs.start())
 
+        result = []
         result.append(contents[:split_pos].strip())
-        result.append("%s;" %
+        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-                      ";\n".join([p['match'].group(1) for p in prototypes]))
         result.append('#line %d "%s"' %
-                      (contents.count("\n", 0, split_pos) + 2,
+                      (self._get_total_lines(contents[:split_pos]),
-                       file_path.replace("\\", "/")))
+                       self._main_ino.replace("\\", "/")))
         result.append(contents[split_pos:].strip())
-
-        return result
-
-    def convert(self):
-        prototypes = []
-        data = []
-        for node in self.nodes:
-            ino_contents = node.get_text_contents()
-            prototypes += self._parse_prototypes(node.get_path(), ino_contents)
-
-            item = (node.get_path(), ino_contents)
-            if self.is_main_node(ino_contents):
-                data = [item] + data
-            else:
-                data.append(item)
-
-        if not data:
-            return None
-
-        result = ["#include <Arduino.h>"]
-        is_first = True
-        for file_path, contents in data:
-            result.append('#line 1 "%s"' % file_path.replace("\\", "/"))
-
-            if is_first and prototypes:
-                result += self.append_prototypes(file_path, contents,
-                                                 prototypes)
-            else:
-                result.append(contents)
-            is_first = False
-
         return "\n".join(result)
 
 
 def ConvertInoToCpp(env):
 
-    def delete_tmpcpp_file(file_):
+    def _delete_file(path):
         try:
-            remove(file_)
+            if isfile(path):
+                remove(path)
         except:  # pylint: disable=bare-except
-            if isfile(file_):
+            if path and isfile(path):
-                print("Warning: Could not remove temporary file '%s'. "
+                sys.stderr.write(
-                      "Please remove it manually." % file_)
+                    "Warning: Could not remove temporary file '%s'. "
+                    "Please remove it manually.\n" % path)
 
     ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) +
                  env.Glob(join("$PROJECTSRC_DIR", "*.pde")))
+    c = InoToCPPConverter(env)
+    out_file = c.convert(ino_nodes)
 
-    c = InoToCPPConverter(ino_nodes)
+    atexit.register(_delete_file, out_file)
-    data = c.convert()
-
-    if not data:
-        return
-
-    tmpcpp_file = join(env.subst("$PROJECTSRC_DIR"), "tmp_ino_to.cpp")
-    with open(tmpcpp_file, "w") as f:
-        f.write(data)
-
-    atexit.register(delete_tmpcpp_file, tmpcpp_file)
 
 
 def DumpIDEData(env):
@@ -146,14 +159,7 @@ def DumpIDEData(env):
         includes = []
 
         for item in env_.get("CPPPATH", []):
-            invardir = False
+            includes.append(env_.subst(item))
-            for vardiritem in env_.get("VARIANT_DIRS", []):
-                if item == vardiritem[0]:
-                    includes.append(env_.subst(vardiritem[1]))
-                    invardir = True
-                    break
-            if not invardir:
-                includes.append(env_.subst(item))
 
         # installed libs
         for lb in env.GetLibBuilders():

platformio/builder/tools/platformio.py

@@ -198,11 +198,6 @@ def MatchSourceFiles(env, src_dir, src_filter=None):
     return sorted(list(matches))
 
 
-def VariantDirWrap(env, variant_dir, src_dir, duplicate=False):
-    DefaultEnvironment().Append(VARIANT_DIRS=[(variant_dir, src_dir)])
-    env.VariantDir(variant_dir, src_dir, duplicate)
-
-
 def CollectBuildFiles(env,
                       variant_dir,
                       src_dir,
@@ -222,7 +217,7 @@ def CollectBuildFiles(env,
 
         if _var_dir not in variants:
             variants.append(_var_dir)
-            env.VariantDirWrap(_var_dir, _src_dir, duplicate)
+            env.VariantDir(_var_dir, _src_dir, duplicate)
 
         if env.IsFileWithExt(item, SRC_BUILD_EXT):
             sources.append(env.File(join(_var_dir, basename(item))))
@@ -283,7 +278,6 @@ def generate(env):
     env.AddMethod(ProcessUnFlags)
     env.AddMethod(IsFileWithExt)
     env.AddMethod(MatchSourceFiles)
-    env.AddMethod(VariantDirWrap)
     env.AddMethod(CollectBuildFiles)
     env.AddMethod(BuildFrameworks)
     env.AddMethod(BuildLibrary)

platformio/commands/platform.py

@@ -165,7 +165,7 @@ def platform_show(platform):
     if p.homepage:
         click.echo("Home: %s" % p.homepage)
     if p.license:
-        click.echo("License: %s" % p.license.get("type"))
+        click.echo("License: %s" % p.license)
     if p.frameworks:
         click.echo("Frameworks: %s" % ", ".join(p.frameworks.keys()))
 

platformio/managers/package.py

@@ -27,7 +27,6 @@ from platformio.downloader import FileDownloader
 from platformio.unpacker import FileUnpacker
 from platformio.vcsclient import VCSClientFactory
 
-
 # pylint: disable=too-many-arguments
 
 
@@ -504,7 +503,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 label=manifest['name'])
         return True
 
-    def update(self, name, requirements=None, only_check=False):
+    def update(self,  # pylint: disable=too-many-return-statements
+               name,
+               requirements=None,
+               only_check=False):
         name, requirements, url = self.parse_pkg_name(name, requirements)
         package_dir = self.get_package_dir(name, requirements, url)
         if not package_dir:

tests/ino2cpp/basic/basic.ino

@@ -0,0 +1,18 @@
+
+struct MyItem {
+   byte  foo[50];
+   int   bar;
+};
+
+void setup() {
+    struct MyItem item1;
+    myFunction(&item1);
+}
+
+void loop() {
+
+}
+
+void myFunction(struct MyItem *item) {
+
+}

tests/ino2cpp/multifiles/bar.ino

@@ -0,0 +1,3 @@
+void barFunc () { // my comment
+
+}

tests/ino2cpp/multifiles/foo.pde

@@ -0,0 +1,13 @@
+
+void setup() {
+    barFunc();
+    fooFunc();
+}
+
+void loop() {
+
+}
+
+char* fooFunc() {
+
+}

tests/test_ino2cpp.py

@@ -0,0 +1,41 @@
+# Copyright 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os import listdir
+from os.path import dirname, isdir, join, normpath
+
+import pytest
+
+from platformio import util
+
+
+def pytest_generate_tests(metafunc):
+    if "piotest_dir" not in metafunc.fixturenames:
+        return
+    test_dir = normpath(join(dirname(__file__), "ino2cpp"))
+    test_dirs = []
+    for name in listdir(test_dir):
+        if isdir(join(test_dir, name)):
+            test_dirs.append(join(test_dir, name))
+    test_dirs.sort()
+    metafunc.parametrize("piotest_dir", test_dirs)
+
+
+@pytest.mark.examples
+def test_ci(platformio_setup, piotest_dir):
+    result = util.exec_command(
+        ["platformio", "--force", "ci", piotest_dir, "-b", "uno"]
+    )
+    if result['returncode'] != 0:
+        pytest.fail(result)