New Library Build System: intelligent dependency finder that interprets C Preprocessor conditional macros // Resolve #432

HISTORY.rst

@@ -13,6 +13,9 @@ PlatformIO 3.0
   (`issue #479 <https://github.com/platformio/platformio/issues/479>`_)
 * Unit Testing for Embedded (`docs <http://docs.platformio.org/en/latest/platforms/unit_testing.html>`__)
   (`issue #408 <https://github.com/platformio/platformio/issues/408>`_)
+* New Library Build System: intelligent dependency finder that interprets
+  C Preprocessor conditional macros
+  (`issue #432 <https://github.com/platformio/platformio/issues/432>`_)
 
 PlatformIO 2.0
 --------------

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 0, "0.dev1")
+VERSION = (3, 0, "0.dev2")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/builder/main.py

@@ -63,7 +63,8 @@ commonvars.AddVariables(
 DefaultEnvironment(
     tools=[
         "gcc", "g++", "as", "ar", "gnulink",
-        "platformio", "devplatform", "piotest", "pioupload", "pioar", "piomisc"
+        "platformio", "devplatform",
+        "piolib", "piotest", "pioupload", "pioar", "piomisc"
     ],
     toolpath=[join(util.get_source_dir(), "builder", "tools")],
     variables=commonvars,

platformio/builder/tools/piolib.py

@@ -0,0 +1,242 @@
+# Copyright 2014-present Ivan Kravets <me@ikravets.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+from os.path import basename, commonprefix, isdir, isfile, join
+from sys import modules
+
+import SCons.Scanner
+
+from platformio import util
+
+
+class LibBuilderFactory(object):
+
+    @staticmethod
+    def new(env, path):
+        clsname = "UnknownLibBuilder"
+        if isfile(join(path, "library.json")):
+            clsname = "PlatformIOLibBuilder"
+        else:
+            env_frameworks = [
+                f.lower().strip() for f in env.get("FRAMEWORK", "").split(",")]
+            used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
+            common_frameworks = set(env_frameworks) & set(used_frameworks)
+            if common_frameworks:
+                clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
+            elif used_frameworks:
+                clsname = "%sLibBuilder" % used_frameworks[0].title()
+
+        obj = getattr(modules[__name__], clsname)(env, path)
+        assert isinstance(obj, LibBuilderBase)
+        return obj
+
+    @staticmethod
+    def get_used_frameworks(env, path):
+        if any([isfile(join(path, fname))
+                for fname in ("library.properties", "keywords.txt")]):
+            return ["arduino"]
+
+        if isfile(join(path, "module.json")):
+            return ["mbed"]
+
+        # check source files
+        for root, _, files in os.walk(path, followlinks=True):
+            for fname in files:
+                if not env.IsFileWithExt(fname, ("c", "cpp", "h")):
+                    continue
+                with open(join(root, fname)) as f:
+                    content = f.read()
+                    if "Arduino.h" in content:
+                        return ["arduino"]
+                    elif "mbed.h" in content:
+                        return ["mbed"]
+        return []
+
+
+class LibBuilderBase(object):
+
+    def __init__(self, env, path):
+        self.env = env
+        self.path = path
+        self._is_built = False
+
+    def __repr__(self):
+        return "%s(%r)" % (self.__class__, self.path)
+
+    def __contains__(self, path):
+        return commonprefix((self.path, path)) == self.path
+
+    @property
+    def name(self):
+        return basename(self.path)
+
+    @property
+    def src_filter(self):
+        return " ".join([
+            "+<*>", "-<.git%s>" % os.sep, "-<svn%s>" % os.sep,
+            "-<example%s>" % os.sep, "-<examples%s>" % os.sep,
+            "-<test%s>" % os.sep, "-<tests%s>" % os.sep
+        ])
+
+    @property
+    def src_dir(self):
+        return (join(self.path, "src") if isdir(join(self.path, "src"))
+                else self.path)
+
+    @property
+    def build_dir(self):
+        return join("$BUILD_DIR", "lib", self.name)
+
+    @property
+    def is_built(self):
+        return self._is_built
+
+    def get_path_dirs(self, use_build_dir=False):
+        return [self.build_dir if use_build_dir else self.src_dir]
+
+    def build(self):
+        print "Depends on: %s" % self.name
+        assert self._is_built is False
+        self._is_built = True
+        return self.env.BuildLibrary(self.build_dir, self.src_dir)
+
+
+class UnknownLibBuilder(LibBuilderBase):
+    pass
+
+
+class ArduinoLibBuilder(LibBuilderBase):
+
+    def get_path_dirs(self, use_build_dir=False):
+        path_dirs = LibBuilderBase.get_path_dirs(self, use_build_dir)
+        if not isdir(join(self.src_dir, "utility")):
+            return path_dirs
+        path_dirs.append(join(self.build_dir if use_build_dir
+                              else self.src_dir, "utility"))
+        return path_dirs
+
+
+class MbedLibBuilder(LibBuilderBase):
+
+    def __init__(self, env, path):
+        self.module_json = {}
+        if isfile(join(path, "module.json")):
+            self.module_json = util.load_json(join(path, "module.json"))
+
+        LibBuilderBase.__init__(self, env, path)
+
+    @property
+    def src_dir(self):
+        if isdir(join(self.path, "source")):
+            return join(self.path, "source")
+        return LibBuilderBase.src_dir.fget(self)
+
+    def get_path_dirs(self, use_build_dir=False):
+        path_dirs = LibBuilderBase.get_path_dirs(self, use_build_dir)
+        for p in self.module_json.get("extraIncludes", []):
+            if p.startswith("source/"):
+                p = p[7:]
+            path_dirs.append(
+                join(self.build_dir if use_build_dir else self.src_dir, p))
+        return path_dirs
+
+
+class PlatformIOLibBuilder(LibBuilderBase):
+
+    def __init__(self, env, path):
+        self.library_json = {}
+        if isfile(join(path, "library.json")):
+            self.library_json = util.load_json(join(path, "library.json"))
+
+        LibBuilderBase.__init__(self, env, path)
+
+
+def find_deps(env, scanner, path_dirs, src_dir, src_filter):
+    result = []
+    for item in env.MatchSourceFiles(src_dir, src_filter):
+        result.extend(env.File(join(src_dir, item)).get_implicit_deps(
+            env, scanner, path_dirs))
+    return result
+
+
+def find_and_build_deps(env, lib_builders, scanner,
+                        src_dir, src_filter):
+    path_dirs = tuple()
+    built_path_dirs = tuple()
+    for lb in lib_builders:
+        items = [env.Dir(d) for d in lb.get_path_dirs()]
+        if lb.is_built:
+            built_path_dirs += tuple(items)
+        else:
+            path_dirs += tuple(items)
+    path_dirs = built_path_dirs + path_dirs
+
+    target_lbs = []
+    deps = find_deps(env, scanner, path_dirs, src_dir, src_filter)
+    for d in deps:
+        for lb in lib_builders:
+            if d.get_abspath() in lb:
+                if lb not in target_lbs and not lb.is_built:
+                    target_lbs.append(lb)
+                break
+
+    libs = []
+    # add build dirs to global CPPPATH
+    for lb in target_lbs:
+        env.Append(
+            CPPPATH=lb.get_path_dirs(use_build_dir=True)
+        )
+    # start builder
+    for lb in target_lbs:
+        libs.append(lb.build())
+
+    if env.get("LIB_DFCYCLIC", "").lower() == "true":
+        for lb in target_lbs:
+            libs.extend(find_and_build_deps(
+                env, lib_builders, scanner, lb.src_dir, lb.src_filter))
+
+    return libs
+
+
+def BuildDependentLibraries(env, src_dir):
+    lib_builders = []
+    libs_dirs = [env.subst(d) for d in env.get("LIBSOURCE_DIRS", [])
+                 if isdir(env.subst(d))]
+    for libs_dir in libs_dirs:
+        if not isdir(libs_dir):
+            continue
+        for item in sorted(os.listdir(libs_dir)):
+            if isdir(join(libs_dir, item)):
+                lib_builders.append(
+                    LibBuilderFactory.new(env, join(libs_dir, item)))
+
+    print "Looking for dependencies..."
+    print "Library locations: " + ", ".join(libs_dirs)
+    print "Collecting %d libraries" % len(lib_builders)
+
+    return find_and_build_deps(
+        env, lib_builders, SCons.Scanner.C.CScanner(),
+        src_dir, env.get("SRC_FILTER"))
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(BuildDependentLibraries)
+    return env

platformio/builder/tools/platformio.py

@@ -16,8 +16,8 @@ from __future__ import absolute_import
 
 import re
 from glob import glob
-from os import listdir, sep, walk
+from os import sep, walk
-from os.path import basename, dirname, isdir, isfile, join, normpath, realpath
+from os.path import basename, dirname, isdir, join, normpath, realpath
 
 from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript
 from SCons.Util import case_sensitive_suffixes
@@ -26,11 +26,7 @@ from platformio.util import pioversion_to_intstr
 
 SRC_BUILD_EXT = ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]
 SRC_HEADER_EXT = ["h", "hpp"]
-SRC_DEFAULT_FILTER = " ".join([
+SRC_FILTER_DEFAULT = " ".join(["+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep])
-    "+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep,
-    "-<example%s>" % sep, "-<examples%s>" % sep,
-    "-<test%s>" % sep, "-<tests%s>" % sep
-])
 
 
 def BuildProgram(env):
@@ -173,7 +169,7 @@ def IsFileWithExt(env, file_, ext):  # pylint: disable=W0613
     return False
 
 
-def MatchSourceFiles(env, src_dir, src_filter):
+def MatchSourceFiles(env, src_dir, src_filter=None):
 
     SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
 
@@ -181,6 +177,9 @@ def MatchSourceFiles(env, src_dir, src_filter):
         if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT):
             items.add(item.replace(src_dir + sep, ""))
 
+    src_dir = env.subst(src_dir)
+    src_filter = src_filter or SRC_FILTER_DEFAULT
+
     matches = set()
     # correct fs directory separator
     src_filter = src_filter.replace("/", sep).replace("\\", sep)
@@ -214,8 +213,7 @@ def CollectBuildFiles(env, variant_dir, src_dir,
     if src_dir.endswith(sep):
         src_dir = src_dir[:-1]
 
-    for item in env.MatchSourceFiles(
+    for item in env.MatchSourceFiles(src_dir, src_filter):
-            src_dir, src_filter or SRC_DEFAULT_FILTER):
         _reldir = dirname(item)
         _src_dir = join(src_dir, _reldir) if _reldir else src_dir
         _var_dir = join(variant_dir, _reldir) if _reldir else variant_dir
@@ -265,149 +263,6 @@ def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
     )
 
 
-def BuildDependentLibraries(env, src_dir):  # pylint: disable=R0914
-
-    INCLUDES_RE = re.compile(
-        r"^\s*#include\s+(\<|\")([^\>\"\']+)(?:\>|\")", re.M)
-    LIBSOURCE_DIRS = [env.subst(d) for d in env.get("LIBSOURCE_DIRS", [])]
-
-    # start internal prototypes
-
-    class IncludeFinder(object):
-
-        def __init__(self, base_dir, name, is_system=False):
-            self.base_dir = base_dir
-            self.name = name
-            self.is_system = is_system
-
-            self._inc_path = None
-            self._lib_dir = None
-            self._lib_name = None
-
-        def getIncPath(self):
-            return self._inc_path
-
-        def getLibDir(self):
-            return self._lib_dir
-
-        def getLibName(self):
-            return self._lib_name
-
-        def run(self):
-            if not self.is_system and self._find_in_local():
-                return True
-            return self._find_in_system()
-
-        def _find_in_local(self):
-            if isfile(join(self.base_dir, self.name)):
-                self._inc_path = join(self.base_dir, self.name)
-                return True
-            else:
-                return False
-
-        def _find_in_system(self):
-            for lsd_dir in LIBSOURCE_DIRS:
-                if not isdir(lsd_dir):
-                    continue
-
-                for ld in env.get("LIB_USE", []) + sorted(listdir(lsd_dir)):
-                    if not isdir(join(lsd_dir, ld)):
-                        continue
-
-                    inc_path = normpath(join(lsd_dir, ld, self.name))
-                    try:
-                        lib_dir = inc_path[:inc_path.index(
-                            sep, len(lsd_dir) + 1)]
-                    except ValueError:
-                        continue
-                    lib_name = basename(lib_dir)
-
-                    # ignore user's specified libs
-                    if lib_name in env.get("LIB_IGNORE", []):
-                        continue
-
-                    if not isfile(inc_path):
-                        # if source code is in "src" dir
-                        lib_dir = join(lsd_dir, lib_name, "src")
-                        inc_path = join(lib_dir, self.name)
-
-                    if isfile(inc_path):
-                        self._lib_dir = lib_dir
-                        self._lib_name = lib_name
-                        self._inc_path = inc_path
-                        return True
-            return False
-
-    def _get_dep_libs(src_dir):
-        state = {
-            "paths": set(),
-            "libs": set(),
-            "ordered": set()
-        }
-
-        state = _process_src_dir(state, env.subst(src_dir))
-
-        result = []
-        for item in sorted(state['ordered'], key=lambda s: s[0]):
-            result.append((item[1], item[2]))
-        return result
-
-    def _process_src_dir(state, src_dir):
-        for root, _, files in walk(src_dir, followlinks=True):
-            for f in files:
-                if env.IsFileWithExt(f, SRC_BUILD_EXT + SRC_HEADER_EXT):
-                    state = _parse_includes(state, env.File(join(root, f)))
-        return state
-
-    def _parse_includes(state, node):
-        skip_includes = ("arduino.h", "energia.h")
-        matches = INCLUDES_RE.findall(node.get_text_contents())
-        for (inc_type, inc_name) in matches:
-            base_dir = dirname(node.get_abspath())
-            if inc_name.lower() in skip_includes:
-                continue
-            if join(base_dir, inc_name) in state['paths']:
-                continue
-            else:
-                state['paths'].add(join(base_dir, inc_name))
-
-            finder = IncludeFinder(base_dir, inc_name, inc_type == "<")
-            if finder.run():
-                _parse_includes(state, env.File(finder.getIncPath()))
-
-                _lib_dir = finder.getLibDir()
-                if _lib_dir and _lib_dir not in state['libs']:
-                    state['ordered'].add((
-                        len(state['ordered']) + 1, finder.getLibName(),
-                        _lib_dir))
-                    state['libs'].add(_lib_dir)
-
-                    if env.subst("$LIB_DFCYCLIC").lower() == "true":
-                        state = _process_src_dir(state, _lib_dir)
-        return state
-
-    # end internal prototypes
-
-    deplibs = _get_dep_libs(src_dir)
-    for l, ld in deplibs:
-        env.Append(
-            CPPPATH=[join("$BUILD_DIR", l)]
-        )
-        # add automatically "utility" dir from the lib (Arduino issue)
-        if isdir(join(ld, "utility")):
-            env.Append(
-                CPPPATH=[join("$BUILD_DIR", l, "utility")]
-            )
-
-    libs = []
-    for (libname, inc_dir) in deplibs:
-        lib = env.BuildLibrary(
-            join("$BUILD_DIR", libname), inc_dir)
-        env.Clean(libname, lib)
-        libs.append(lib)
-    return libs
-
-
 def exists(_):
     return True
 
@@ -422,5 +277,4 @@ def generate(env):
     env.AddMethod(CollectBuildFiles)
     env.AddMethod(BuildFrameworks)
     env.AddMethod(BuildLibrary)
-    env.AddMethod(BuildDependentLibraries)
     return env