Allow to exclude/include source files from build process using src_filter // Resolve #240

HISTORY.rst

@@ -4,6 +4,9 @@ Release History
 2.2.0 (2015-??-??)
 ------------------
 
+* Allowed to exclude/include source files from build process using
+  `src_filter <http://docs.platformio.org/en/latest/projectconf.html#src-filter>`__
+  (`issue #240 <https://github.com/platformio/platformio/issues/240>`_)
 * Launch own extra script before firmware building/uploading processes
   (`issue #239 <https://github.com/platformio/platformio/issues/239>`_)
 * Specify own path to the linker script (ld) using

docs/envvars.rst

@@ -77,6 +77,13 @@ PLATFORMIO_SRCBUILD_FLAGS
 
 Allows to set :ref:`projectconf` option :ref:`projectconf_srcbuild_flags`.
 
+.. _envvar_PLATFORMIO_SRC_FILTER:
+
+PLATFORMIO_SRC_FILTER
+~~~~~~~~~~~~~~~~~~~~~
+
+Allows to set :ref:`projectconf` option :ref:`projectconf_src_filter`.
+
 .. _envvar_PLATFORMIO_LDF_CYCLIC:
 
 PLATFORMIO_LDF_CYCLIC

docs/projectconf.rst

@@ -319,6 +319,28 @@ but will be applied only for the project source code from
 This option can be set by global environment variable
 :ref:`envvar_PLATFORMIO_SRCBUILD_FLAGS`.
 
+.. _projectconf_src_filter:
+
+``src_filter``
+^^^^^^^^^^^^^^
+
+This option allows to specify which source files should be included/excluded
+from build process. Filter supports 2 templates:
+
+* ``+<PATH>`` include template
+* ``-<PATH>`` exclude template
+
+``PATH`` MAST BE related from :ref:`projectconf_pio_src_dir`. All patterns will
+be applied in theirs order.
+`GLOB Patterns <http://en.wikipedia.org/wiki/Glob_(programming)>`_ are allowed.
+
+By default, ``src_filter`` is predefined to
+``+<*> -<.git/> -<svn/> -<examples/>``, which means "includes ALL files, then
+exclude ``.git`` and ``svn`` repository folders and exclude ``examples`` folder.
+
+This option can be set by global environment variable
+:ref:`envvar_PLATFORMIO_SRC_FILTER`.
+
 ``install_libs``
 ^^^^^^^^^^^^^^^^
 

platformio/__init__.py

@@ -1,7 +1,7 @@
 # Copyright (C) Ivan Kravets <me@ikravets.com>
 # See LICENSE for details.
 
-VERSION = (2, 2, "0.dev2")
+VERSION = (2, 2, "0.dev3")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/builder/main.py

@@ -39,6 +39,7 @@ commonvars.AddVariables(
     ("FRAMEWORK",),
     ("BUILD_FLAGS",),
     ("SRCBUILD_FLAGS",),
+    ("SRC_FILTER",),
     ("IGNORE_LIBS",),
     ("USE_LIBS",),
     ("LDF_CYCLIC",),
@@ -57,7 +58,7 @@ commonvars.AddVariables(
 DefaultEnvironment(
     tools=[
         "gcc", "g++", "as", "ar", "gnulink",
-        "platformio", "pioupload", "pioar"
+        "platformio", "pioupload", "pioar", "piomisc"
     ],
     toolpath=[join("$PIOBUILDER_DIR", "tools")],
     variables=commonvars,
@@ -74,6 +75,7 @@ DefaultEnvironment(
     PIOPACKAGES_DIR=join("$PIOHOME_DIR", "packages"),
 
     BUILD_DIR=join("$PIOENVS_DIR", "$PIOENV"),
+    BUILDSRC_DIR=join("$BUILD_DIR", "ProjectSrc"),
     LIBSOURCE_DIRS=[
         "$PROJECTLIB_DIR",
         util.get_lib_dir(),

platformio/builder/scripts/frameworks/spl.py

@@ -38,10 +38,7 @@ env.Append(
 envsafe = env.Clone()
 
 envsafe.Append(
-    CPPPATH=[
+    CPPPATH=["$BUILDSRC_DIR"],
-        join("$BUILD_DIR", "src")
-    ],
-
     CPPDEFINES=[
         "USE_STDPERIPH_DRIVER"
     ]
@@ -52,22 +49,22 @@ envsafe.Append(
 #
 
 extra_flags = env.get("BOARD_OPTIONS", {}).get("build", {}).get("extra_flags")
-ignore_files = []
+src_filter_patterns = ["+<*>"]
 if "STM32F40_41xxx" in extra_flags:
-    ignore_files += ["stm32f4xx_fmc.c"]
+    src_filter_patterns += ["-<stm32f4xx_fmc.c>"]
 if "STM32F427_437xx" in extra_flags:
-    ignore_files += ["stm32f4xx_fsmc.c"]
+    src_filter_patterns += ["-<stm32f4xx_fsmc.c>"]
 elif "STM32F303xC" in extra_flags:
-    ignore_files += ["stm32f30x_hrtim.c"]
+    src_filter_patterns += ["-<stm32f30x_hrtim.c>"]
 elif "STM32L1XX_MD" in extra_flags:
-    ignore_files += ["stm32l1xx_flash_ramfunc.c"]
+    src_filter_patterns += ["-<stm32l1xx_flash_ramfunc.c>"]
 
 libs = []
 libs.append(envsafe.BuildLibrary(
     join("$BUILD_DIR", "FrameworkSPL"),
     join("$PLATFORMFW_DIR", "${BOARD_OPTIONS['build']['core']}", "variants",
          "${BOARD_OPTIONS['build']['variant']}", "src"),
-    ignore_files
+    src_filter=" ".join(src_filter_patterns)
 ))
 
 env.Append(LIBS=libs)

platformio/builder/tools/piomisc.py

@@ -0,0 +1,165 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+import atexit
+import re
+from glob import glob
+from os import remove
+from os.path import basename, join
+
+
+class InoToCPPConverter(object):
+
+    PROTOTYPE_RE = re.compile(
+        r"""^(
+        (\s*[a-z_\d]+){1,2}         # return type
+        (\s+[a-z_\d]+\s*)           # name of prototype
+        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
+        )\s*\{                      # must end with {
+        """,
+        re.X | re.M | re.I
+    )
+
+    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
+
+    STRIPCOMMENTS_RE = re.compile(r"(/\*.*?\*/|(^|\s+)//[^\r\n]*$)",
+                                  re.M | re.S)
+
+    def __init__(self, nodes):
+        self.nodes = nodes
+
+    def is_main_node(self, contents):
+        return self.DETECTMAIN_RE.search(contents)
+
+    @staticmethod
+    def _replace_comments_callback(match):
+        if "\n" in match.group(1):
+            return "\n" * match.group(1).count("\n")
+        else:
+            return " "
+
+    def _parse_prototypes(self, contents):
+        prototypes = []
+        reserved_keywords = set(["if", "else", "while"])
+        for item in self.PROTOTYPE_RE.findall(contents):
+            if set([item[1].strip(), item[2].strip()]) & reserved_keywords:
+                continue
+            prototypes.append(item[0])
+        return prototypes
+
+    def append_prototypes(self, fname, contents, prototypes):
+        contents = self.STRIPCOMMENTS_RE.sub(self._replace_comments_callback,
+                                             contents)
+        result = []
+        is_appended = False
+        linenum = 0
+        for line in contents.splitlines():
+            linenum += 1
+            line = line.strip()
+
+            if not is_appended and line and not line.startswith("#"):
+                is_appended = True
+                result.append("%s;" % ";\n".join(prototypes))
+                result.append('#line %d "%s"' % (linenum, fname))
+
+            result.append(line)
+
+        return result
+
+    def convert(self):
+        prototypes = []
+        data = []
+        for node in self.nodes:
+            ino_contents = node.get_text_contents()
+            prototypes += self._parse_prototypes(ino_contents)
+
+            item = (basename(node.get_path()), ino_contents)
+            if self.is_main_node(ino_contents):
+                data = [item] + data
+            else:
+                data.append(item)
+
+        if not data:
+            return None
+
+        result = ["#include <Arduino.h>"]
+        is_first = True
+
+        for name, contents in data:
+            if is_first and prototypes:
+                result += self.append_prototypes(name, contents, prototypes)
+            else:
+                result.append('#line 1 "%s"' % name)
+                result.append(contents)
+            is_first = False
+
+        return "\n".join(result)
+
+
+def ConvertInoToCpp(env):
+
+    def delete_tmpcpp_file(file_):
+        remove(file_)
+
+    ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) +
+                 env.Glob(join("$PROJECTSRC_DIR", "*.pde")))
+
+    c = InoToCPPConverter(ino_nodes)
+    data = c.convert()
+
+    if not data:
+        return
+
+    tmpcpp_file = join(env.subst("$PROJECTSRC_DIR"), "tmp_ino_to.cpp")
+    with open(tmpcpp_file, "w") as f:
+        f.write(data)
+
+    atexit.register(delete_tmpcpp_file, tmpcpp_file)
+
+
+def DumpIDEData(env):
+    data = {
+        "defines": [],
+        "includes": []
+    }
+
+    # includes from framework and libs
+    for item in env.get("VARIANT_DIRS", []):
+        if "$BUILDSRC_DIR" in item[0]:
+            continue
+        data['includes'].append(env.subst(item[1]))
+
+    # includes from toolchain
+    toolchain_dir = env.subst(
+        join("$PIOPACKAGES_DIR", "$PIOPACKAGE_TOOLCHAIN"))
+    toolchain_incglobs = [
+        join(toolchain_dir, "*", "include"),
+        join(toolchain_dir, "lib", "gcc", "*", "*", "include")
+    ]
+    for g in toolchain_incglobs:
+        data['includes'].extend(glob(g))
+
+    # global symbols
+    for item in env.get("CPPDEFINES", []):
+        data['defines'].append(env.subst(item))
+
+    # special symbol for Atmel AVR MCU
+    board = env.get("BOARD_OPTIONS", {})
+    if board and board['platform'] == "atmelavr":
+        data['defines'].append(
+            "__AVR_%s__" % board['build']['mcu'].upper()
+            .replace("ATMEGA", "ATmega")
+            .replace("ATTINY", "ATtiny")
+        )
+
+    return data
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(ConvertInoToCpp)
+    env.AddMethod(DumpIDEData)
+    return env

platformio/builder/tools/platformio.py

@@ -1,10 +1,9 @@
 # Copyright (C) Ivan Kravets <me@ikravets.com>
 # See LICENSE for details.
 
-import atexit
 import re
 from glob import glob
-from os import getenv, listdir, remove, sep, walk
+from os import getenv, listdir, sep, walk
 from os.path import basename, dirname, isdir, isfile, join, normpath
 
 from SCons.Script import DefaultEnvironment, Exit, SConscript
@@ -13,6 +12,13 @@ from SCons.Util import case_sensitive_suffixes
 from platformio.util import pioversion_to_intstr
 
 
+SRC_BUILD_EXT = ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]
+SRC_HEADER_EXT = ["h", "hpp"]
+SRC_DEFAULT_FILTER = " ".join([
+    "+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep, "-<examples%s>" % sep
+])
+
+
 def BuildFirmware(env):
 
     # fix ASM handling under non-casitive OS
@@ -25,9 +31,6 @@ def BuildFirmware(env):
     env.ProcessFlags()
     env.BuildFramework()
 
-    vdirs = env.VariantDirRecursive(
-        join("$BUILD_DIR", "src"), "$PROJECTSRC_DIR", duplicate=False)
-
     # build dependent libs
     deplibs = env.BuildDependentLibraries("$PROJECTSRC_DIR")
 
@@ -59,7 +62,10 @@ def BuildFirmware(env):
 
     return env.Program(
         join("$BUILD_DIR", "firmware"),
-        [env.GlobCXXFiles(vdir) for vdir in vdirs],
+        env.LookupSources(
+            "$BUILDSRC_DIR", "$PROJECTSRC_DIR", duplicate=False,
+            src_filter=getenv("PLATFORMIO_SRC_FILTER",
+                              env.get("SRC_FILTER", None))),
         LIBS=env.get("LIBS", []) + deplibs,
         LIBPATH=env.get("LIBPATH", []) + ["$BUILD_DIR"],
         PROGSUFFIX=".elf"
@@ -85,13 +91,13 @@ def ProcessFlags(env):
         env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines))
 
 
-def GlobCXXFiles(env, path):
+def IsFileWithExt(env, file_, ext):  # pylint: disable=W0613
-    files = []
+    if basename(file_).startswith("."):
-    for suff in ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]:
+        return False
-        _list = env.Glob(join(path, "*.%s" % suff))
+    for e in ext:
-        if _list:
+        if file_.endswith(".%s" % e):
-            files += _list
+            return True
-    return files
+    return False
 
 
 def VariantDirWrap(env, variant_dir, src_dir, duplicate=True):
@@ -99,20 +105,51 @@ def VariantDirWrap(env, variant_dir, src_dir, duplicate=True):
     env.VariantDir(variant_dir, src_dir, duplicate)
 
 
-def VariantDirRecursive(env, variant_dir, src_dir, duplicate=True,
+def LookupSources(env, variant_dir, src_dir, duplicate=True, src_filter=None):
-                        ignore_pattern=None):
+
-    if not ignore_pattern:
+    SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
-        ignore_pattern = (".git", ".svn")
+
+    def _append_build_item(items, item, src_dir):
+        if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT):
+            items.add(item.replace(src_dir + sep, ""))
+
+    def _match_sources(src_dir, src_filter):
+        matches = set()
+        for (action, pattern) in SRC_FILTER_PATTERNS_RE.findall(src_filter):
+            items = set()
+            for item in glob(join(src_dir, pattern)):
+                if isdir(item):
+                    for root, _, files in walk(item, followlinks=True):
+                        for f in files:
+                            _append_build_item(items, join(root, f), src_dir)
+                else:
+                    _append_build_item(items, item, src_dir)
+            if action == "+":
+                matches |= items
+            else:
+                matches -= items
+        return sorted(list(matches))
+
+    sources = []
     variants = []
+
     src_dir = env.subst(src_dir)
-    for root, _, _ in walk(src_dir, followlinks=True):
+    if src_dir.endswith(sep):
-        _src_dir = root
+        src_dir = src_dir[:-1]
-        _var_dir = variant_dir + root.replace(src_dir, "")
+
-        if any([s in _var_dir.lower() for s in ignore_pattern]):
+    for item in _match_sources(src_dir, src_filter or SRC_DEFAULT_FILTER):
-            continue
+        _reldir = dirname(item)
-        env.VariantDirWrap(_var_dir, _src_dir, duplicate)
+        _src_dir = join(src_dir, _reldir)
-        variants.append(_var_dir)
+        _var_dir = join(variant_dir, _reldir)
-    return variants
+
+        if _var_dir not in variants:
+            variants.append(_var_dir)
+            env.VariantDirWrap(_var_dir, _src_dir, duplicate)
+
+        if env.IsFileWithExt(item, SRC_BUILD_EXT):
+            sources.append(env.File(join(_var_dir, basename(item))))
+
+    return sources
 
 
 def BuildFramework(env):
@@ -134,18 +171,11 @@ def BuildFramework(env):
                  framework)
 
 
-def BuildLibrary(env, variant_dir, library_dir, ignore_files=None):
+def BuildLibrary(env, variant_dir, library_dir, src_filter=None):
     lib = env.Clone()
-    vdirs = lib.VariantDirRecursive(
-        variant_dir, library_dir, ignore_pattern=(".git", ".svn", "examples"))
-    srcfiles = []
-    for vdir in vdirs:
-        for item in lib.GlobCXXFiles(vdir):
-            if not ignore_files or item.name not in ignore_files:
-                srcfiles.append(item)
     return lib.Library(
         lib.subst(variant_dir),
-        srcfiles
+        lib.LookupSources(variant_dir, library_dir, src_filter=src_filter)
     )
 
 
@@ -239,10 +269,10 @@ def BuildDependentLibraries(env, src_dir):  # pylint: disable=R0914
         return result
 
     def _process_src_dir(state, src_dir):
-        for root, _, _ in walk(src_dir, followlinks=True):
+        for root, _, files in walk(src_dir, followlinks=True):
-            for node in (env.GlobCXXFiles(root) +
+            for f in files:
-                         env.Glob(join(root, "*.h"))):
+                if env.IsFileWithExt(f, SRC_BUILD_EXT + SRC_HEADER_EXT):
-                state = _parse_includes(state, node)
+                    state = _parse_includes(state, env.File(join(root, f)))
         return state
 
     def _parse_includes(state, node):
@@ -295,151 +325,6 @@ def BuildDependentLibraries(env, src_dir):  # pylint: disable=R0914
     return libs
 
 
-class InoToCPPConverter(object):
-
-    PROTOTYPE_RE = re.compile(
-        r"""^(
-        (\s*[a-z_\d]+){1,2}         # return type
-        (\s+[a-z_\d]+\s*)           # name of prototype
-        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
-        )\s*\{                      # must end with {
-        """,
-        re.X | re.M | re.I
-    )
-
-    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
-
-    STRIPCOMMENTS_RE = re.compile(r"(/\*.*?\*/|(^|\s+)//[^\r\n]*$)",
-                                  re.M | re.S)
-
-    def __init__(self, nodes):
-        self.nodes = nodes
-
-    def is_main_node(self, contents):
-        return self.DETECTMAIN_RE.search(contents)
-
-    @staticmethod
-    def _replace_comments_callback(match):
-        if "\n" in match.group(1):
-            return "\n" * match.group(1).count("\n")
-        else:
-            return " "
-
-    def _parse_prototypes(self, contents):
-        prototypes = []
-        reserved_keywords = set(["if", "else", "while"])
-        for item in self.PROTOTYPE_RE.findall(contents):
-            if set([item[1].strip(), item[2].strip()]) & reserved_keywords:
-                continue
-            prototypes.append(item[0])
-        return prototypes
-
-    def append_prototypes(self, fname, contents, prototypes):
-        contents = self.STRIPCOMMENTS_RE.sub(self._replace_comments_callback,
-                                             contents)
-        result = []
-        is_appended = False
-        linenum = 0
-        for line in contents.splitlines():
-            linenum += 1
-            line = line.strip()
-
-            if not is_appended and line and not line.startswith("#"):
-                is_appended = True
-                result.append("%s;" % ";\n".join(prototypes))
-                result.append('#line %d "%s"' % (linenum, fname))
-
-            result.append(line)
-
-        return result
-
-    def convert(self):
-        prototypes = []
-        data = []
-        for node in self.nodes:
-            ino_contents = node.get_text_contents()
-            prototypes += self._parse_prototypes(ino_contents)
-
-            item = (basename(node.get_path()), ino_contents)
-            if self.is_main_node(ino_contents):
-                data = [item] + data
-            else:
-                data.append(item)
-
-        if not data:
-            return None
-
-        result = ["#include <Arduino.h>"]
-        is_first = True
-
-        for name, contents in data:
-            if is_first and prototypes:
-                result += self.append_prototypes(name, contents, prototypes)
-            else:
-                result.append('#line 1 "%s"' % name)
-                result.append(contents)
-            is_first = False
-
-        return "\n".join(result)
-
-
-def ConvertInoToCpp(env):
-
-    def delete_tmpcpp_file(file_):
-        remove(file_)
-
-    ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) +
-                 env.Glob(join("$PROJECTSRC_DIR", "*.pde")))
-
-    c = InoToCPPConverter(ino_nodes)
-    data = c.convert()
-
-    if not data:
-        return
-
-    tmpcpp_file = join(env.subst("$PROJECTSRC_DIR"), "tmp_ino_to.cpp")
-    with open(tmpcpp_file, "w") as f:
-        f.write(data)
-
-    atexit.register(delete_tmpcpp_file, tmpcpp_file)
-
-
-def DumpIDEData(env):
-    data = {
-        "defines": [],
-        "includes": []
-    }
-
-    # includes from framework and libs
-    for item in env.get("VARIANT_DIRS", []):
-        data['includes'].append(env.subst(item[1]))
-
-    # includes from toolchain
-    toolchain_dir = env.subst(
-        join("$PIOPACKAGES_DIR", "$PIOPACKAGE_TOOLCHAIN"))
-    toolchain_incglobs = [
-        join(toolchain_dir, "*", "include"),
-        join(toolchain_dir, "lib", "gcc", "*", "*", "include")
-    ]
-    for g in toolchain_incglobs:
-        data['includes'].extend(glob(g))
-
-    # global symbols
-    for item in env.get("CPPDEFINES", []):
-        data['defines'].append(env.subst(item))
-
-    # special symbol for Atmel AVR MCU
-    board = env.get("BOARD_OPTIONS", {})
-    if board and board['platform'] == "atmelavr":
-        data['defines'].append(
-            "__AVR_%s__" % board['build']['mcu'].upper()
-            .replace("ATMEGA", "ATmega")
-            .replace("ATTINY", "ATtiny")
-        )
-
-    return data
-
-
 def exists(_):
     return True
 
@@ -447,12 +332,10 @@ def exists(_):
 def generate(env):
     env.AddMethod(BuildFirmware)
     env.AddMethod(ProcessFlags)
-    env.AddMethod(GlobCXXFiles)
+    env.AddMethod(IsFileWithExt)
     env.AddMethod(VariantDirWrap)
-    env.AddMethod(VariantDirRecursive)
+    env.AddMethod(LookupSources)
     env.AddMethod(BuildFramework)
     env.AddMethod(BuildLibrary)
     env.AddMethod(BuildDependentLibraries)
-    env.AddMethod(ConvertInoToCpp)
-    env.AddMethod(DumpIDEData)
     return env