Allow to exclude/include source files from build process using src_filter // Resolve #240

This commit is contained in:
Ivan Kravets
2015-06-22 15:06:39 +03:00
parent 46461b0721
commit 3232ba6a6c
8 changed files with 278 additions and 199 deletions

View File

@ -4,6 +4,9 @@ Release History
2.2.0 (2015-??-??)
------------------
* Allowed to exclude/include source files from build process using
`src_filter <http://docs.platformio.org/en/latest/projectconf.html#src-filter>`__
(`issue #240 <https://github.com/platformio/platformio/issues/240>`_)
* Launch own extra script before firmware building/uploading processes
(`issue #239 <https://github.com/platformio/platformio/issues/239>`_)
* Specify own path to the linker script (ld) using

View File

@ -77,6 +77,13 @@ PLATFORMIO_SRCBUILD_FLAGS
Allows to set :ref:`projectconf` option :ref:`projectconf_srcbuild_flags`.
.. _envvar_PLATFORMIO_SRC_FILTER:
PLATFORMIO_SRC_FILTER
~~~~~~~~~~~~~~~~~~~~~
Allows to set :ref:`projectconf` option :ref:`projectconf_src_filter`.
.. _envvar_PLATFORMIO_LDF_CYCLIC:
PLATFORMIO_LDF_CYCLIC

View File

@ -319,6 +319,28 @@ but will be applied only for the project source code from
This option can be set by global environment variable
:ref:`envvar_PLATFORMIO_SRCBUILD_FLAGS`.
.. _projectconf_src_filter:
``src_filter``
^^^^^^^^^^^^^^
This option allows to specify which source files should be included/excluded
from build process. Filter supports 2 templates:
* ``+<PATH>`` include template
* ``-<PATH>`` exclude template
``PATH`` MAST BE related from :ref:`projectconf_pio_src_dir`. All patterns will
be applied in theirs order.
`GLOB Patterns <http://en.wikipedia.org/wiki/Glob_(programming)>`_ are allowed.
By default, ``src_filter`` is predefined to
``+<*> -<.git/> -<svn/> -<examples/>``, which means "includes ALL files, then
exclude ``.git`` and ``svn`` repository folders and exclude ``examples`` folder.
This option can be set by global environment variable
:ref:`envvar_PLATFORMIO_SRC_FILTER`.
``install_libs``
^^^^^^^^^^^^^^^^

View File

@ -1,7 +1,7 @@
# Copyright (C) Ivan Kravets <me@ikravets.com>
# See LICENSE for details.
VERSION = (2, 2, "0.dev2")
VERSION = (2, 2, "0.dev3")
__version__ = ".".join([str(s) for s in VERSION])
__title__ = "platformio"

View File

@ -39,6 +39,7 @@ commonvars.AddVariables(
("FRAMEWORK",),
("BUILD_FLAGS",),
("SRCBUILD_FLAGS",),
("SRC_FILTER",),
("IGNORE_LIBS",),
("USE_LIBS",),
("LDF_CYCLIC",),
@ -57,7 +58,7 @@ commonvars.AddVariables(
DefaultEnvironment(
tools=[
"gcc", "g++", "as", "ar", "gnulink",
"platformio", "pioupload", "pioar"
"platformio", "pioupload", "pioar", "piomisc"
],
toolpath=[join("$PIOBUILDER_DIR", "tools")],
variables=commonvars,
@ -74,6 +75,7 @@ DefaultEnvironment(
PIOPACKAGES_DIR=join("$PIOHOME_DIR", "packages"),
BUILD_DIR=join("$PIOENVS_DIR", "$PIOENV"),
BUILDSRC_DIR=join("$BUILD_DIR", "ProjectSrc"),
LIBSOURCE_DIRS=[
"$PROJECTLIB_DIR",
util.get_lib_dir(),

View File

@ -38,10 +38,7 @@ env.Append(
envsafe = env.Clone()
envsafe.Append(
CPPPATH=[
join("$BUILD_DIR", "src")
],
CPPPATH=["$BUILDSRC_DIR"],
CPPDEFINES=[
"USE_STDPERIPH_DRIVER"
]
@ -52,22 +49,22 @@ envsafe.Append(
#
extra_flags = env.get("BOARD_OPTIONS", {}).get("build", {}).get("extra_flags")
ignore_files = []
src_filter_patterns = ["+<*>"]
if "STM32F40_41xxx" in extra_flags:
ignore_files += ["stm32f4xx_fmc.c"]
src_filter_patterns += ["-<stm32f4xx_fmc.c>"]
if "STM32F427_437xx" in extra_flags:
ignore_files += ["stm32f4xx_fsmc.c"]
src_filter_patterns += ["-<stm32f4xx_fsmc.c>"]
elif "STM32F303xC" in extra_flags:
ignore_files += ["stm32f30x_hrtim.c"]
src_filter_patterns += ["-<stm32f30x_hrtim.c>"]
elif "STM32L1XX_MD" in extra_flags:
ignore_files += ["stm32l1xx_flash_ramfunc.c"]
src_filter_patterns += ["-<stm32l1xx_flash_ramfunc.c>"]
libs = []
libs.append(envsafe.BuildLibrary(
join("$BUILD_DIR", "FrameworkSPL"),
join("$PLATFORMFW_DIR", "${BOARD_OPTIONS['build']['core']}", "variants",
"${BOARD_OPTIONS['build']['variant']}", "src"),
ignore_files
src_filter=" ".join(src_filter_patterns)
))
env.Append(LIBS=libs)

View File

@ -0,0 +1,165 @@
# Copyright (C) Ivan Kravets <me@ikravets.com>
# See LICENSE for details.
import atexit
import re
from glob import glob
from os import remove
from os.path import basename, join
class InoToCPPConverter(object):
PROTOTYPE_RE = re.compile(
r"""^(
(\s*[a-z_\d]+){1,2} # return type
(\s+[a-z_\d]+\s*) # name of prototype
\([a-z_,\.\*\&\[\]\s\d]*\) # arguments
)\s*\{ # must end with {
""",
re.X | re.M | re.I
)
DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
STRIPCOMMENTS_RE = re.compile(r"(/\*.*?\*/|(^|\s+)//[^\r\n]*$)",
re.M | re.S)
def __init__(self, nodes):
self.nodes = nodes
def is_main_node(self, contents):
return self.DETECTMAIN_RE.search(contents)
@staticmethod
def _replace_comments_callback(match):
if "\n" in match.group(1):
return "\n" * match.group(1).count("\n")
else:
return " "
def _parse_prototypes(self, contents):
prototypes = []
reserved_keywords = set(["if", "else", "while"])
for item in self.PROTOTYPE_RE.findall(contents):
if set([item[1].strip(), item[2].strip()]) & reserved_keywords:
continue
prototypes.append(item[0])
return prototypes
def append_prototypes(self, fname, contents, prototypes):
contents = self.STRIPCOMMENTS_RE.sub(self._replace_comments_callback,
contents)
result = []
is_appended = False
linenum = 0
for line in contents.splitlines():
linenum += 1
line = line.strip()
if not is_appended and line and not line.startswith("#"):
is_appended = True
result.append("%s;" % ";\n".join(prototypes))
result.append('#line %d "%s"' % (linenum, fname))
result.append(line)
return result
def convert(self):
prototypes = []
data = []
for node in self.nodes:
ino_contents = node.get_text_contents()
prototypes += self._parse_prototypes(ino_contents)
item = (basename(node.get_path()), ino_contents)
if self.is_main_node(ino_contents):
data = [item] + data
else:
data.append(item)
if not data:
return None
result = ["#include <Arduino.h>"]
is_first = True
for name, contents in data:
if is_first and prototypes:
result += self.append_prototypes(name, contents, prototypes)
else:
result.append('#line 1 "%s"' % name)
result.append(contents)
is_first = False
return "\n".join(result)
def ConvertInoToCpp(env):
def delete_tmpcpp_file(file_):
remove(file_)
ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) +
env.Glob(join("$PROJECTSRC_DIR", "*.pde")))
c = InoToCPPConverter(ino_nodes)
data = c.convert()
if not data:
return
tmpcpp_file = join(env.subst("$PROJECTSRC_DIR"), "tmp_ino_to.cpp")
with open(tmpcpp_file, "w") as f:
f.write(data)
atexit.register(delete_tmpcpp_file, tmpcpp_file)
def DumpIDEData(env):
data = {
"defines": [],
"includes": []
}
# includes from framework and libs
for item in env.get("VARIANT_DIRS", []):
if "$BUILDSRC_DIR" in item[0]:
continue
data['includes'].append(env.subst(item[1]))
# includes from toolchain
toolchain_dir = env.subst(
join("$PIOPACKAGES_DIR", "$PIOPACKAGE_TOOLCHAIN"))
toolchain_incglobs = [
join(toolchain_dir, "*", "include"),
join(toolchain_dir, "lib", "gcc", "*", "*", "include")
]
for g in toolchain_incglobs:
data['includes'].extend(glob(g))
# global symbols
for item in env.get("CPPDEFINES", []):
data['defines'].append(env.subst(item))
# special symbol for Atmel AVR MCU
board = env.get("BOARD_OPTIONS", {})
if board and board['platform'] == "atmelavr":
data['defines'].append(
"__AVR_%s__" % board['build']['mcu'].upper()
.replace("ATMEGA", "ATmega")
.replace("ATTINY", "ATtiny")
)
return data
def exists(_):
return True
def generate(env):
env.AddMethod(ConvertInoToCpp)
env.AddMethod(DumpIDEData)
return env

View File

@ -1,10 +1,9 @@
# Copyright (C) Ivan Kravets <me@ikravets.com>
# See LICENSE for details.
import atexit
import re
from glob import glob
from os import getenv, listdir, remove, sep, walk
from os import getenv, listdir, sep, walk
from os.path import basename, dirname, isdir, isfile, join, normpath
from SCons.Script import DefaultEnvironment, Exit, SConscript
@ -13,6 +12,13 @@ from SCons.Util import case_sensitive_suffixes
from platformio.util import pioversion_to_intstr
SRC_BUILD_EXT = ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]
SRC_HEADER_EXT = ["h", "hpp"]
SRC_DEFAULT_FILTER = " ".join([
"+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep, "-<examples%s>" % sep
])
def BuildFirmware(env):
# fix ASM handling under non-casitive OS
@ -25,9 +31,6 @@ def BuildFirmware(env):
env.ProcessFlags()
env.BuildFramework()
vdirs = env.VariantDirRecursive(
join("$BUILD_DIR", "src"), "$PROJECTSRC_DIR", duplicate=False)
# build dependent libs
deplibs = env.BuildDependentLibraries("$PROJECTSRC_DIR")
@ -59,7 +62,10 @@ def BuildFirmware(env):
return env.Program(
join("$BUILD_DIR", "firmware"),
[env.GlobCXXFiles(vdir) for vdir in vdirs],
env.LookupSources(
"$BUILDSRC_DIR", "$PROJECTSRC_DIR", duplicate=False,
src_filter=getenv("PLATFORMIO_SRC_FILTER",
env.get("SRC_FILTER", None))),
LIBS=env.get("LIBS", []) + deplibs,
LIBPATH=env.get("LIBPATH", []) + ["$BUILD_DIR"],
PROGSUFFIX=".elf"
@ -85,13 +91,13 @@ def ProcessFlags(env):
env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines))
def GlobCXXFiles(env, path):
files = []
for suff in ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]:
_list = env.Glob(join(path, "*.%s" % suff))
if _list:
files += _list
return files
def IsFileWithExt(env, file_, ext): # pylint: disable=W0613
if basename(file_).startswith("."):
return False
for e in ext:
if file_.endswith(".%s" % e):
return True
return False
def VariantDirWrap(env, variant_dir, src_dir, duplicate=True):
@ -99,20 +105,51 @@ def VariantDirWrap(env, variant_dir, src_dir, duplicate=True):
env.VariantDir(variant_dir, src_dir, duplicate)
def VariantDirRecursive(env, variant_dir, src_dir, duplicate=True,
ignore_pattern=None):
if not ignore_pattern:
ignore_pattern = (".git", ".svn")
def LookupSources(env, variant_dir, src_dir, duplicate=True, src_filter=None):
SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
def _append_build_item(items, item, src_dir):
if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT):
items.add(item.replace(src_dir + sep, ""))
def _match_sources(src_dir, src_filter):
matches = set()
for (action, pattern) in SRC_FILTER_PATTERNS_RE.findall(src_filter):
items = set()
for item in glob(join(src_dir, pattern)):
if isdir(item):
for root, _, files in walk(item, followlinks=True):
for f in files:
_append_build_item(items, join(root, f), src_dir)
else:
_append_build_item(items, item, src_dir)
if action == "+":
matches |= items
else:
matches -= items
return sorted(list(matches))
sources = []
variants = []
src_dir = env.subst(src_dir)
for root, _, _ in walk(src_dir, followlinks=True):
_src_dir = root
_var_dir = variant_dir + root.replace(src_dir, "")
if any([s in _var_dir.lower() for s in ignore_pattern]):
continue
env.VariantDirWrap(_var_dir, _src_dir, duplicate)
variants.append(_var_dir)
return variants
if src_dir.endswith(sep):
src_dir = src_dir[:-1]
for item in _match_sources(src_dir, src_filter or SRC_DEFAULT_FILTER):
_reldir = dirname(item)
_src_dir = join(src_dir, _reldir)
_var_dir = join(variant_dir, _reldir)
if _var_dir not in variants:
variants.append(_var_dir)
env.VariantDirWrap(_var_dir, _src_dir, duplicate)
if env.IsFileWithExt(item, SRC_BUILD_EXT):
sources.append(env.File(join(_var_dir, basename(item))))
return sources
def BuildFramework(env):
@ -134,18 +171,11 @@ def BuildFramework(env):
framework)
def BuildLibrary(env, variant_dir, library_dir, ignore_files=None):
def BuildLibrary(env, variant_dir, library_dir, src_filter=None):
lib = env.Clone()
vdirs = lib.VariantDirRecursive(
variant_dir, library_dir, ignore_pattern=(".git", ".svn", "examples"))
srcfiles = []
for vdir in vdirs:
for item in lib.GlobCXXFiles(vdir):
if not ignore_files or item.name not in ignore_files:
srcfiles.append(item)
return lib.Library(
lib.subst(variant_dir),
srcfiles
lib.LookupSources(variant_dir, library_dir, src_filter=src_filter)
)
@ -239,10 +269,10 @@ def BuildDependentLibraries(env, src_dir): # pylint: disable=R0914
return result
def _process_src_dir(state, src_dir):
for root, _, _ in walk(src_dir, followlinks=True):
for node in (env.GlobCXXFiles(root) +
env.Glob(join(root, "*.h"))):
state = _parse_includes(state, node)
for root, _, files in walk(src_dir, followlinks=True):
for f in files:
if env.IsFileWithExt(f, SRC_BUILD_EXT + SRC_HEADER_EXT):
state = _parse_includes(state, env.File(join(root, f)))
return state
def _parse_includes(state, node):
@ -295,151 +325,6 @@ def BuildDependentLibraries(env, src_dir): # pylint: disable=R0914
return libs
class InoToCPPConverter(object):
PROTOTYPE_RE = re.compile(
r"""^(
(\s*[a-z_\d]+){1,2} # return type
(\s+[a-z_\d]+\s*) # name of prototype
\([a-z_,\.\*\&\[\]\s\d]*\) # arguments
)\s*\{ # must end with {
""",
re.X | re.M | re.I
)
DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
STRIPCOMMENTS_RE = re.compile(r"(/\*.*?\*/|(^|\s+)//[^\r\n]*$)",
re.M | re.S)
def __init__(self, nodes):
self.nodes = nodes
def is_main_node(self, contents):
return self.DETECTMAIN_RE.search(contents)
@staticmethod
def _replace_comments_callback(match):
if "\n" in match.group(1):
return "\n" * match.group(1).count("\n")
else:
return " "
def _parse_prototypes(self, contents):
prototypes = []
reserved_keywords = set(["if", "else", "while"])
for item in self.PROTOTYPE_RE.findall(contents):
if set([item[1].strip(), item[2].strip()]) & reserved_keywords:
continue
prototypes.append(item[0])
return prototypes
def append_prototypes(self, fname, contents, prototypes):
contents = self.STRIPCOMMENTS_RE.sub(self._replace_comments_callback,
contents)
result = []
is_appended = False
linenum = 0
for line in contents.splitlines():
linenum += 1
line = line.strip()
if not is_appended and line and not line.startswith("#"):
is_appended = True
result.append("%s;" % ";\n".join(prototypes))
result.append('#line %d "%s"' % (linenum, fname))
result.append(line)
return result
def convert(self):
prototypes = []
data = []
for node in self.nodes:
ino_contents = node.get_text_contents()
prototypes += self._parse_prototypes(ino_contents)
item = (basename(node.get_path()), ino_contents)
if self.is_main_node(ino_contents):
data = [item] + data
else:
data.append(item)
if not data:
return None
result = ["#include <Arduino.h>"]
is_first = True
for name, contents in data:
if is_first and prototypes:
result += self.append_prototypes(name, contents, prototypes)
else:
result.append('#line 1 "%s"' % name)
result.append(contents)
is_first = False
return "\n".join(result)
def ConvertInoToCpp(env):
def delete_tmpcpp_file(file_):
remove(file_)
ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) +
env.Glob(join("$PROJECTSRC_DIR", "*.pde")))
c = InoToCPPConverter(ino_nodes)
data = c.convert()
if not data:
return
tmpcpp_file = join(env.subst("$PROJECTSRC_DIR"), "tmp_ino_to.cpp")
with open(tmpcpp_file, "w") as f:
f.write(data)
atexit.register(delete_tmpcpp_file, tmpcpp_file)
def DumpIDEData(env):
data = {
"defines": [],
"includes": []
}
# includes from framework and libs
for item in env.get("VARIANT_DIRS", []):
data['includes'].append(env.subst(item[1]))
# includes from toolchain
toolchain_dir = env.subst(
join("$PIOPACKAGES_DIR", "$PIOPACKAGE_TOOLCHAIN"))
toolchain_incglobs = [
join(toolchain_dir, "*", "include"),
join(toolchain_dir, "lib", "gcc", "*", "*", "include")
]
for g in toolchain_incglobs:
data['includes'].extend(glob(g))
# global symbols
for item in env.get("CPPDEFINES", []):
data['defines'].append(env.subst(item))
# special symbol for Atmel AVR MCU
board = env.get("BOARD_OPTIONS", {})
if board and board['platform'] == "atmelavr":
data['defines'].append(
"__AVR_%s__" % board['build']['mcu'].upper()
.replace("ATMEGA", "ATmega")
.replace("ATTINY", "ATtiny")
)
return data
def exists(_):
return True
@ -447,12 +332,10 @@ def exists(_):
def generate(env):
env.AddMethod(BuildFirmware)
env.AddMethod(ProcessFlags)
env.AddMethod(GlobCXXFiles)
env.AddMethod(IsFileWithExt)
env.AddMethod(VariantDirWrap)
env.AddMethod(VariantDirRecursive)
env.AddMethod(LookupSources)
env.AddMethod(BuildFramework)
env.AddMethod(BuildLibrary)
env.AddMethod(BuildDependentLibraries)
env.AddMethod(ConvertInoToCpp)
env.AddMethod(DumpIDEData)
return env