From 3232ba6a6c1053de7f6710b1d6063d4f1cc35302 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2015 15:06:39 +0300 Subject: [PATCH] Allow to exclude/include source files from build process using src_filter // Resolve #240 --- HISTORY.rst | 3 + docs/envvars.rst | 7 + docs/projectconf.rst | 22 ++ platformio/__init__.py | 2 +- platformio/builder/main.py | 4 +- platformio/builder/scripts/frameworks/spl.py | 17 +- platformio/builder/tools/piomisc.py | 165 ++++++++++++ platformio/builder/tools/platformio.py | 257 +++++-------------- 8 files changed, 278 insertions(+), 199 deletions(-) create mode 100644 platformio/builder/tools/piomisc.py diff --git a/HISTORY.rst b/HISTORY.rst index 656ebfcd..398550c6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,9 @@ Release History 2.2.0 (2015-??-??) ------------------ +* Allowed to exclude/include source files from build process using + `src_filter `__ + (`issue #240 `_) * Launch own extra script before firmware building/uploading processes (`issue #239 `_) * Specify own path to the linker script (ld) using diff --git a/docs/envvars.rst b/docs/envvars.rst index 95a441f5..d3d0c084 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -77,6 +77,13 @@ PLATFORMIO_SRCBUILD_FLAGS Allows to set :ref:`projectconf` option :ref:`projectconf_srcbuild_flags`. +.. _envvar_PLATFORMIO_SRC_FILTER: + +PLATFORMIO_SRC_FILTER +~~~~~~~~~~~~~~~~~~~~~ + +Allows to set :ref:`projectconf` option :ref:`projectconf_src_filter`. + .. _envvar_PLATFORMIO_LDF_CYCLIC: PLATFORMIO_LDF_CYCLIC diff --git a/docs/projectconf.rst b/docs/projectconf.rst index 1d8462c4..8db258c4 100644 --- a/docs/projectconf.rst +++ b/docs/projectconf.rst @@ -319,6 +319,28 @@ but will be applied only for the project source code from This option can be set by global environment variable :ref:`envvar_PLATFORMIO_SRCBUILD_FLAGS`. +.. _projectconf_src_filter: + +``src_filter`` +^^^^^^^^^^^^^^ + +This option allows to specify which source files should be included/excluded +from build process. Filter supports 2 templates: + +* ``+`` include template +* ``-`` exclude template + +``PATH`` MAST BE related from :ref:`projectconf_pio_src_dir`. All patterns will +be applied in theirs order. +`GLOB Patterns `_ are allowed. + +By default, ``src_filter`` is predefined to +``+<*> -<.git/> - -``, which means "includes ALL files, then +exclude ``.git`` and ``svn`` repository folders and exclude ``examples`` folder. + +This option can be set by global environment variable +:ref:`envvar_PLATFORMIO_SRC_FILTER`. + ``install_libs`` ^^^^^^^^^^^^^^^^ diff --git a/platformio/__init__.py b/platformio/__init__.py index 6dbbfe25..cfeb39e7 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) Ivan Kravets # See LICENSE for details. -VERSION = (2, 2, "0.dev2") +VERSION = (2, 2, "0.dev3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 073c7f71..5db1f0df 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -39,6 +39,7 @@ commonvars.AddVariables( ("FRAMEWORK",), ("BUILD_FLAGS",), ("SRCBUILD_FLAGS",), + ("SRC_FILTER",), ("IGNORE_LIBS",), ("USE_LIBS",), ("LDF_CYCLIC",), @@ -57,7 +58,7 @@ commonvars.AddVariables( DefaultEnvironment( tools=[ "gcc", "g++", "as", "ar", "gnulink", - "platformio", "pioupload", "pioar" + "platformio", "pioupload", "pioar", "piomisc" ], toolpath=[join("$PIOBUILDER_DIR", "tools")], variables=commonvars, @@ -74,6 +75,7 @@ DefaultEnvironment( PIOPACKAGES_DIR=join("$PIOHOME_DIR", "packages"), BUILD_DIR=join("$PIOENVS_DIR", "$PIOENV"), + BUILDSRC_DIR=join("$BUILD_DIR", "ProjectSrc"), LIBSOURCE_DIRS=[ "$PROJECTLIB_DIR", util.get_lib_dir(), diff --git a/platformio/builder/scripts/frameworks/spl.py b/platformio/builder/scripts/frameworks/spl.py index 130d8bf2..857de8c5 100644 --- a/platformio/builder/scripts/frameworks/spl.py +++ b/platformio/builder/scripts/frameworks/spl.py @@ -38,10 +38,7 @@ env.Append( envsafe = env.Clone() envsafe.Append( - CPPPATH=[ - join("$BUILD_DIR", "src") - ], - + CPPPATH=["$BUILDSRC_DIR"], CPPDEFINES=[ "USE_STDPERIPH_DRIVER" ] @@ -52,22 +49,22 @@ envsafe.Append( # extra_flags = env.get("BOARD_OPTIONS", {}).get("build", {}).get("extra_flags") -ignore_files = [] +src_filter_patterns = ["+<*>"] if "STM32F40_41xxx" in extra_flags: - ignore_files += ["stm32f4xx_fmc.c"] + src_filter_patterns += ["-"] if "STM32F427_437xx" in extra_flags: - ignore_files += ["stm32f4xx_fsmc.c"] + src_filter_patterns += ["-"] elif "STM32F303xC" in extra_flags: - ignore_files += ["stm32f30x_hrtim.c"] + src_filter_patterns += ["-"] elif "STM32L1XX_MD" in extra_flags: - ignore_files += ["stm32l1xx_flash_ramfunc.c"] + src_filter_patterns += ["-"] libs = [] libs.append(envsafe.BuildLibrary( join("$BUILD_DIR", "FrameworkSPL"), join("$PLATFORMFW_DIR", "${BOARD_OPTIONS['build']['core']}", "variants", "${BOARD_OPTIONS['build']['variant']}", "src"), - ignore_files + src_filter=" ".join(src_filter_patterns) )) env.Append(LIBS=libs) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py new file mode 100644 index 00000000..80539c09 --- /dev/null +++ b/platformio/builder/tools/piomisc.py @@ -0,0 +1,165 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +import atexit +import re +from glob import glob +from os import remove +from os.path import basename, join + + +class InoToCPPConverter(object): + + PROTOTYPE_RE = re.compile( + r"""^( + (\s*[a-z_\d]+){1,2} # return type + (\s+[a-z_\d]+\s*) # name of prototype + \([a-z_,\.\*\&\[\]\s\d]*\) # arguments + )\s*\{ # must end with { + """, + re.X | re.M | re.I + ) + + DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) + + STRIPCOMMENTS_RE = re.compile(r"(/\*.*?\*/|(^|\s+)//[^\r\n]*$)", + re.M | re.S) + + def __init__(self, nodes): + self.nodes = nodes + + def is_main_node(self, contents): + return self.DETECTMAIN_RE.search(contents) + + @staticmethod + def _replace_comments_callback(match): + if "\n" in match.group(1): + return "\n" * match.group(1).count("\n") + else: + return " " + + def _parse_prototypes(self, contents): + prototypes = [] + reserved_keywords = set(["if", "else", "while"]) + for item in self.PROTOTYPE_RE.findall(contents): + if set([item[1].strip(), item[2].strip()]) & reserved_keywords: + continue + prototypes.append(item[0]) + return prototypes + + def append_prototypes(self, fname, contents, prototypes): + contents = self.STRIPCOMMENTS_RE.sub(self._replace_comments_callback, + contents) + result = [] + is_appended = False + linenum = 0 + for line in contents.splitlines(): + linenum += 1 + line = line.strip() + + if not is_appended and line and not line.startswith("#"): + is_appended = True + result.append("%s;" % ";\n".join(prototypes)) + result.append('#line %d "%s"' % (linenum, fname)) + + result.append(line) + + return result + + def convert(self): + prototypes = [] + data = [] + for node in self.nodes: + ino_contents = node.get_text_contents() + prototypes += self._parse_prototypes(ino_contents) + + item = (basename(node.get_path()), ino_contents) + if self.is_main_node(ino_contents): + data = [item] + data + else: + data.append(item) + + if not data: + return None + + result = ["#include "] + is_first = True + + for name, contents in data: + if is_first and prototypes: + result += self.append_prototypes(name, contents, prototypes) + else: + result.append('#line 1 "%s"' % name) + result.append(contents) + is_first = False + + return "\n".join(result) + + +def ConvertInoToCpp(env): + + def delete_tmpcpp_file(file_): + remove(file_) + + ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) + + env.Glob(join("$PROJECTSRC_DIR", "*.pde"))) + + c = InoToCPPConverter(ino_nodes) + data = c.convert() + + if not data: + return + + tmpcpp_file = join(env.subst("$PROJECTSRC_DIR"), "tmp_ino_to.cpp") + with open(tmpcpp_file, "w") as f: + f.write(data) + + atexit.register(delete_tmpcpp_file, tmpcpp_file) + + +def DumpIDEData(env): + data = { + "defines": [], + "includes": [] + } + + # includes from framework and libs + for item in env.get("VARIANT_DIRS", []): + if "$BUILDSRC_DIR" in item[0]: + continue + data['includes'].append(env.subst(item[1])) + + # includes from toolchain + toolchain_dir = env.subst( + join("$PIOPACKAGES_DIR", "$PIOPACKAGE_TOOLCHAIN")) + toolchain_incglobs = [ + join(toolchain_dir, "*", "include"), + join(toolchain_dir, "lib", "gcc", "*", "*", "include") + ] + for g in toolchain_incglobs: + data['includes'].extend(glob(g)) + + # global symbols + for item in env.get("CPPDEFINES", []): + data['defines'].append(env.subst(item)) + + # special symbol for Atmel AVR MCU + board = env.get("BOARD_OPTIONS", {}) + if board and board['platform'] == "atmelavr": + data['defines'].append( + "__AVR_%s__" % board['build']['mcu'].upper() + .replace("ATMEGA", "ATmega") + .replace("ATTINY", "ATtiny") + ) + + return data + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(ConvertInoToCpp) + env.AddMethod(DumpIDEData) + return env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 5e373105..a0e2edff 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -1,10 +1,9 @@ # Copyright (C) Ivan Kravets # See LICENSE for details. -import atexit import re from glob import glob -from os import getenv, listdir, remove, sep, walk +from os import getenv, listdir, sep, walk from os.path import basename, dirname, isdir, isfile, join, normpath from SCons.Script import DefaultEnvironment, Exit, SConscript @@ -13,6 +12,13 @@ from SCons.Util import case_sensitive_suffixes from platformio.util import pioversion_to_intstr +SRC_BUILD_EXT = ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"] +SRC_HEADER_EXT = ["h", "hpp"] +SRC_DEFAULT_FILTER = " ".join([ + "+<*>", "-<.git%s>" % sep, "-" % sep, "-" % sep +]) + + def BuildFirmware(env): # fix ASM handling under non-casitive OS @@ -25,9 +31,6 @@ def BuildFirmware(env): env.ProcessFlags() env.BuildFramework() - vdirs = env.VariantDirRecursive( - join("$BUILD_DIR", "src"), "$PROJECTSRC_DIR", duplicate=False) - # build dependent libs deplibs = env.BuildDependentLibraries("$PROJECTSRC_DIR") @@ -59,7 +62,10 @@ def BuildFirmware(env): return env.Program( join("$BUILD_DIR", "firmware"), - [env.GlobCXXFiles(vdir) for vdir in vdirs], + env.LookupSources( + "$BUILDSRC_DIR", "$PROJECTSRC_DIR", duplicate=False, + src_filter=getenv("PLATFORMIO_SRC_FILTER", + env.get("SRC_FILTER", None))), LIBS=env.get("LIBS", []) + deplibs, LIBPATH=env.get("LIBPATH", []) + ["$BUILD_DIR"], PROGSUFFIX=".elf" @@ -85,13 +91,13 @@ def ProcessFlags(env): env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines)) -def GlobCXXFiles(env, path): - files = [] - for suff in ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]: - _list = env.Glob(join(path, "*.%s" % suff)) - if _list: - files += _list - return files +def IsFileWithExt(env, file_, ext): # pylint: disable=W0613 + if basename(file_).startswith("."): + return False + for e in ext: + if file_.endswith(".%s" % e): + return True + return False def VariantDirWrap(env, variant_dir, src_dir, duplicate=True): @@ -99,20 +105,51 @@ def VariantDirWrap(env, variant_dir, src_dir, duplicate=True): env.VariantDir(variant_dir, src_dir, duplicate) -def VariantDirRecursive(env, variant_dir, src_dir, duplicate=True, - ignore_pattern=None): - if not ignore_pattern: - ignore_pattern = (".git", ".svn") +def LookupSources(env, variant_dir, src_dir, duplicate=True, src_filter=None): + + SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>") + + def _append_build_item(items, item, src_dir): + if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT): + items.add(item.replace(src_dir + sep, "")) + + def _match_sources(src_dir, src_filter): + matches = set() + for (action, pattern) in SRC_FILTER_PATTERNS_RE.findall(src_filter): + items = set() + for item in glob(join(src_dir, pattern)): + if isdir(item): + for root, _, files in walk(item, followlinks=True): + for f in files: + _append_build_item(items, join(root, f), src_dir) + else: + _append_build_item(items, item, src_dir) + if action == "+": + matches |= items + else: + matches -= items + return sorted(list(matches)) + + sources = [] variants = [] + src_dir = env.subst(src_dir) - for root, _, _ in walk(src_dir, followlinks=True): - _src_dir = root - _var_dir = variant_dir + root.replace(src_dir, "") - if any([s in _var_dir.lower() for s in ignore_pattern]): - continue - env.VariantDirWrap(_var_dir, _src_dir, duplicate) - variants.append(_var_dir) - return variants + if src_dir.endswith(sep): + src_dir = src_dir[:-1] + + for item in _match_sources(src_dir, src_filter or SRC_DEFAULT_FILTER): + _reldir = dirname(item) + _src_dir = join(src_dir, _reldir) + _var_dir = join(variant_dir, _reldir) + + if _var_dir not in variants: + variants.append(_var_dir) + env.VariantDirWrap(_var_dir, _src_dir, duplicate) + + if env.IsFileWithExt(item, SRC_BUILD_EXT): + sources.append(env.File(join(_var_dir, basename(item)))) + + return sources def BuildFramework(env): @@ -134,18 +171,11 @@ def BuildFramework(env): framework) -def BuildLibrary(env, variant_dir, library_dir, ignore_files=None): +def BuildLibrary(env, variant_dir, library_dir, src_filter=None): lib = env.Clone() - vdirs = lib.VariantDirRecursive( - variant_dir, library_dir, ignore_pattern=(".git", ".svn", "examples")) - srcfiles = [] - for vdir in vdirs: - for item in lib.GlobCXXFiles(vdir): - if not ignore_files or item.name not in ignore_files: - srcfiles.append(item) return lib.Library( lib.subst(variant_dir), - srcfiles + lib.LookupSources(variant_dir, library_dir, src_filter=src_filter) ) @@ -239,10 +269,10 @@ def BuildDependentLibraries(env, src_dir): # pylint: disable=R0914 return result def _process_src_dir(state, src_dir): - for root, _, _ in walk(src_dir, followlinks=True): - for node in (env.GlobCXXFiles(root) + - env.Glob(join(root, "*.h"))): - state = _parse_includes(state, node) + for root, _, files in walk(src_dir, followlinks=True): + for f in files: + if env.IsFileWithExt(f, SRC_BUILD_EXT + SRC_HEADER_EXT): + state = _parse_includes(state, env.File(join(root, f))) return state def _parse_includes(state, node): @@ -295,151 +325,6 @@ def BuildDependentLibraries(env, src_dir): # pylint: disable=R0914 return libs -class InoToCPPConverter(object): - - PROTOTYPE_RE = re.compile( - r"""^( - (\s*[a-z_\d]+){1,2} # return type - (\s+[a-z_\d]+\s*) # name of prototype - \([a-z_,\.\*\&\[\]\s\d]*\) # arguments - )\s*\{ # must end with { - """, - re.X | re.M | re.I - ) - - DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) - - STRIPCOMMENTS_RE = re.compile(r"(/\*.*?\*/|(^|\s+)//[^\r\n]*$)", - re.M | re.S) - - def __init__(self, nodes): - self.nodes = nodes - - def is_main_node(self, contents): - return self.DETECTMAIN_RE.search(contents) - - @staticmethod - def _replace_comments_callback(match): - if "\n" in match.group(1): - return "\n" * match.group(1).count("\n") - else: - return " " - - def _parse_prototypes(self, contents): - prototypes = [] - reserved_keywords = set(["if", "else", "while"]) - for item in self.PROTOTYPE_RE.findall(contents): - if set([item[1].strip(), item[2].strip()]) & reserved_keywords: - continue - prototypes.append(item[0]) - return prototypes - - def append_prototypes(self, fname, contents, prototypes): - contents = self.STRIPCOMMENTS_RE.sub(self._replace_comments_callback, - contents) - result = [] - is_appended = False - linenum = 0 - for line in contents.splitlines(): - linenum += 1 - line = line.strip() - - if not is_appended and line and not line.startswith("#"): - is_appended = True - result.append("%s;" % ";\n".join(prototypes)) - result.append('#line %d "%s"' % (linenum, fname)) - - result.append(line) - - return result - - def convert(self): - prototypes = [] - data = [] - for node in self.nodes: - ino_contents = node.get_text_contents() - prototypes += self._parse_prototypes(ino_contents) - - item = (basename(node.get_path()), ino_contents) - if self.is_main_node(ino_contents): - data = [item] + data - else: - data.append(item) - - if not data: - return None - - result = ["#include "] - is_first = True - - for name, contents in data: - if is_first and prototypes: - result += self.append_prototypes(name, contents, prototypes) - else: - result.append('#line 1 "%s"' % name) - result.append(contents) - is_first = False - - return "\n".join(result) - - -def ConvertInoToCpp(env): - - def delete_tmpcpp_file(file_): - remove(file_) - - ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) + - env.Glob(join("$PROJECTSRC_DIR", "*.pde"))) - - c = InoToCPPConverter(ino_nodes) - data = c.convert() - - if not data: - return - - tmpcpp_file = join(env.subst("$PROJECTSRC_DIR"), "tmp_ino_to.cpp") - with open(tmpcpp_file, "w") as f: - f.write(data) - - atexit.register(delete_tmpcpp_file, tmpcpp_file) - - -def DumpIDEData(env): - data = { - "defines": [], - "includes": [] - } - - # includes from framework and libs - for item in env.get("VARIANT_DIRS", []): - data['includes'].append(env.subst(item[1])) - - # includes from toolchain - toolchain_dir = env.subst( - join("$PIOPACKAGES_DIR", "$PIOPACKAGE_TOOLCHAIN")) - toolchain_incglobs = [ - join(toolchain_dir, "*", "include"), - join(toolchain_dir, "lib", "gcc", "*", "*", "include") - ] - for g in toolchain_incglobs: - data['includes'].extend(glob(g)) - - # global symbols - for item in env.get("CPPDEFINES", []): - data['defines'].append(env.subst(item)) - - # special symbol for Atmel AVR MCU - board = env.get("BOARD_OPTIONS", {}) - if board and board['platform'] == "atmelavr": - data['defines'].append( - "__AVR_%s__" % board['build']['mcu'].upper() - .replace("ATMEGA", "ATmega") - .replace("ATTINY", "ATtiny") - ) - - return data - - def exists(_): return True @@ -447,12 +332,10 @@ def exists(_): def generate(env): env.AddMethod(BuildFirmware) env.AddMethod(ProcessFlags) - env.AddMethod(GlobCXXFiles) + env.AddMethod(IsFileWithExt) env.AddMethod(VariantDirWrap) - env.AddMethod(VariantDirRecursive) + env.AddMethod(LookupSources) env.AddMethod(BuildFramework) env.AddMethod(BuildLibrary) env.AddMethod(BuildDependentLibraries) - env.AddMethod(ConvertInoToCpp) - env.AddMethod(DumpIDEData) return env