From 3d8183a2e1aa2f02f0761d5b33f1c01051cd888b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Dec 2017 15:38:13 +0200 Subject: [PATCH] LDF: handle "include" folder per project; fix issue with header files // Resolve #1235 --- examples | 2 +- platformio/builder/tools/pioide.py | 2 +- platformio/builder/tools/piolib.py | 178 +++++++++++++++---------- platformio/builder/tools/piomisc.py | 3 +- platformio/builder/tools/platformio.py | 18 ++- 5 files changed, 119 insertions(+), 84 deletions(-) diff --git a/examples b/examples index 2998735e..2d716306 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 2998735ea5870291766161fae7c9ecc611a33d66 +Subproject commit 2d716306f33cbaa3d9146e417d02e15747cadb2a diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index d344b228..726d52dd 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -32,7 +32,7 @@ def _dump_includes(env): # installed libs for lb in env.GetLibBuilders(): - includes.extend(lb.get_inc_dirs()) + includes.extend(lb.get_include_dirs()) # includes from toolchains p = env.PioPlatform() diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index fdf351bb..f1f4c33c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -24,7 +24,7 @@ from os.path import (basename, commonprefix, dirname, isdir, isfile, join, from platform import system import SCons.Scanner -from SCons.Script import ARGUMENTS, DefaultEnvironment +from SCons.Script import ARGUMENTS, COMMAND_LINE_TARGETS, DefaultEnvironment from platformio import util from platformio.builder.tools import platformio as piotool @@ -89,7 +89,7 @@ class LibBuilderBase(object): CLASSIC_SCANNER = SCons.Scanner.C.CScanner() ADVANCED_SCANNER = SCons.Scanner.C.CScanner(advanced=True) PARSE_SRC_BY_H_NAME = True - _INC_DIRS_CACHE = None + _INCLUDE_DIRS_CACHE = None def __init__(self, env, path, manifest=None, verbose=False): self.env = env.Clone() @@ -102,7 +102,7 @@ class LibBuilderBase(object): self._is_built = False self._depbuilders = list() self._circular_deps = list() - self._scanned_paths = list() + self._processed_files = list() # reset source filter, could be overridden with extra script self.env['SRC_FILTER'] = "" @@ -143,21 +143,28 @@ class LibBuilderBase(object): "-" % os.sep ] + @property + def include_dir(self): + if not all([isdir(join(self.path, d)) for d in ("include", "src")]): + return None + return join(self.path, "include") + @property def src_dir(self): return (join(self.path, "src") if isdir(join(self.path, "src")) else self.path) + def get_include_dirs(self): + items = [self.src_dir] + include_dir = self.include_dir + if include_dir and include_dir not in items: + items.append(include_dir) + return items + @property def build_dir(self): return join("$BUILD_DIR", "lib", basename(self.path)) - def get_inc_dirs(self): - items = [self.src_dir] - if all([isdir(join(self.path, d)) for d in ("include", "src")]): - items.append(join(self.path, "include")) - return items - @property def build_flags(self): return None @@ -239,13 +246,6 @@ class LibBuilderBase(object): def load_manifest(self): return {} - def get_src_files(self): - return [ - join(self.src_dir, item) - for item in self.env.MatchSourceFiles(self.src_dir, - self.src_filter) - ] - def process_extra_options(self): with util.cd(self.path): self.env.ProcessUnFlags(self.build_unflags) @@ -298,44 +298,60 @@ class LibBuilderBase(object): "library\n" % (item['name'], self.name)) self.env.Exit(1) - def _validate_search_paths(self, search_paths=None): - if not search_paths: - search_paths = [] - assert isinstance(search_paths, list) + def get_search_files(self): + items = [ + join(self.src_dir, item) + for item in self.env.MatchSourceFiles(self.src_dir, + self.src_filter) + ] + include_dir = self.include_dir + if include_dir: + items.extend([ + join(include_dir, item) + for item in self.env.MatchSourceFiles(include_dir) + ]) + return items - _search_paths = [] - for path in search_paths: - if path not in self._scanned_paths: - _search_paths.append(path) - self._scanned_paths.append(path) + def _validate_search_files(self, search_files=None): + if not search_files: + search_files = [] + assert isinstance(search_files, list) - return _search_paths + _search_files = [] + for path in search_files: + if path not in self._processed_files: + _search_files.append(path) + self._processed_files.append(path) - def _get_found_includes(self, search_paths=None): + return _search_files + + def _get_found_includes(self, search_files=None): # all include directories - if not LibBuilderBase._INC_DIRS_CACHE: - LibBuilderBase._INC_DIRS_CACHE = [] + if not LibBuilderBase._INCLUDE_DIRS_CACHE: + LibBuilderBase._INCLUDE_DIRS_CACHE = [] for lb in self.env.GetLibBuilders(): - LibBuilderBase._INC_DIRS_CACHE.extend( - [self.env.Dir(d) for d in lb.get_inc_dirs()]) + LibBuilderBase._INCLUDE_DIRS_CACHE.extend( + [self.env.Dir(d) for d in lb.get_include_dirs()]) # append self include directories - inc_dirs = [self.env.Dir(d) for d in self.get_inc_dirs()] - inc_dirs.extend(LibBuilderBase._INC_DIRS_CACHE) + include_dirs = [self.env.Dir(d) for d in self.get_include_dirs()] + include_dirs.extend(LibBuilderBase._INCLUDE_DIRS_CACHE) result = [] - for path in self._validate_search_paths(search_paths): + for path in self._validate_search_files(search_files): try: assert "+" in self.lib_ldf_mode incs = self.env.File(path).get_found_includes( - self.env, LibBuilderBase.ADVANCED_SCANNER, tuple(inc_dirs)) + self.env, LibBuilderBase.ADVANCED_SCANNER, + tuple(include_dirs)) except Exception as e: # pylint: disable=broad-except if self.verbose and "+" in self.lib_ldf_mode: sys.stderr.write( "Warning! Classic Pre Processor is used for `%s`, " "advanced has failed with `%s`\n" % (path, e)) _incs = self.env.File(path).get_found_includes( - self.env, LibBuilderBase.CLASSIC_SCANNER, tuple(inc_dirs)) + self.env, LibBuilderBase.CLASSIC_SCANNER, + tuple(include_dirs)) incs = [] for inc in _incs: incs.append(inc) @@ -356,7 +372,7 @@ class LibBuilderBase(object): result.append(inc) return result - def depend_recursive(self, lb, search_paths=None): + def depend_recursive(self, lb, search_files=None): def _already_depends(_lb): if self in _lb.depbuilders: @@ -376,23 +392,23 @@ class LibBuilderBase(object): self._circular_deps.append(lb) elif lb not in self._depbuilders: self._depbuilders.append(lb) - LibBuilderBase._INC_DIRS_CACHE = None - lb.search_deps_recursive(search_paths) + LibBuilderBase._INCLUDE_DIRS_CACHE = None + lb.search_deps_recursive(search_files) - def search_deps_recursive(self, search_paths=None): + def search_deps_recursive(self, search_files=None): if not self._is_dependent: self._is_dependent = True self.process_dependencies() if self.lib_ldf_mode.startswith("deep"): - search_paths = self.get_src_files() + search_files = self.get_search_files() # when LDF is disabled if self.lib_ldf_mode == "off": return lib_inc_map = {} - for inc in self._get_found_includes(search_paths): + for inc in self._get_found_includes(search_files): for lb in self.env.GetLibBuilders(): if inc.get_abspath() in lb: if lb not in lib_inc_map: @@ -400,8 +416,8 @@ class LibBuilderBase(object): lib_inc_map[lb].append(inc.get_abspath()) break - for lb, lb_search_paths in lib_inc_map.items(): - self.depend_recursive(lb, lb_search_paths) + for lb, lb_search_files in lib_inc_map.items(): + self.depend_recursive(lb, lb_search_files) def build(self): libs = [] @@ -412,13 +428,13 @@ class LibBuilderBase(object): self.env.AppendUnique(**{key: lb.env.get(key)}) for lb in self._circular_deps: - self.env.AppendUnique(CPPPATH=lb.get_inc_dirs()) + self.env.AppendUnique(CPPPATH=lb.get_include_dirs()) if self._is_built: return libs self._is_built = True - self.env.AppendUnique(CPPPATH=self.get_inc_dirs()) + self.env.AppendUnique(CPPPATH=self.get_include_dirs()) if self.lib_ldf_mode == "off": for lb in self.env.GetLibBuilders(): @@ -455,13 +471,13 @@ class ArduinoLibBuilder(LibBuilderBase): manifest[key.strip()] = value.strip() return manifest - def get_inc_dirs(self): - inc_dirs = LibBuilderBase.get_inc_dirs(self) + def get_include_dirs(self): + include_dirs = LibBuilderBase.get_include_dirs(self) if isdir(join(self.path, "src")): - return inc_dirs + return include_dirs if isdir(join(self.path, "utility")): - inc_dirs.append(join(self.path, "utility")) - return inc_dirs + include_dirs.append(join(self.path, "utility")) + return include_dirs @property def src_filter(self): @@ -486,19 +502,25 @@ class MbedLibBuilder(LibBuilderBase): return {} return util.load_json(join(self.path, "module.json")) + @property + def include_dir(self): + if isdir(join(self.path, "include")): + return join(self.path, "include") + return None + @property def src_dir(self): if isdir(join(self.path, "source")): return join(self.path, "source") return LibBuilderBase.src_dir.fget(self) - def get_inc_dirs(self): - inc_dirs = LibBuilderBase.get_inc_dirs(self) - if self.path not in inc_dirs: - inc_dirs.append(self.path) + def get_include_dirs(self): + include_dirs = LibBuilderBase.get_include_dirs(self) + if self.path not in include_dirs: + include_dirs.append(self.path) for p in self._manifest.get("extraIncludes", []): - inc_dirs.append(join(self.path, p)) - return inc_dirs + include_dirs.append(join(self.path, p)) + return include_dirs def is_frameworks_compatible(self, frameworks): return self.items_in_list(frameworks, ["mbed"]) @@ -592,34 +614,48 @@ class PlatformIOLibBuilder(LibBuilderBase): return LibBuilderBase.is_frameworks_compatible(self, frameworks) return self.items_in_list(frameworks, items) - def get_inc_dirs(self): - inc_dirs = LibBuilderBase.get_inc_dirs(self) + def get_include_dirs(self): + include_dirs = LibBuilderBase.get_include_dirs(self) - # backwards compatibility with PlatformIO 2.0 + # backwards compatibility with PlatformIO 2.0 if ("build" not in self._manifest and self._is_arduino_manifest() and not isdir(join(self.path, "src")) and isdir(join(self.path, "utility"))): - inc_dirs.append(join(self.path, "utility")) + include_dirs.append(join(self.path, "utility")) for path in self.env.get("CPPPATH", []): if path not in self.envorigin.get("CPPPATH", []): - inc_dirs.append(self.env.subst(path)) - return inc_dirs + include_dirs.append(self.env.subst(path)) + return include_dirs class ProjectAsLibBuilder(LibBuilderBase): + @property + def include_dir(self): + include_dir = self.env.subst("$PROJECTINCLUDE_DIR") + return include_dir if isdir(include_dir) else None + @property def src_dir(self): return self.env.subst("$PROJECTSRC_DIR") - def get_inc_dirs(self): - inc_dirs = LibBuilderBase.get_inc_dirs(self) - inc_dirs.append(self.env.subst("$PROJECTINCLUDE_DIR")) - return inc_dirs + def get_include_dirs(self): + include_dirs = LibBuilderBase.get_include_dirs(self) + include_dirs.append(self.env.subst("$PROJECTINCLUDE_DIR")) + return include_dirs - def get_src_files(self): - return self.env.get("PROJECTBUILDFILES", []) + def get_search_files(self): + # project files + items = LibBuilderBase.get_search_files(self) + # test files + if "__test" in COMMAND_LINE_TARGETS: + items.extend([ + join("$PROJECTTEST_DIR", item) + for item in self.env.MatchSourceFiles("$PROJECTTEST_DIR", + "$PIOTEST_SRC_FILTER") + ]) + return items @property def lib_ldf_mode(self): @@ -673,7 +709,7 @@ class ProjectAsLibBuilder(LibBuilderBase): def build(self): self._is_built = True # do not build Project now - self.env.AppendUnique(CPPPATH=self.get_inc_dirs()) + self.env.AppendUnique(CPPPATH=self.get_include_dirs()) return LibBuilderBase.build(self) @@ -753,7 +789,7 @@ def BuildProjectLibraries(env): found_lbs = [lb for lb in lib_builders if lb.dependent] for lb in lib_builders: if lb in found_lbs: - lb.search_deps_recursive(lb.get_src_files()) + lb.search_deps_recursive(lb.get_search_files()) for lb in lib_builders: for deplb in lb.depbuilders[:]: if deplb not in found_lbs: diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 16dae086..7c08237f 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -293,11 +293,12 @@ def ProcessTest(env): src_filter = ["+<*.cpp>", "+<*.c>"] if "PIOTEST" in env: src_filter.append("+<%s%s>" % (env['PIOTEST'], sep)) + env.Replace(PIOTEST_SRC_FILTER=src_filter) return env.CollectBuildFiles( "$BUILDTEST_DIR", "$PROJECTTEST_DIR", - src_filter=src_filter, + "$PIOTEST_SRC_FILTER", duplicate=False) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index adfd6209..dd6be6a4 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -63,16 +63,14 @@ def BuildProgram(env): # Search for project source files env.Append( LIBPATH=["$BUILD_DIR"], - PROJECTBUILDFILES=env.CollectBuildFiles( - "$BUILDSRC_DIR", - "$PROJECTSRC_DIR", - src_filter=env.get("SRC_FILTER"), + PIOBUILDFILES=env.CollectBuildFiles( + "$BUILDSRC_DIR", "$PROJECTSRC_DIR", "$SRC_FILTER", duplicate=False)) if "__debug" in COMMAND_LINE_TARGETS: env.ProcessDebug() if "__test" in COMMAND_LINE_TARGETS: - env.Append(PROJECTBUILDFILES=env.ProcessTest()) + env.Append(PIOBUILDFILES=env.ProcessTest()) # build dependent libs env.Append(LIBS=env.BuildProjectLibraries()) @@ -90,15 +88,14 @@ def BuildProgram(env): # Handle SRC_BUILD_FLAGS env.ProcessFlags(env.get("SRC_BUILD_FLAGS")) - if not env.get("PROJECTBUILDFILES") and not COMMAND_LINE_TARGETS: + if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: sys.stderr.write( "Error: Nothing to build. Please put your source code files " "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR")) env.Exit(1) program = env.Program( - join("$BUILD_DIR", env.subst("$PROGNAME")), - env['PROJECTBUILDFILES'] + env.get("PIOBUILDFILES", [])) + join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES']) checksize_action = Action(env.CheckUploadSize, "Checking program size") AlwaysBuild(env.Alias("checkprogsize", program, checksize_action)) @@ -186,6 +183,7 @@ def MatchSourceFiles(env, src_dir, src_filter=None): items.add(item.replace(src_dir + sep, "")) src_dir = env.subst(src_dir) + src_filter = env.subst(src_filter) if src_filter else None src_filter = src_filter or SRC_FILTER_DEFAULT if isinstance(src_filter, (list, tuple)): src_filter = " ".join(src_filter) @@ -270,12 +268,12 @@ def BuildLibrary(env, variant_dir, src_dir, src_filter=None): lib = env.Clone() return lib.StaticLibrary( lib.subst(variant_dir), - lib.CollectBuildFiles(variant_dir, src_dir, src_filter=src_filter)) + lib.CollectBuildFiles(variant_dir, src_dir, src_filter)) def BuildSources(env, variant_dir, src_dir, src_filter=None): DefaultEnvironment().Append(PIOBUILDFILES=env.Clone().CollectBuildFiles( - variant_dir, src_dir, src_filter=src_filter)) + variant_dir, src_dir, src_filter)) def exists(_):