From dd63c8002aabc1e400bcc27123da84c18e2336f0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jun 2022 21:09:44 +0300 Subject: [PATCH 1/6] Make "MatchSourceFiles' configurable for source extensions --- platformio/builder/tools/platformio.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c828a943..803669f8 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -262,12 +262,11 @@ def StringifyMacro(env, value): # pylint: disable=unused-argument return '\\"%s\\"' % value.replace('"', '\\\\\\"') -def MatchSourceFiles(env, src_dir, src_filter=None): +def MatchSourceFiles(env, src_dir, src_filter=None, src_exts=None): src_filter = env.subst(src_filter) if src_filter else None src_filter = src_filter or SRC_FILTER_DEFAULT - return fs.match_src_files( - env.subst(src_dir), src_filter, SRC_BUILD_EXT + SRC_HEADER_EXT - ) + src_exts = src_exts or (SRC_BUILD_EXT + SRC_HEADER_EXT) + return fs.match_src_files(env.subst(src_dir), src_filter, src_exts) def CollectBuildFiles( @@ -280,7 +279,7 @@ def CollectBuildFiles( if src_dir.endswith(os.sep): src_dir = src_dir[:-1] - for item in env.MatchSourceFiles(src_dir, src_filter): + for item in env.MatchSourceFiles(src_dir, src_filter, SRC_BUILD_EXT): _reldir = os.path.dirname(item) _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir @@ -289,8 +288,7 @@ def CollectBuildFiles( variants.append(_var_dir) env.VariantDir(_var_dir, _src_dir, duplicate) - if fs.path_endswith_ext(item, SRC_BUILD_EXT): - sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) + sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) middlewares = env.get("__PIO_BUILD_MIDDLEWARES") if not middlewares: From 86c4bd69d2869297ec12c114d4b7799c5accc68f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jun 2022 21:17:26 +0300 Subject: [PATCH 2/6] Fixed an issue with the LDF when recursively scanning dependencies in the "chain" mode --- HISTORY.rst | 1 + platformio/builder/tools/piolib.py | 54 ++++++++++++------------------ tests/commands/test_test.py | 7 +++- 3 files changed, 28 insertions(+), 34 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6c313133..ef4e815d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -39,6 +39,7 @@ PlatformIO Core 6 * Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) * Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope * Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ +* Fixed an issue with the `LDF `__ when recursively scanning dependencies in the ``chain`` mode 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3a0072c2..75a977a0 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -318,19 +318,12 @@ class LibBuilderBase: ) def get_search_files(self): - items = [ + return [ os.path.join(self.src_dir, item) - for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter) - ] - include_dir = self.include_dir - if include_dir: - items.extend( - [ - os.path.join(include_dir, item) - for item in self.env.MatchSourceFiles(include_dir) - ] + for item in self.env.MatchSourceFiles( + self.src_dir, self.src_filter, piotool.SRC_BUILD_EXT ) - return items + ] def _get_found_includes( # pylint: disable=too-many-branches self, search_files=None @@ -366,24 +359,28 @@ class LibBuilderBase: tuple(include_dirs), depth=self.CCONDITIONAL_SCANNER_DEPTH, ) - # mark candidates already processed via Conditional Scanner - self._processed_files.extend( - [ - c.get_abspath() - for c in candidates - if c.get_abspath() not in self._processed_files - ] - ) + except Exception as e: # pylint: disable=broad-except if self.verbose and "+" in self.lib_ldf_mode: sys.stderr.write( "Warning! Classic Pre Processor is used for `%s`, " "advanced has failed with `%s`\n" % (path, e) ) - candidates = LibBuilderBase.CLASSIC_SCANNER( - self.env.File(path), self.env, tuple(include_dirs) + candidates = self.env.File(path).get_implicit_deps( + self.env, + LibBuilderBase.CLASSIC_SCANNER, + lambda _: tuple(include_dirs), ) + # mark candidates already processed + self._processed_files.extend( + [ + c.get_abspath() + for c in candidates + if c.get_abspath() not in self._processed_files + ] + ) + # print(path, [c.get_abspath() for c in candidates]) for item in candidates: if item not in result: @@ -415,11 +412,12 @@ class LibBuilderBase: lib_inc_map = {} for inc in self._get_found_includes(search_files): + inc_path = inc.get_abspath() for lb in self.env.GetLibBuilders(): - if inc.get_abspath() in lb: + if inc_path in lb: if lb not in lib_inc_map: lib_inc_map[lb] = [] - lib_inc_map[lb].append(inc.get_abspath()) + lib_inc_map[lb].append(inc_path) break for lb, lb_search_files in lib_inc_map.items(): @@ -878,16 +876,6 @@ class ProjectAsLibBuilder(LibBuilderBase): def src_dir(self): return self.env.subst("$PROJECT_SRC_DIR") - def get_include_dirs(self): - include_dirs = [] - project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - if os.path.isdir(project_include_dir): - include_dirs.append(project_include_dir) - for include_dir in super().get_include_dirs(): - if include_dir not in include_dirs: - include_dirs.append(include_dir) - return include_dirs - def get_search_files(self): items = [] build_type = self.env.GetBuildType() diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 2a6c7c56..9d7c1dd5 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -309,10 +309,15 @@ platform = native """ ) test_dir = project_dir.mkdir("test") - test_dir.join("test_main.c").write( + test_dir.join("test_main.h").write( """ #include #include + """ + ) + test_dir.join("test_main.c").write( + """ +#include "test_main.h" void setUp(){ printf("setUp called"); From 300b7b21387c1fb2e6ae22489c5ed498db8593ba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 20:10:18 +0300 Subject: [PATCH 3/6] Minor improvements to the ProjectAsLibBuilder --- platformio/builder/tools/piolib.py | 51 +++++++++++++++----------- platformio/builder/tools/platformio.py | 13 +++---- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 75a977a0..7269c708 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -332,9 +332,7 @@ class LibBuilderBase: if not LibBuilderBase._INCLUDE_DIRS_CACHE: LibBuilderBase._INCLUDE_DIRS_CACHE = [ self.env.Dir(d) - for d in ProjectAsLibBuilder( - self.envorigin, "$PROJECT_DIR" - ).get_include_dirs() + for d in ProjectAsLibBuilder.get_instance().get_include_dirs() ] for lb in self.env.GetLibBuilders(): LibBuilderBase._INCLUDE_DIRS_CACHE.extend( @@ -766,6 +764,24 @@ class PlatformIOLibBuilder(LibBuilderBase): return os.path.abspath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member + def get_include_dirs(self): + include_dirs = super().get_include_dirs() + + # backwards compatibility with PlatformIO 2.0 + if ( + "build" not in self._manifest + and self._has_arduino_manifest() + and not os.path.isdir(os.path.join(self.path, "src")) + and os.path.isdir(os.path.join(self.path, "utility")) + ): + include_dirs.append(os.path.join(self.path, "utility")) + + for path in self.env.get("CPPPATH", []): + if path not in self.envorigin.get("CPPPATH", []): + include_dirs.append(self.env.subst(path)) + + return include_dirs + @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): @@ -841,32 +857,23 @@ class PlatformIOLibBuilder(LibBuilderBase): def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"]) - def get_include_dirs(self): - include_dirs = super().get_include_dirs() - - # backwards compatibility with PlatformIO 2.0 - if ( - "build" not in self._manifest - and self._has_arduino_manifest() - and not os.path.isdir(os.path.join(self.path, "src")) - and os.path.isdir(os.path.join(self.path, "utility")) - ): - include_dirs.append(os.path.join(self.path, "utility")) - - for path in self.env.get("CPPPATH", []): - if path not in self.envorigin.get("CPPPATH", []): - include_dirs.append(self.env.subst(path)) - - return include_dirs - class ProjectAsLibBuilder(LibBuilderBase): + + _INSTANCE = None + def __init__(self, env, *args, **kwargs): # backup original value, will be reset in base.__init__ project_src_filter = env.get("SRC_FILTER") super().__init__(env, *args, **kwargs) self.env["SRC_FILTER"] = project_src_filter + @classmethod + def get_instance(cls, *args, **kwargs): + if not cls._INSTANCE: + cls._INSTANCE = ProjectAsLibBuilder(*args, **kwargs) + return cls._INSTANCE + @property def include_dir(self): include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") @@ -1129,7 +1136,7 @@ def ConfigureProjectLibBuilder(env): if lb.depbuilders: _print_deps_tree(lb, level + 1) - project = ProjectAsLibBuilder(env, "$PROJECT_DIR") + project = ProjectAsLibBuilder.get_instance(env, "$PROJECT_DIR") env.Export(dict(projenv=project.env)) ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 803669f8..e6c9d7ba 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -141,23 +141,22 @@ def ProcessProgramDeps(env): def ProcessProjectDeps(env): - project_lib_builder = env.ConfigureProjectLibBuilder() - projenv = project_lib_builder.env + plb = env.ConfigureProjectLibBuilder() # prepend project libs to the beginning of list - env.Prepend(LIBS=project_lib_builder.build()) + env.Prepend(LIBS=plb.build()) # prepend extra linker related options from libs env.PrependUnique( **{ - key: project_lib_builder.env.get(key) + key: plb.env.get(key) for key in ("LIBS", "LIBPATH", "LINKFLAGS") - if project_lib_builder.env.get(key) + if plb.env.get(key) } ) if "test" in env.GetBuildType(): build_files_before_nums = len(env.get("PIOBUILDFILES", [])) - projenv.BuildSources( + plb.env.BuildSources( "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1: @@ -168,7 +167,7 @@ def ProcessProjectDeps(env): env.Exit(1) if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"): - projenv.BuildSources( + plb.env.BuildSources( "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") ) From bb6490d6f2b62342ba8062dc5fe4bd908830f93d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 20:11:17 +0300 Subject: [PATCH 4/6] Do not automatically configure any flags for the testing frameworks --- platformio/test/runners/doctest.py | 6 ------ platformio/test/runners/googletest.py | 8 -------- 2 files changed, 14 deletions(-) diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py index d0fc931f..9a6df7cf 100644 --- a/platformio/test/runners/doctest.py +++ b/platformio/test/runners/doctest.py @@ -108,12 +108,6 @@ class DoctestTestRunner(TestRunnerBase): super().__init__(*args, **kwargs) self._tc_parser = DoctestTestCaseParser() - def configure_build_env(self, env): - env.Append(CPPDEFINES=["DOCTEST_CONFIG_COLORS_NONE"]) - if self.platform.is_embedded(): - return - env.Append(CXXFLAGS=["-std=c++11"]) - def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index 11e51ff6..973b16fa 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -17,7 +17,6 @@ import re import click -from platformio.compat import IS_WINDOWS from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase @@ -98,13 +97,6 @@ class GoogletestTestRunner(TestRunnerBase): self._tc_parser = GoogletestTestCaseParser() os.environ["GTEST_COLOR"] = "no" # disable ANSI symbols - def configure_build_env(self, env): - if self.platform.is_embedded(): - return - env.Append(CXXFLAGS=["-std=c++11"]) - if not IS_WINDOWS: - env.Append(CCFLAGS=["-pthread"], LINKFLAGS=["-pthread"]) - def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) From 655eedd7b060ad5c08a1a30a0225c40f8397bee8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 20:20:13 +0300 Subject: [PATCH 5/6] Export Unit Testing flags only to the project build environment --- HISTORY.rst | 3 ++- platformio/builder/tools/piolib.py | 2 ++ platformio/builder/tools/platformio.py | 2 -- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ef4e815d..2b307a33 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,7 +26,8 @@ PlatformIO Core 6 * **Unit Testing** - - Merged the |UNITTESTING| "building" stage with "uploading" for the embedded target (`issue #4307 `_) + - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder) + - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 7269c708..addc3ce8 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -867,6 +867,8 @@ class ProjectAsLibBuilder(LibBuilderBase): project_src_filter = env.get("SRC_FILTER") super().__init__(env, *args, **kwargs) self.env["SRC_FILTER"] = project_src_filter + if "test" in self.env.GetBuildType(): + self.env.ConfigureTestTarget() @classmethod def get_instance(cls, *args, **kwargs): diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index e6c9d7ba..c08c7438 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -126,8 +126,6 @@ def ProcessProgramDeps(env): if "debug" in env.GetBuildType(): env.ConfigureDebugTarget() - if "test" in env.GetBuildType(): - env.ConfigureTestTarget() # remove specified flags env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) From 284ccc9e8ada1bf3034228b7d4f4b7c9e0355226 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 20:53:08 +0300 Subject: [PATCH 6/6] Remove ProjectAsLibBuilder's singleton --- platformio/builder/tools/piolib.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index addc3ce8..32907106 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -332,7 +332,9 @@ class LibBuilderBase: if not LibBuilderBase._INCLUDE_DIRS_CACHE: LibBuilderBase._INCLUDE_DIRS_CACHE = [ self.env.Dir(d) - for d in ProjectAsLibBuilder.get_instance().get_include_dirs() + for d in ProjectAsLibBuilder( + self.envorigin, "$PROJECT_DIR" + ).get_include_dirs() ] for lb in self.env.GetLibBuilders(): LibBuilderBase._INCLUDE_DIRS_CACHE.extend( @@ -859,9 +861,6 @@ class PlatformIOLibBuilder(LibBuilderBase): class ProjectAsLibBuilder(LibBuilderBase): - - _INSTANCE = None - def __init__(self, env, *args, **kwargs): # backup original value, will be reset in base.__init__ project_src_filter = env.get("SRC_FILTER") @@ -870,12 +869,6 @@ class ProjectAsLibBuilder(LibBuilderBase): if "test" in self.env.GetBuildType(): self.env.ConfigureTestTarget() - @classmethod - def get_instance(cls, *args, **kwargs): - if not cls._INSTANCE: - cls._INSTANCE = ProjectAsLibBuilder(*args, **kwargs) - return cls._INSTANCE - @property def include_dir(self): include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") @@ -1138,7 +1131,7 @@ def ConfigureProjectLibBuilder(env): if lb.depbuilders: _print_deps_tree(lb, level + 1) - project = ProjectAsLibBuilder.get_instance(env, "$PROJECT_DIR") + project = ProjectAsLibBuilder(env, "$PROJECT_DIR") env.Export(dict(projenv=project.env)) ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member