Construct build directory based on the build type // Resolve #4373

Makefile

@@ -11,7 +11,7 @@ format:
 	black ./tests
 
 test:
-	py.test --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
+	py.test --verbose -n 6 --dist=loadscope tests --ignore tests/test_examples.py
 
 before-commit: isort format lint
 

platformio/builder/main.py

@@ -31,7 +31,7 @@ from SCons.Script import Variables  # pylint: disable=import-error
 from platformio import app, fs
 from platformio.platform.base import PlatformBase
 from platformio.proc import get_pythonexe_path
-from platformio.project.helpers import get_project_dir
+from platformio.project.helpers import get_build_type, get_project_dir
 
 AllowSubstExceptions(NameError)
 
@@ -72,14 +72,6 @@ DEFAULT_ENV_OPTIONS = dict(
     # Propagating External Environment
     ENV=os.environ,
     UNIX_TIME=int(time()),
-    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
-    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
-    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
-    LIBPATH=["$BUILD_DIR"],
-    PROGNAME="program",
-    PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
-    PROG_PATH="$PROGPATH",  # deprecated
     PYTHONEXE=get_pythonexe_path(),
 )
 
@@ -126,13 +118,21 @@ env.Replace(
     PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
     PROJECTDATA_DIR="$PROJECT_DATA_DIR",  # legacy for dev/platform
     PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
-    BUILD_TYPE=env.GetBuildType(),
+    BUILD_TYPE=get_build_type(config, env["PIOENV"], COMMAND_LINE_TARGETS),
+    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV", "$BUILD_TYPE"),
+    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
+    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
     BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
+    LIBPATH=["$BUILD_DIR"],
     LIBSOURCE_DIRS=[
         config.get("platformio", "lib_dir"),
         os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
         config.get("platformio", "globallib_dir"),
     ],
+    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
+    PROGNAME="program",
+    PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
+    PROG_PATH="$PROGPATH",  # deprecated
 )
 
 if int(ARGUMENTS.get("ISATTY", 0)):
@@ -224,14 +224,14 @@ if env.IsIntegrationDump():
     data = projenv.DumpIntegrationData(env)
     # dump to file for the further reading by project.helpers.load_build_metadata
     with open(
-        projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")),
+        projenv.subst(os.path.join("$BUILD_DIR", "metadata.json")),
         mode="w",
         encoding="utf8",
     ) as fp:
         json.dump(data, fp)
     click.echo(
-        "Data has been saved to the following location %s"
+        "Metadata has been saved to the following location: %s"
-        % projenv.subst(os.path.join("$BUILD_DIR", "idedata.json"))
+        % projenv.subst(os.path.join("$BUILD_DIR", "metadata.json"))
     )
     env.Exit(0)
 

platformio/builder/tools/piobuild.py

@@ -44,19 +44,6 @@ def scons_patched_match_splitext(path, suffixes=None):
     return tokens
 
 
-def GetBuildType(env):
-    modes = []
-    if (
-        set(["__debug", "sizedata"])  # sizedata = for memory inspection
-        & set(COMMAND_LINE_TARGETS)
-        or env.GetProjectOption("build_type") == "debug"
-    ):
-        modes.append("debug")
-    if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test":
-        modes.append("test")
-    return "+".join(modes or ["release"])
-
-
 def BuildProgram(env):
     env.ProcessProgramDeps()
     env.ProcessProjectDeps()
@@ -74,7 +61,7 @@ def BuildProgram(env):
         env.Prepend(_LIBFLAGS="-Wl,--start-group ")
         env.Append(_LIBFLAGS=" -Wl,--end-group")
 
-    program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"])
+    program = env.Program(env.subst("$PROGPATH"), env.get("PIOBUILDFILES", []))
     env.Replace(PIOMAINPROG=program)
 
     AlwaysBuild(
@@ -157,6 +144,9 @@ def ProcessProjectDeps(env):
         }
     )
 
+    if env.IsIntegrationDump():
+        return
+
     if "test" in env["BUILD_TYPE"]:
         build_files_before_nums = len(env.get("PIOBUILDFILES", []))
         plb.env.BuildSources(
@@ -373,7 +363,6 @@ def exists(_):
 
 
 def generate(env):
-    env.AddMethod(GetBuildType)
     env.AddMethod(BuildProgram)
     env.AddMethod(ProcessProgramDeps)
     env.AddMethod(ProcessProjectDeps)

platformio/builder/tools/piointegration.py

@@ -23,7 +23,7 @@ from platformio.proc import exec_command, where_is_program
 
 
 def IsIntegrationDump(_):
-    return "__idedata" in COMMAND_LINE_TARGETS
+    return set(["__idedata", "__metadata"]) & set(COMMAND_LINE_TARGETS)
 
 
 def DumpIntegrationIncludes(env):
@@ -146,7 +146,7 @@ def _split_flags_string(env, s):
 def DumpIntegrationData(*args):
     projenv, globalenv = args[0:2]  # pylint: disable=unbalanced-tuple-unpacking
     data = {
-        "build_type": globalenv.GetBuildType(),
+        "build_type": globalenv["BUILD_TYPE"],
         "env_name": globalenv["PIOENV"],
         "libsource_dirs": [
             globalenv.subst(item) for item in globalenv.GetLibSourceDirs()

platformio/check/tools/base.py

@@ -57,7 +57,7 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
         ]
 
     def _load_cpp_data(self):
-        data = load_build_metadata(self.project_dir, self.envname)
+        data = load_build_metadata(None, self.envname)
         if not data:
             return
         self.cc_flags = data.get("cc_flags", [])

platformio/debug/config/base.py

@@ -29,7 +29,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         self.project_config = project_config
         self.env_name = env_name
         self.env_options = project_config.items(env=env_name, as_dict=True)
-        self.build_data = self._load_build_data()
+        self.build_metadata = self._load_debug_build_metadata()
 
         self.tool_name = None
         self.board_config = {}
@@ -63,11 +63,11 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
 
     @property
     def program_path(self):
-        return self.build_data["prog_path"]
+        return self.build_metadata["prog_path"]
 
     @property
     def client_executable_path(self):
-        return self.build_data["gdb_path"]
+        return self.build_metadata["gdb_path"]
 
     @property
     def load_cmds(self):
@@ -144,8 +144,14 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
             "debug_server_ready_pattern", (self.server or {}).get("ready_pattern")
         )
 
-    def _load_build_data(self):
+    def _load_debug_build_metadata(self):
-        data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True)
+        data = load_build_metadata(
+            None,
+            self.env_name,
+            cache=True,
+            debug=True,
+            test=self.env_options.get("debug_test", False),
+        )
         if not data:
             raise DebugInvalidOptionsError("Could not load a build configuration")
         return data

platformio/fs.py

@@ -34,10 +34,12 @@ class cd:
         self.prev_path = os.getcwd()
 
     def __enter__(self):
-        os.chdir(self.new_path)
+        if self.new_path != self.prev_path:
+            os.chdir(self.new_path)
 
     def __exit__(self, etype, value, traceback):
-        os.chdir(self.prev_path)
+        if self.new_path != self.prev_path:
+            os.chdir(self.prev_path)
 
 
 def get_source_dir():

platformio/project/commands/metadata.py

@@ -42,7 +42,7 @@ def project_metadata_cmd(project_dir, environments, json_output, json_output_pat
         config = ProjectConfig.get_instance()
         config.validate(environments)
         environments = list(environments or config.envs())
-        build_metadata = load_build_metadata(project_dir, environments)
+        build_metadata = load_build_metadata(None, environments)
 
     if not json_output:
         install_project_dependencies(

platformio/project/helpers.py

@@ -131,27 +131,37 @@ def compute_project_checksum(config):
     return checksum.hexdigest()
 
 
-def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False):
+def get_build_type(config, env, run_targets=None):
-    assert env_or_envs
+    types = []
-    env_names = env_or_envs
+    run_targets = run_targets or []
-    if not isinstance(env_names, list):
+    declared_build_type = config.get(f"env:{env}", "build_type")
-        env_names = [env_names]
+    if (
+        set(["__debug", "sizedata"])  # sizedata = for memory inspection
+        & set(run_targets)
+        or declared_build_type == "debug"
+    ):
+        types.append("debug")
+    if "__test" in run_targets or declared_build_type == "test":
+        types.append("test")
+    return "+".join(types or ["release"])
 
-    with fs.cd(project_dir):
+
-        result = _get_cached_build_metadata(project_dir, env_names) if cache else {}
+def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False, test=False):
-        # incompatible build-type data
+    assert env_or_envs
-        for name in list(result.keys()):
+    envs = env_or_envs
-            build_type = result[name].get("build_type", "")
+    if not isinstance(envs, list):
-            outdated_conds = [
+        envs = [envs]
-                not build_type,
+    run_targets = []
-                debug and "debug" not in build_type,
+    if debug:
-                not debug and "debug" in build_type,
+        run_targets.append("__debug")
-            ]
+    if test:
-            if any(outdated_conds):
+        run_targets.append("__test")
-                del result[name]
+
-        missed_env_names = set(env_names) - set(result.keys())
+    with fs.cd(project_dir or os.getcwd()):
-        if missed_env_names:
+        result = _get_cached_build_metadata(envs, run_targets) if cache else {}
-            result.update(_load_build_metadata(project_dir, missed_env_names, debug))
+        missed_envs = set(envs) - set(result.keys())
+        if missed_envs:
+            result.update(_load_build_metadata(missed_envs, run_targets))
 
     if not isinstance(env_or_envs, list) and env_or_envs in result:
         return result[env_or_envs]
@@ -162,16 +172,28 @@ def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False):
 load_project_ide_data = load_build_metadata
 
 
-def _load_build_metadata(project_dir, env_names, debug=False):
+def _get_cached_build_metadata(envs, run_targets=None):
+    config = ProjectConfig.get_instance(os.path.join(os.getcwd(), "platformio.ini"))
+    build_dir = config.get("platformio", "build_dir")
+    result = {}
+    for env in envs:
+        build_type = get_build_type(config, env, run_targets)
+        json_path = os.path.join(build_dir, env, build_type, "metadata.json")
+        if os.path.isfile(json_path):
+            result[env] = fs.load_json(json_path)
+    return result
+
+
+def _load_build_metadata(envs, run_targets=None):
     # pylint: disable=import-outside-toplevel
     from platformio import app
     from platformio.run.cli import cli as cmd_run
 
-    args = ["--project-dir", project_dir, "--target", "__idedata"]
+    args = ["--target", "__metadata"]
-    if debug:
+    for target in run_targets or []:
-        args.extend(["--target", "__debug"])
+        args.extend(["--target", target])
-    for name in env_names:
+    for env in envs:
-        args.extend(["-e", name])
+        args.extend(["-e", env])
     app.set_session_var("pause_telemetry", True)
     result = CliRunner().invoke(cmd_run, args)
     app.set_session_var("pause_telemetry", False)
@@ -179,18 +201,6 @@ def _load_build_metadata(project_dir, env_names, debug=False):
         result.exception, exception.ReturnErrorCode
     ):
         raise result.exception
-    if "Data has been saved to the following location" not in result.output:
+    if "Metadata has been saved to the following location" not in result.output:
         raise exception.UserSideException(result.output)
-    return _get_cached_build_metadata(project_dir, env_names)
+    return _get_cached_build_metadata(envs, run_targets)
-
-
-def _get_cached_build_metadata(project_dir, env_names):
-    build_dir = ProjectConfig.get_instance(
-        os.path.join(project_dir, "platformio.ini")
-    ).get("platformio", "build_dir")
-    result = {}
-    for name in env_names:
-        if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")):
-            continue
-        result[name] = fs.load_json(os.path.join(build_dir, name, "idedata.json"))
-    return result

platformio/project/integration/generator.py

@@ -27,7 +27,7 @@ from platformio.project.helpers import load_build_metadata
 class ProjectGenerator:
     def __init__(self, config, env_name, ide, boards=None):
         self.config = config
-        self.project_dir = os.path.dirname(config.path)
+        self.project_dir = os.getcwd()
         self.forced_env_name = env_name
         self.env_name = str(env_name or self.get_best_envname(boards))
         self.ide = str(ide)
@@ -101,20 +101,18 @@ class ProjectGenerator:
         # default env configuration
         tpl_vars.update(self.config.items(env=self.env_name, as_dict=True))
         # build data
-        tpl_vars.update(load_build_metadata(self.project_dir, self.env_name) or {})
+        tpl_vars.update(load_build_metadata(None, self.env_name) or {})
-
+        tpl_vars.update(
-        with fs.cd(self.project_dir):
+            {
-            tpl_vars.update(
+                "src_files": self.get_src_files(),
-                {
+                "project_src_dir": self.config.get("platformio", "src_dir"),
-                    "src_files": self.get_src_files(),
+                "project_lib_dir": self.config.get("platformio", "lib_dir"),
-                    "project_src_dir": self.config.get("platformio", "src_dir"),
+                "project_test_dir": self.config.get("platformio", "test_dir"),
-                    "project_lib_dir": self.config.get("platformio", "lib_dir"),
+                "project_libdeps_dir": os.path.join(
-                    "project_test_dir": self.config.get("platformio", "test_dir"),
+                    self.config.get("platformio", "libdeps_dir"), self.env_name
-                    "project_libdeps_dir": os.path.join(
+                ),
-                        self.config.get("platformio", "libdeps_dir"), self.env_name
+            }
-                    ),
+        )
-                }
-            )
 
         for key, value in tpl_vars.items():
             if key.endswith(("_path", "_dir")):
@@ -130,12 +128,9 @@ class ProjectGenerator:
 
     def get_src_files(self):
         result = []
-        with fs.cd(self.project_dir):
+        for root, _, files in os.walk(self.config.get("platformio", "src_dir")):
-            for root, _, files in os.walk(self.config.get("platformio", "src_dir")):
+            for f in files:
-                for f in files:
+                result.append(os.path.relpath(os.path.join(os.path.abspath(root), f)))
-                    result.append(
-                        os.path.relpath(os.path.join(os.path.abspath(root), f))
-                    )
         return result
 
     def get_tpls(self):

platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -97,7 +97,7 @@ endif()
 %
 % ide_data = {}
 % if leftover_envs:
-%   ide_data = load_build_metadata(project_dir, leftover_envs)
+%   ide_data = load_build_metadata(None, leftover_envs)
 % end
 %
 % for env, data in ide_data.items():

platformio/run/cli.py

@@ -308,7 +308,7 @@ def print_processing_summary(results, verbose=False):
 
 def print_target_list(envs):
     tabular_data = []
-    for env, data in load_build_metadata(os.getcwd(), envs).items():
+    for env, data in load_build_metadata(None, envs, cache=True).items():
         tabular_data.extend(
             sorted(
                 [

platformio/test/runners/readers/native.py

@@ -76,13 +76,18 @@ class NativeTestOutputReader:
             os.path.join(
                 build_dir,
                 self.test_runner.test_suite.env_name,
+                "test",
                 "program.exe" if IS_WINDOWS else "program",
             )
         ]
         # if user changed PROGNAME
         if not os.path.exists(cmd[0]):
             build_data = load_build_metadata(
-                os.getcwd(), self.test_runner.test_suite.env_name, cache=True
+                None,
+                self.test_runner.test_suite.env_name,
+                cache=True,
+                debug=not self.test_runner.options.without_debugging,
+                test=True,
             )
             if build_data:
                 cmd[0] = build_data["prog_path"]

tests/commands/test_test.py

@@ -268,7 +268,7 @@ test_testing_command =
     atmega328p
     -f
     16000000L
-    ${platformio.build_dir}/${this.__env__}/firmware.elf
+    ${platformio.build_dir}/${this.__env__}/test/firmware.elf
 """
     )
     test_dir = project_dir / "test" / "test_dummy"