diff --git a/Makefile b/Makefile index 09c0b838..a0f78f40 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ format: black ./tests test: - py.test --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py + py.test --verbose -n 6 --dist=loadscope tests --ignore tests/test_examples.py before-commit: isort format lint diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 50667d4c..4e579741 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -31,7 +31,7 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import app, fs from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path -from platformio.project.helpers import get_project_dir +from platformio.project.helpers import get_build_type, get_project_dir AllowSubstExceptions(NameError) @@ -72,14 +72,6 @@ DEFAULT_ENV_OPTIONS = dict( # Propagating External Environment ENV=os.environ, UNIX_TIME=int(time()), - BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"), - BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"), - BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"), - COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"), - LIBPATH=["$BUILD_DIR"], - PROGNAME="program", - PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), - PROG_PATH="$PROGPATH", # deprecated PYTHONEXE=get_pythonexe_path(), ) @@ -126,13 +118,21 @@ env.Replace( PROJECT_DATA_DIR=config.get("platformio", "data_dir"), PROJECTDATA_DIR="$PROJECT_DATA_DIR", # legacy for dev/platform PROJECT_BUILD_DIR=config.get("platformio", "build_dir"), - BUILD_TYPE=env.GetBuildType(), + BUILD_TYPE=get_build_type(config, env["PIOENV"], COMMAND_LINE_TARGETS), + BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV", "$BUILD_TYPE"), + BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"), + BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"), BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"), + LIBPATH=["$BUILD_DIR"], LIBSOURCE_DIRS=[ config.get("platformio", "lib_dir"), os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), config.get("platformio", "globallib_dir"), ], + COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"), + PROGNAME="program", + PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), + PROG_PATH="$PROGPATH", # deprecated ) if int(ARGUMENTS.get("ISATTY", 0)): @@ -224,14 +224,14 @@ if env.IsIntegrationDump(): data = projenv.DumpIntegrationData(env) # dump to file for the further reading by project.helpers.load_build_metadata with open( - projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")), + projenv.subst(os.path.join("$BUILD_DIR", "metadata.json")), mode="w", encoding="utf8", ) as fp: json.dump(data, fp) click.echo( - "Data has been saved to the following location %s" - % projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")) + "Metadata has been saved to the following location: %s" + % projenv.subst(os.path.join("$BUILD_DIR", "metadata.json")) ) env.Exit(0) diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 74725a02..46ac5a1d 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -44,19 +44,6 @@ def scons_patched_match_splitext(path, suffixes=None): return tokens -def GetBuildType(env): - modes = [] - if ( - set(["__debug", "sizedata"]) # sizedata = for memory inspection - & set(COMMAND_LINE_TARGETS) - or env.GetProjectOption("build_type") == "debug" - ): - modes.append("debug") - if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test": - modes.append("test") - return "+".join(modes or ["release"]) - - def BuildProgram(env): env.ProcessProgramDeps() env.ProcessProjectDeps() @@ -74,7 +61,7 @@ def BuildProgram(env): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") - program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"]) + program = env.Program(env.subst("$PROGPATH"), env.get("PIOBUILDFILES", [])) env.Replace(PIOMAINPROG=program) AlwaysBuild( @@ -157,6 +144,9 @@ def ProcessProjectDeps(env): } ) + if env.IsIntegrationDump(): + return + if "test" in env["BUILD_TYPE"]: build_files_before_nums = len(env.get("PIOBUILDFILES", [])) plb.env.BuildSources( @@ -373,7 +363,6 @@ def exists(_): def generate(env): - env.AddMethod(GetBuildType) env.AddMethod(BuildProgram) env.AddMethod(ProcessProgramDeps) env.AddMethod(ProcessProjectDeps) diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index e98a9bd3..db47dc79 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -23,7 +23,7 @@ from platformio.proc import exec_command, where_is_program def IsIntegrationDump(_): - return "__idedata" in COMMAND_LINE_TARGETS + return set(["__idedata", "__metadata"]) & set(COMMAND_LINE_TARGETS) def DumpIntegrationIncludes(env): @@ -146,7 +146,7 @@ def _split_flags_string(env, s): def DumpIntegrationData(*args): projenv, globalenv = args[0:2] # pylint: disable=unbalanced-tuple-unpacking data = { - "build_type": globalenv.GetBuildType(), + "build_type": globalenv["BUILD_TYPE"], "env_name": globalenv["PIOENV"], "libsource_dirs": [ globalenv.subst(item) for item in globalenv.GetLibSourceDirs() diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index ccf331f4..e8c19013 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -57,7 +57,7 @@ class CheckToolBase: # pylint: disable=too-many-instance-attributes ] def _load_cpp_data(self): - data = load_build_metadata(self.project_dir, self.envname) + data = load_build_metadata(None, self.envname) if not data: return self.cc_flags = data.get("cc_flags", []) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index cb496581..69122fcf 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -29,7 +29,7 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes self.project_config = project_config self.env_name = env_name self.env_options = project_config.items(env=env_name, as_dict=True) - self.build_data = self._load_build_data() + self.build_metadata = self._load_debug_build_metadata() self.tool_name = None self.board_config = {} @@ -63,11 +63,11 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes @property def program_path(self): - return self.build_data["prog_path"] + return self.build_metadata["prog_path"] @property def client_executable_path(self): - return self.build_data["gdb_path"] + return self.build_metadata["gdb_path"] @property def load_cmds(self): @@ -144,8 +144,14 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes "debug_server_ready_pattern", (self.server or {}).get("ready_pattern") ) - def _load_build_data(self): - data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True) + def _load_debug_build_metadata(self): + data = load_build_metadata( + None, + self.env_name, + cache=True, + debug=True, + test=self.env_options.get("debug_test", False), + ) if not data: raise DebugInvalidOptionsError("Could not load a build configuration") return data diff --git a/platformio/fs.py b/platformio/fs.py index 1f43a7a3..178d5a49 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -34,10 +34,12 @@ class cd: self.prev_path = os.getcwd() def __enter__(self): - os.chdir(self.new_path) + if self.new_path != self.prev_path: + os.chdir(self.new_path) def __exit__(self, etype, value, traceback): - os.chdir(self.prev_path) + if self.new_path != self.prev_path: + os.chdir(self.prev_path) def get_source_dir(): diff --git a/platformio/project/commands/metadata.py b/platformio/project/commands/metadata.py index ee9f5b19..8aa750e6 100644 --- a/platformio/project/commands/metadata.py +++ b/platformio/project/commands/metadata.py @@ -42,7 +42,7 @@ def project_metadata_cmd(project_dir, environments, json_output, json_output_pat config = ProjectConfig.get_instance() config.validate(environments) environments = list(environments or config.envs()) - build_metadata = load_build_metadata(project_dir, environments) + build_metadata = load_build_metadata(None, environments) if not json_output: install_project_dependencies( diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index e75167f7..b1ae5bee 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -131,27 +131,37 @@ def compute_project_checksum(config): return checksum.hexdigest() -def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False): - assert env_or_envs - env_names = env_or_envs - if not isinstance(env_names, list): - env_names = [env_names] +def get_build_type(config, env, run_targets=None): + types = [] + run_targets = run_targets or [] + declared_build_type = config.get(f"env:{env}", "build_type") + if ( + set(["__debug", "sizedata"]) # sizedata = for memory inspection + & set(run_targets) + or declared_build_type == "debug" + ): + types.append("debug") + if "__test" in run_targets or declared_build_type == "test": + types.append("test") + return "+".join(types or ["release"]) - with fs.cd(project_dir): - result = _get_cached_build_metadata(project_dir, env_names) if cache else {} - # incompatible build-type data - for name in list(result.keys()): - build_type = result[name].get("build_type", "") - outdated_conds = [ - not build_type, - debug and "debug" not in build_type, - not debug and "debug" in build_type, - ] - if any(outdated_conds): - del result[name] - missed_env_names = set(env_names) - set(result.keys()) - if missed_env_names: - result.update(_load_build_metadata(project_dir, missed_env_names, debug)) + +def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False, test=False): + assert env_or_envs + envs = env_or_envs + if not isinstance(envs, list): + envs = [envs] + run_targets = [] + if debug: + run_targets.append("__debug") + if test: + run_targets.append("__test") + + with fs.cd(project_dir or os.getcwd()): + result = _get_cached_build_metadata(envs, run_targets) if cache else {} + missed_envs = set(envs) - set(result.keys()) + if missed_envs: + result.update(_load_build_metadata(missed_envs, run_targets)) if not isinstance(env_or_envs, list) and env_or_envs in result: return result[env_or_envs] @@ -162,16 +172,28 @@ def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False): load_project_ide_data = load_build_metadata -def _load_build_metadata(project_dir, env_names, debug=False): +def _get_cached_build_metadata(envs, run_targets=None): + config = ProjectConfig.get_instance(os.path.join(os.getcwd(), "platformio.ini")) + build_dir = config.get("platformio", "build_dir") + result = {} + for env in envs: + build_type = get_build_type(config, env, run_targets) + json_path = os.path.join(build_dir, env, build_type, "metadata.json") + if os.path.isfile(json_path): + result[env] = fs.load_json(json_path) + return result + + +def _load_build_metadata(envs, run_targets=None): # pylint: disable=import-outside-toplevel from platformio import app from platformio.run.cli import cli as cmd_run - args = ["--project-dir", project_dir, "--target", "__idedata"] - if debug: - args.extend(["--target", "__debug"]) - for name in env_names: - args.extend(["-e", name]) + args = ["--target", "__metadata"] + for target in run_targets or []: + args.extend(["--target", target]) + for env in envs: + args.extend(["-e", env]) app.set_session_var("pause_telemetry", True) result = CliRunner().invoke(cmd_run, args) app.set_session_var("pause_telemetry", False) @@ -179,18 +201,6 @@ def _load_build_metadata(project_dir, env_names, debug=False): result.exception, exception.ReturnErrorCode ): raise result.exception - if "Data has been saved to the following location" not in result.output: + if "Metadata has been saved to the following location" not in result.output: raise exception.UserSideException(result.output) - return _get_cached_build_metadata(project_dir, env_names) - - -def _get_cached_build_metadata(project_dir, env_names): - build_dir = ProjectConfig.get_instance( - os.path.join(project_dir, "platformio.ini") - ).get("platformio", "build_dir") - result = {} - for name in env_names: - if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")): - continue - result[name] = fs.load_json(os.path.join(build_dir, name, "idedata.json")) - return result + return _get_cached_build_metadata(envs, run_targets) diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index d55f9aa7..d64de579 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -27,7 +27,7 @@ from platformio.project.helpers import load_build_metadata class ProjectGenerator: def __init__(self, config, env_name, ide, boards=None): self.config = config - self.project_dir = os.path.dirname(config.path) + self.project_dir = os.getcwd() self.forced_env_name = env_name self.env_name = str(env_name or self.get_best_envname(boards)) self.ide = str(ide) @@ -101,20 +101,18 @@ class ProjectGenerator: # default env configuration tpl_vars.update(self.config.items(env=self.env_name, as_dict=True)) # build data - tpl_vars.update(load_build_metadata(self.project_dir, self.env_name) or {}) - - with fs.cd(self.project_dir): - tpl_vars.update( - { - "src_files": self.get_src_files(), - "project_src_dir": self.config.get("platformio", "src_dir"), - "project_lib_dir": self.config.get("platformio", "lib_dir"), - "project_test_dir": self.config.get("platformio", "test_dir"), - "project_libdeps_dir": os.path.join( - self.config.get("platformio", "libdeps_dir"), self.env_name - ), - } - ) + tpl_vars.update(load_build_metadata(None, self.env_name) or {}) + tpl_vars.update( + { + "src_files": self.get_src_files(), + "project_src_dir": self.config.get("platformio", "src_dir"), + "project_lib_dir": self.config.get("platformio", "lib_dir"), + "project_test_dir": self.config.get("platformio", "test_dir"), + "project_libdeps_dir": os.path.join( + self.config.get("platformio", "libdeps_dir"), self.env_name + ), + } + ) for key, value in tpl_vars.items(): if key.endswith(("_path", "_dir")): @@ -130,12 +128,9 @@ class ProjectGenerator: def get_src_files(self): result = [] - with fs.cd(self.project_dir): - for root, _, files in os.walk(self.config.get("platformio", "src_dir")): - for f in files: - result.append( - os.path.relpath(os.path.join(os.path.abspath(root), f)) - ) + for root, _, files in os.walk(self.config.get("platformio", "src_dir")): + for f in files: + result.append(os.path.relpath(os.path.join(os.path.abspath(root), f))) return result def get_tpls(self): diff --git a/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl index c65848f6..3d6a59c0 100644 --- a/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl @@ -97,7 +97,7 @@ endif() % % ide_data = {} % if leftover_envs: -% ide_data = load_build_metadata(project_dir, leftover_envs) +% ide_data = load_build_metadata(None, leftover_envs) % end % % for env, data in ide_data.items(): diff --git a/platformio/run/cli.py b/platformio/run/cli.py index 1ed84a48..ead354b4 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -308,7 +308,7 @@ def print_processing_summary(results, verbose=False): def print_target_list(envs): tabular_data = [] - for env, data in load_build_metadata(os.getcwd(), envs).items(): + for env, data in load_build_metadata(None, envs, cache=True).items(): tabular_data.extend( sorted( [ diff --git a/platformio/test/runners/readers/native.py b/platformio/test/runners/readers/native.py index 981febc3..ac339697 100644 --- a/platformio/test/runners/readers/native.py +++ b/platformio/test/runners/readers/native.py @@ -76,13 +76,18 @@ class NativeTestOutputReader: os.path.join( build_dir, self.test_runner.test_suite.env_name, + "test", "program.exe" if IS_WINDOWS else "program", ) ] # if user changed PROGNAME if not os.path.exists(cmd[0]): build_data = load_build_metadata( - os.getcwd(), self.test_runner.test_suite.env_name, cache=True + None, + self.test_runner.test_suite.env_name, + cache=True, + debug=not self.test_runner.options.without_debugging, + test=True, ) if build_data: cmd[0] = build_data["prog_path"] diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 29ff20ab..4db19547 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -268,7 +268,7 @@ test_testing_command = atmega328p -f 16000000L - ${platformio.build_dir}/${this.__env__}/firmware.elf + ${platformio.build_dir}/${this.__env__}/test/firmware.elf """ ) test_dir = project_dir / "test" / "test_dummy"