diff --git a/tools/build_apps.py b/tools/build_apps.py new file mode 100755 index 0000000000..d25a8e279f --- /dev/null +++ b/tools/build_apps.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# coding=utf-8 +# +# ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py. +# + +import argparse +import sys +import logging +from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS + + +def main(): + parser = argparse.ArgumentParser(description="ESP-IDF app builder") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Increase the logging level of the script. Can be specified multiple times.", + ) + parser.add_argument( + "--build-verbose", + action="store_true", + help="Enable verbose output from build system.", + ) + parser.add_argument( + "--log-file", + type=argparse.FileType("w"), + help="Write the script log to the specified file, instead of stderr", + ) + parser.add_argument( + "--parallel-count", + default=1, + type=int, + help="Number of parallel build jobs. Note that this script doesn't start the jobs, " + + "it needs to be executed multiple times with same value of --parallel-count and " + + "different values of --parallel-index.", + ) + parser.add_argument( + "--parallel-index", + default=1, + type=int, + help="Index (1-based) of the job, out of the number specified by --parallel-count.", + ) + parser.add_argument( + "--format", + default="json", + choices=["json"], + help="Format to read the list of builds", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Don't actually build, only print the build commands", + ) + parser.add_argument( + "--keep-going", + action="store_true", + help="Don't exit immediately when a build fails.", + ) + parser.add_argument( + "--output-build-list", + type=argparse.FileType("w"), + help="If specified, the list of builds (with all the placeholders expanded) will be written to this file.", + ) + parser.add_argument( + "build_list", + type=argparse.FileType("r"), + nargs="?", + default=sys.stdin, + help="Name of the file to read the list of builds from. If not specified, read from stdin.", + ) + args = parser.parse_args() + + setup_logging(args) + + build_items = [BuildItem.from_json(line) for line in args.build_list] + + if not build_items: + logging.error("Empty build list!") + raise SystemExit(1) + + num_builds = len(build_items) + num_jobs = args.parallel_count + job_index = args.parallel_index - 1 # convert to 0-based index + num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs + min_job_index = num_builds_per_job * job_index + if min_job_index >= num_builds: + logging.warn("Nothing to do for job {} (build total: {}, per job: {})".format( + job_index + 1, num_builds, num_builds_per_job)) + raise SystemExit(0) + + max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1) + logging.info("Total {} builds, max. {} builds per job, running builds {}-{}".format( + num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1)) + + builds_for_current_job = build_items[min_job_index:max_job_index + 1] + for i, build_info in enumerate(builds_for_current_job): + index = i + min_job_index + 1 + build_info.index = index + build_info.dry_run = args.dry_run + build_info.verbose = args.build_verbose + build_info.keep_going = args.keep_going + logging.debug(" Build {}: {}".format(index, repr(build_info))) + if args.output_build_list: + args.output_build_list.write(build_info.to_json_expanded() + "\n") + + failed_builds = [] + for build_info in builds_for_current_job: + logging.info("Running build {}: {}".format(build_info.index, repr(build_info))) + build_system_class = BUILD_SYSTEMS[build_info.build_system] + try: + build_system_class.build(build_info) + except BuildError as e: + logging.error(e.message) + if args.keep_going: + failed_builds.append(build_info) + else: + raise SystemExit(1) + + if failed_builds: + logging.error("The following build have failed:") + for build in failed_builds: + logging.error(" {}".format(build)) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index cd83d9aa2b..f5682b98e0 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -27,6 +27,7 @@ examples/system/ota/otatool/get_running_partition.py examples/system/ota/otatool/otatool_example.py examples/system/ota/otatool/otatool_example.sh install.sh +tools/build_apps.py tools/check_kconfigs.py tools/check_python_dependencies.py tools/ci/apply_bot_filter.py @@ -59,6 +60,7 @@ tools/esp_app_trace/logtrace_proc.py tools/esp_app_trace/sysviewtrace_proc.py tools/esp_app_trace/test/logtrace/test.sh tools/esp_app_trace/test/sysview/test.sh +tools/find_apps.py tools/format.sh tools/gen_esp_err_to_name.py tools/idf.py diff --git a/tools/find_apps.py b/tools/find_apps.py new file mode 100755 index 0000000000..e3dd48c4c0 --- /dev/null +++ b/tools/find_apps.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python +# coding=utf-8 +# +# ESP-IDF helper script to enumerate the builds of multiple configurations of multiple apps. +# Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds. + +import argparse +import os +import sys +import re +import glob +import logging +import typing +from find_build_apps import ( + BUILD_SYSTEMS, + BUILD_SYSTEM_CMAKE, + BuildSystem, + BuildItem, + setup_logging, + ConfigRule, + config_rules_from_str, + DEFAULT_TARGET, +) + +# Helper functions + + +def dict_from_sdkconfig(path): + """ + Parse the sdkconfig file at 'path', return name:value pairs as a dict + """ + regex = re.compile(r"^([^#=]+)=(.+)$") + result = {} + with open(path) as f: + for line in f: + m = regex.match(line) + if m: + result[m.group(1)] = m.group(2) + return result + + +# Main logic: enumerating apps and builds + + +def find_builds_for_app( + app_path, work_dir, build_dir, build_log, target_arg, build_system, + config_rules): # type: (str, str, str, str, str, str, typing.List[ConfigRule]) -> typing.List[BuildItem] + """ + Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects + :param app_path: app directory (can be / usually will be a relative path) + :param work_dir: directory where the app should be copied before building. + May contain env. variables and placeholders. + :param build_dir: directory where the build will be done, relative to the work_dir. May contain placeholders. + :param build_log: path of the build log. May contain placeholders. May be None, in which case the log should go + into stdout/stderr. + :param target_arg: the value of IDF_TARGET passed to the script. Used to filter out configurations with + a different CONFIG_IDF_TARGET value. + :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary + :param config_rules: mapping of sdkconfig file name patterns to configuration names + :return: list of BuildItems representing build configuration of the app + """ + build_items = [] # type: typing.List[BuildItem] + default_config_name = "" + + for rule in config_rules: + if not rule.file_name: + default_config_name = rule.config_name + continue + + sdkconfig_paths = glob.glob(os.path.join(app_path, rule.file_name)) + sdkconfig_paths = sorted(sdkconfig_paths) + for sdkconfig_path in sdkconfig_paths: + + # Check if the sdkconfig file specifies IDF_TARGET, and if it is matches the --target argument. + sdkconfig_dict = dict_from_sdkconfig(sdkconfig_path) + target_from_config = sdkconfig_dict.get("CONFIG_IDF_TARGET") + if target_from_config is not None and target_from_config != target_arg: + logging.debug("Skipping sdkconfig {} which requires target {}".format( + sdkconfig_path, target_from_config)) + continue + + # Figure out the config name + config_name = rule.config_name or "" + if "*" in rule.file_name: + # convert glob pattern into a regex + regex_str = r".*" + rule.file_name.replace(".", r"\.").replace("*", r"(.*)") + groups = re.match(regex_str, sdkconfig_path) + assert groups + config_name = groups.group(1) + + sdkconfig_path = os.path.relpath(sdkconfig_path, app_path) + logging.debug('Adding build: app {}, sdkconfig {}, config name "{}"'.format( + app_path, sdkconfig_path, config_name)) + build_items.append( + BuildItem( + app_path, + work_dir, + build_dir, + build_log, + target_arg, + sdkconfig_path, + config_name, + build_system, + )) + + if not build_items: + logging.debug('Adding build: app {}, default sdkconfig, config name "{}"'.format(app_path, default_config_name)) + return [ + BuildItem( + app_path, + work_dir, + build_dir, + build_log, + target_arg, + None, + default_config_name, + build_system, + ) + ] + + return build_items + + +def find_apps(build_system_class, path, recursive, exclude_list, + target): # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str] + """ + Find app directories in path (possibly recursively), which contain apps for the given build system, compatible + with the given target. + :param build_system_class: class derived from BuildSystem, representing the build system in use + :param path: path where to look for apps + :param recursive: whether to recursively descend into nested directories if no app is found + :param exclude_list: list of paths to be excluded from the recursive search + :param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped. + :return: list of paths of the apps found + """ + build_system_name = build_system_class.NAME + logging.debug("Looking for {} apps in {}{}".format(build_system_name, path, " recursively" if recursive else "")) + if not recursive: + if exclude_list: + logging.warn("--exclude option is ignored when used without --recursive") + if not build_system_class.is_app(path): + logging.warn("Path {} specified without --recursive flag, but no {} app found there".format( + path, build_system_name)) + return [] + return [path] + + # The remaining part is for recursive == True + apps_found = [] # type: typing.List[str] + for root, dirs, _ in os.walk(path, topdown=True): + logging.debug("Entering {}".format(root)) + if root in exclude_list: + logging.debug("Skipping {} (excluded)".format(root)) + del dirs[:] + continue + + if build_system_class.is_app(root): + logging.debug("Found {} app in {}".format(build_system_name, root)) + # Don't recurse into app subdirectories + del dirs[:] + + supported_targets = build_system_class.supported_targets(root) + if supported_targets and target not in supported_targets: + logging.debug("Skipping, app only supports targets: " + ", ".join(supported_targets)) + continue + + apps_found.append(root) + + return apps_found + + +def main(): + parser = argparse.ArgumentParser(description="Tool to generate build steps for IDF apps") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Increase the logging level of the script. Can be specified multiple times.", + ) + parser.add_argument( + "--log-file", + type=argparse.FileType("w"), + help="Write the script log to the specified file, instead of stderr", + ) + parser.add_argument( + "--recursive", + action="store_true", + help="Look for apps in the specified directories recursively.", + ) + parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) + parser.add_argument( + "--work-dir", + help="If set, the app is first copied into the specified directory, and then built." + + "If not set, the work directory is the directory of the app.", + ) + parser.add_argument( + "--config", + action="append", + help="Adds configurations (sdkconfig file names) to build. This can either be " + + "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " + + "relative to the project directory, to be used. Optional NAME can be specified, " + + "which can be used as a name of this configuration. FILEPATTERN is the name of " + + "the sdkconfig file, relative to the project directory, with at most one wildcard. " + + "The part captured by the wildcard is used as the name of the configuration.", + ) + parser.add_argument( + "--build-dir", + help="If set, specifies the build directory name. Can expand placeholders. Can be either a " + + "name relative to the work directory, or an absolute path.", + ) + parser.add_argument( + "--build-log", + help="If specified, the build log will be written to this file. Can expand placeholders.", + ) + parser.add_argument("--target", help="Build apps for given target.") + parser.add_argument( + "--format", + default="json", + choices=["json"], + help="Format to write the list of builds as", + ) + parser.add_argument( + "--exclude", + action="append", + help="Ignore specified directory (if --recursive is given). Can be used multiple times.", + ) + parser.add_argument( + "-o", + "--output", + type=argparse.FileType("w"), + help="Output the list of builds to the specified file", + ) + parser.add_argument("paths", nargs="+", help="One or more app paths.") + args = parser.parse_args() + setup_logging(args) + + build_system_class = BUILD_SYSTEMS[args.build_system] + + # If the build target is not set explicitly, get it from the environment or use the default one (esp32) + if not args.target: + env_target = os.environ.get("IDF_TARGET") + if env_target: + logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target)) + args.target = env_target + else: + logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET)) + args.target = DEFAULT_TARGET + + # Prepare the list of app paths + app_paths = [] # type: typing.List[str] + for path in args.paths: + app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) + + if not app_paths: + logging.critical("No {} apps found".format(build_system_class.NAME)) + raise SystemExit(1) + logging.info("Found {} apps".format(len(app_paths))) + + app_paths = sorted(app_paths) + + # Find compatible configurations of each app, collect them as BuildItems + build_items = [] # type: typing.List[BuildItem] + config_rules = config_rules_from_str(args.config or []) + for app_path in app_paths: + build_items += find_builds_for_app( + app_path, + args.work_dir, + args.build_dir, + args.build_log, + args.target, + args.build_system, + config_rules, + ) + logging.info("Found {} builds".format(len(build_items))) + + # Write out the BuildItems. Only JSON supported now (will add YAML later). + if args.format != "json": + raise NotImplementedError() + out = args.output or sys.stdout + out.writelines([item.to_json() + "\n" for item in build_items]) + + +if __name__ == "__main__": + main() diff --git a/tools/find_build_apps/__init__.py b/tools/find_build_apps/__init__.py new file mode 100644 index 0000000000..c87ceebda0 --- /dev/null +++ b/tools/find_build_apps/__init__.py @@ -0,0 +1,31 @@ +from .common import ( + BuildItem, + BuildSystem, + BuildError, + ConfigRule, + config_rules_from_str, + setup_logging, + DEFAULT_TARGET, +) +from .cmake import CMakeBuildSystem, BUILD_SYSTEM_CMAKE +from .make import MakeBuildSystem, BUILD_SYSTEM_MAKE + +BUILD_SYSTEMS = { + BUILD_SYSTEM_MAKE: MakeBuildSystem, + BUILD_SYSTEM_CMAKE: CMakeBuildSystem, +} + +__all__ = [ + "BuildItem", + "BuildSystem", + "BuildError", + "ConfigRule", + "config_rules_from_str", + "setup_logging", + "DEFAULT_TARGET", + "CMakeBuildSystem", + "BUILD_SYSTEM_CMAKE", + "MakeBuildSystem", + "BUILD_SYSTEM_MAKE", + "BUILD_SYSTEMS", +] diff --git a/tools/find_build_apps/cmake.py b/tools/find_build_apps/cmake.py new file mode 100644 index 0000000000..8668eb2184 --- /dev/null +++ b/tools/find_build_apps/cmake.py @@ -0,0 +1,158 @@ +import os +import sys +import subprocess +import logging +import shutil +import re +from .common import BuildSystem, BuildItem, BuildError + +BUILD_SYSTEM_CMAKE = "cmake" +IDF_PY = "idf.py" + +# While ESP-IDF component CMakeLists files can be identified by the presence of 'idf_component_register' string, +# there is no equivalent for the project CMakeLists files. This seems to be the best option... +CMAKE_PROJECT_LINE = r"include($ENV{IDF_PATH}/tools/cmake/project.cmake)" + +SUPPORTED_TARGETS_REGEX = re.compile(r"set\(\s*SUPPORTED_TARGETS\s+([a-z_0-9\- ]+)\s*\)") + + +class CMakeBuildSystem(BuildSystem): + NAME = BUILD_SYSTEM_CMAKE + + @staticmethod + def build(build_item): # type: (BuildItem) -> None + app_path = build_item.app_dir + work_path = build_item.work_dir or app_path + if not build_item.build_dir: + build_path = os.path.join(work_path, "build") + elif os.path.isabs(build_item.build_dir): + build_path = build_item.build_dir + else: + build_path = os.path.join(work_path, build_item.build_dir) + + if work_path != app_path: + if os.path.exists(work_path): + logging.debug("Work directory {} exists, removing".format(work_path)) + if not build_item.dry_run: + shutil.rmtree(work_path) + logging.debug("Copying app from {} to {}".format(app_path, work_path)) + if not build_item.dry_run: + shutil.copytree(app_path, work_path) + + if os.path.exists(build_path): + logging.debug("Build directory {} exists, removing".format(build_path)) + if not build_item.dry_run: + shutil.rmtree(build_path) + + if not build_item.dry_run: + os.makedirs(build_path) + + # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of + # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file. + # + # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS + # CMake variable. However here we do this manually to perform environment variable expansion in the + # sdkconfig files. + sdkconfig_defaults_list = ["sdkconfig.defaults"] + if build_item.sdkconfig_path: + sdkconfig_defaults_list.append(build_item.sdkconfig_path) + + sdkconfig_file = os.path.join(work_path, "sdkconfig") + if os.path.exists(sdkconfig_file): + logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file)) + if not build_item.dry_run: + os.unlink(sdkconfig_file) + + logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file)) + if not build_item.dry_run: + with open(sdkconfig_file, "w") as f_out: + for sdkconfig_name in sdkconfig_defaults_list: + sdkconfig_path = os.path.join(work_path, sdkconfig_name) + if not sdkconfig_path or not os.path.exists(sdkconfig_path): + continue + logging.debug("Appending {} to sdkconfig".format(sdkconfig_name)) + with open(sdkconfig_path, "r") as f_in: + for line in f_in: + f_out.write(os.path.expandvars(line)) + # Also save the sdkconfig file in the build directory + shutil.copyfile( + os.path.join(work_path, "sdkconfig"), + os.path.join(build_path, "sdkconfig"), + ) + + else: + for sdkconfig_name in sdkconfig_defaults_list: + sdkconfig_path = os.path.join(app_path, sdkconfig_name) + if not sdkconfig_path: + continue + logging.debug("Considering sdkconfig {}".format(sdkconfig_path)) + if not os.path.exists(sdkconfig_path): + continue + logging.debug("Appending {} to sdkconfig".format(sdkconfig_name)) + + # Prepare the build arguments + args = [ + # Assume it is the responsibility of the caller to + # set up the environment (run . ./export.sh) + IDF_PY, + "-B", + build_path, + "-C", + work_path, + "-DIDF_TARGET=" + build_item.target, + ] + if build_item.verbose: + args.append("-v") + args.append("build") + cmdline = format(" ".join(args)) + logging.info("Running {}".format(cmdline)) + + if build_item.dry_run: + return + + log_file = None + build_stdout = sys.stdout + build_stderr = sys.stderr + if build_item.build_log_path: + logging.info("Writing build log to {}".format(build_item.build_log_path)) + log_file = open(build_item.build_log_path, "w") + build_stdout = log_file + build_stderr = log_file + + try: + subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr) + except subprocess.CalledProcessError as e: + raise BuildError("Build failed with exit code {}".format(e.returncode)) + finally: + if log_file: + log_file.close() + + @staticmethod + def _read_cmakelists(app_path): + cmakelists_path = os.path.join(app_path, "CMakeLists.txt") + if not os.path.exists(cmakelists_path): + return None + with open(cmakelists_path, "r") as cmakelists_file: + return cmakelists_file.read() + + @staticmethod + def is_app(path): + cmakelists_file_content = CMakeBuildSystem._read_cmakelists(path) + if not cmakelists_file_content: + return False + if CMAKE_PROJECT_LINE not in cmakelists_file_content: + return False + return True + + @staticmethod + def supported_targets(app_path): + cmakelists_file_content = CMakeBuildSystem._read_cmakelists(app_path) + if not cmakelists_file_content: + return None + match = re.findall(SUPPORTED_TARGETS_REGEX, cmakelists_file_content) + if not match: + return None + if len(match) > 1: + raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path)) + targets = match[0].split(" ") + return targets diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py new file mode 100644 index 0000000000..fc4013d561 --- /dev/null +++ b/tools/find_build_apps/common.py @@ -0,0 +1,231 @@ +# coding=utf-8 + +import sys +import os +from collections import namedtuple +import logging +import json +import typing + +DEFAULT_TARGET = "esp32" + +TARGET_PLACEHOLDER = "@t" +WILDCARD_PLACEHOLDER = "@w" +NAME_PLACEHOLDER = "@n" +FULL_NAME_PLACEHOLDER = "@f" +INDEX_PLACEHOLDER = "@i" + +# ConfigRule represents one --config argument of find_apps.py. +# file_name is the name of the sdkconfig file fragment, optionally with a single wildcard ('*' character). +# file_name can also be empty to indicate that the default configuration of the app should be used. +# config_name is the name of the corresponding build configuration, or None if the value of wildcard is to be used. +# For example: +# filename='', config_name='default' — represents the default app configuration, and gives it a name 'default' +# filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value +ConfigRule = namedtuple("ConfigRule", ["file_name", "config_name"]) + + +def config_rules_from_str(rule_strings): # type: (typing.List[str]) -> typing.List[ConfigRule] + """ + Helper function to convert strings like 'file_name=config_name' into ConfigRule objects + :param rule_strings: list of rules as strings + :return: list of ConfigRules + """ + rules = [] # type: typing.List[ConfigRule] + for rule_str in rule_strings: + items = rule_str.split("=", 2) + rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None)) + return rules + + +class BuildItem(object): + """ + Instance of this class represents one build of an application. + The parameters which distinguish the build are passed to the constructor. + """ + def __init__( + self, + app_path, + work_dir, + build_path, + build_log_path, + target, + sdkconfig_path, + config_name, + build_system, + ): + # These internal variables store the paths with environment variables and placeholders; + # Public properties with similar names use the _expand method to get the actual paths. + self._app_dir = app_path + self._work_dir = work_dir + self._build_dir = build_path + self._build_log_path = build_log_path + + self.sdkconfig_path = sdkconfig_path + self.config_name = config_name + self.target = target + self.build_system = build_system + + self._app_name = os.path.basename(os.path.normpath(app_path)) + + # Some miscellaneous build properties which are set later, at the build stage + self.index = None + self.verbose = False + self.dry_run = False + self.keep_going = False + + @property + def app_dir(self): + """ + :return: directory of the app + """ + return self._expand(self._app_dir) + + @property + def work_dir(self): + """ + :return: directory where the app should be copied to, prior to the build. Can be None, which means that the app + directory should be used. + """ + return self._expand(self._work_dir) + + @property + def build_dir(self): + """ + :return: build directory, either relative to the work directory (if relative path is used) or absolute path. + """ + return self._expand(self._build_dir) + + @property + def build_log_path(self): + """ + :return: path of the build log file + """ + return self._expand(self._build_log_path) + + def __repr__(self): + return "Build app {} for target {}, sdkconfig {} in {}".format( + self.app_dir, + self.target, + self.sdkconfig_path or "(default)", + self.build_dir, + ) + + def to_json(self): # type: () -> str + """ + :return: JSON string representing this object + """ + return self._to_json(self._app_dir, self._work_dir, self._build_dir, self._build_log_path) + + def to_json_expanded(self): # type: () -> str + """ + :return: JSON string representing this object, with all placeholders in paths expanded + """ + return self._to_json(self.app_dir, self.work_dir, self.build_dir, self.build_log_path) + + def _to_json(self, app_dir, work_dir, build_dir, build_log_path): # type: (str, str, str, str) -> str + """ + Internal function, called by to_json and to_json_expanded + """ + return json.dumps({ + "build_system": self.build_system, + "app_dir": app_dir, + "work_dir": work_dir, + "build_dir": build_dir, + "build_log_path": build_log_path, + "sdkconfig": self.sdkconfig_path, + "config": self.config_name, + "target": self.target, + "verbose": self.verbose, + }) + + @staticmethod + def from_json(json_str): # type: (typing.Text) -> BuildItem + """ + :return: Get the BuildItem from a JSON string + """ + d = json.loads(str(json_str)) + result = BuildItem( + app_path=d["app_dir"], + work_dir=d["work_dir"], + build_path=d["build_dir"], + build_log_path=d["build_log_path"], + sdkconfig_path=d["sdkconfig"], + config_name=d["config"], + target=d["target"], + build_system=d["build_system"], + ) + result.verbose = d["verbose"] + return result + + def _expand(self, path): # type: (str) -> str + """ + Internal method, expands any of the placeholders in {app,work,build} paths. + """ + if not path: + return path + + if self.index is not None: + path = path.replace(INDEX_PLACEHOLDER, str(self.index)) + path = path.replace(TARGET_PLACEHOLDER, self.target) + path = path.replace(NAME_PLACEHOLDER, self._app_name) + if (FULL_NAME_PLACEHOLDER in path): # to avoid recursion to the call to app_dir in the next line: + path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, "_")) + wildcard_pos = path.find(WILDCARD_PLACEHOLDER) + if wildcard_pos != -1: + if self.config_name: + # if config name is defined, put it in place of the placeholder + path = path.replace(WILDCARD_PLACEHOLDER, self.config_name) + else: + # otherwise, remove the placeholder and one character on the left + # (which is usually an underscore, dash, or other delimiter) + left_of_wildcard = max(0, wildcard_pos - 1) + right_of_wildcard = wildcard_pos + len(WILDCARD_PLACEHOLDER) + path = path[0:left_of_wildcard] + path[right_of_wildcard:] + path = os.path.expandvars(path) + return path + + +class BuildSystem(object): + """ + Class representing a build system. + Derived classes implement the methods below. + Objects of these classes aren't instantiated, instead the class (type object) is used. + """ + + NAME = "undefined" + + @staticmethod + def build(self): + raise NotImplementedError() + + @staticmethod + def is_app(path): + raise NotImplementedError() + + @staticmethod + def supported_targets(app_path): + raise NotImplementedError() + + +class BuildError(RuntimeError): + pass + + +def setup_logging(args): + """ + Configure logging module according to the number of '--verbose'/'-v' arguments and the --log-file argument. + :param args: namespace obtained from argparse + """ + if not args.verbose: + log_level = logging.WARNING + elif args.verbose == 1: + log_level = logging.INFO + else: + log_level = logging.DEBUG + + logging.basicConfig( + format="%(levelname)s: %(message)s", + stream=args.log_file or sys.stderr, + level=log_level, + ) diff --git a/tools/find_build_apps/make.py b/tools/find_build_apps/make.py new file mode 100644 index 0000000000..d296817baa --- /dev/null +++ b/tools/find_build_apps/make.py @@ -0,0 +1,30 @@ +import os +from .common import BuildSystem + +# Same for the Makefile projects: +MAKE_PROJECT_LINE = r"include $(IDF_PATH)/make/project.mk" + +BUILD_SYSTEM_MAKE = "make" + + +class MakeBuildSystem(BuildSystem): + NAME = BUILD_SYSTEM_MAKE + + @staticmethod + def build(build_item): + raise NotImplementedError() + + @staticmethod + def is_app(path): + makefile_path = os.path.join(path, "Makefile") + if not os.path.exists(makefile_path): + return False + with open(makefile_path, "r") as makefile: + makefile_content = makefile.read() + if MAKE_PROJECT_LINE not in makefile_content: + return False + return True + + @staticmethod + def supported_targets(app_path): + return ["esp32"]