Merge branch 'feature/idfpy_unknown_targets_fallback' into 'master'

idf.py: run build system target for unknown sub-commands

Closes IDF-748

See merge request espressif/esp-idf!6644

docs/en/api-guides/build-system.rst

@@ -67,7 +67,7 @@ The :ref:`getting started guide <get-started-configure>` contains a brief introd
 
 ``idf.py`` should be run in an ESP-IDF "project" directory, ie one containing a ``CMakeLists.txt`` file. Older style projects with a Makefile will not work with ``idf.py``.
 
-Type ``idf.py --help`` for a full list of commands. Here are a summary of the most useful ones:
+Type ``idf.py --help`` for a list of commands. Here are a summary of the most useful ones:
 
 - ``idf.py menuconfig`` runs the "menuconfig" tool to configure the project.
 - ``idf.py build`` will build the project found in the current directory. This can involve multiple steps:
@@ -84,6 +84,8 @@ Type ``idf.py --help`` for a full list of commands. Here are a summary of the mo
 
 Multiple ``idf.py`` commands can be combined into one. For example, ``idf.py -p COM4 clean flash monitor`` will clean the source tree, then build the project and flash it to the ESP32 before running the serial monitor.
 
+For commands that are not known to ``idf.py`` an attempt to execute them as a build system target will be made.
+
 .. note:: The environment variables ``ESPPORT`` and ``ESPBAUD`` can be used to set default values for the ``-p`` and ``-b`` options, respectively. Providing these options on the command line overrides the default.
 
 .. _idf.py-size:
@@ -101,8 +103,8 @@ The order of multiple ``idf.py`` commands on the same invocation is not importan
 
 idf.py options
 ^^^^^^^^^^^^^^
-
+To list all available root level options, run ``idf.py --help``. To list options that are specific for a subcommand, run ``idf.py <command> --help``, for example ``idf.py monitor --help``.
-To list all available options, run ``idf.py --help``.
+Here is a list of some useful options:
 
 - ``-C <dir>`` allows overriding the project directory from the default current working directory.
 - ``-B <dir>`` allows overriding the build directory from the default ``build`` subdirectory of the project directory.

tools/idf.py

@@ -70,7 +70,8 @@ def check_environment():
     if "IDF_PATH" in os.environ:
         set_idf_path = realpath(os.environ["IDF_PATH"])
         if set_idf_path != detected_idf_path:
-            print("WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. "
+            print(
+                "WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. "
                 "Using the environment variable directory, but results may be unexpected..." %
                 (set_idf_path, PROG, detected_idf_path))
     else:
@@ -189,7 +190,8 @@ def init_cli(verbose_output=None):
             self.callback(self.name, context, global_args, **action_args)
 
     class Action(click.Command):
-        def __init__(self,
+        def __init__(
+                self,
                 name=None,
                 aliases=None,
                 deprecated=False,
@@ -232,12 +234,12 @@ def init_cli(verbose_output=None):
                 self.help = "\n".join([self.help, aliases_help])
                 self.short_help = " ".join([aliases_help, self.short_help])
 
+            self.unwrapped_callback = self.callback
             if self.callback is not None:
-                callback = self.callback
 
                 def wrapped_callback(**action_args):
                     return Task(
-                        callback=callback,
+                        callback=self.unwrapped_callback,
                         name=self.name,
                         dependencies=dependencies,
                         order_dependencies=order_dependencies,
@@ -397,7 +399,8 @@ def init_cli(verbose_output=None):
                     option = Option(**option_args)
 
                     if option.scope.is_shared:
-                        raise FatalError('"%s" is defined for action "%s". '
+                        raise FatalError(
+                            '"%s" is defined for action "%s". '
                             ' "shared" options can be declared only on global level' % (option.name, name))
 
                     # Promote options to global if see for the first time
@@ -410,8 +413,13 @@ def init_cli(verbose_output=None):
             return sorted(filter(lambda name: not self._actions[name].hidden, self._actions))
 
         def get_command(self, ctx, name):
+            if name in self.commands_with_aliases:
                 return self._actions.get(self.commands_with_aliases.get(name))
 
+            # Trying fallback to build target (from "all" action) if command is not known
+            else:
+                return Action(name=name, callback=self._actions.get('fallback').unwrapped_callback)
+
         def _print_closing_message(self, args, actions):
             # print a closing message of some kind
             #
@@ -450,7 +458,8 @@ def init_cli(verbose_output=None):
                     for o, f in flash_items:
                         cmd += o + " " + flasher_path(f) + " "
 
-                print("%s %s -p %s -b %s --before %s --after %s write_flash %s" % (
+                print(
+                    "%s %s -p %s -b %s --before %s --after %s write_flash %s" % (
                         PYTHON,
                         _safe_relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]),
                         args.port or "(PORT)",
@@ -459,7 +468,8 @@ def init_cli(verbose_output=None):
                         flasher_args["extra_esptool_args"]["after"],
                         cmd.strip(),
                     ))
-                print("or run 'idf.py -p %s %s'" % (
+                print(
+                    "or run 'idf.py -p %s %s'" % (
                         args.port or "(PORT)",
                         key + "-flash" if key != "project" else "flash",
                     ))
@@ -483,7 +493,8 @@ def init_cli(verbose_output=None):
                 [item for item, count in Counter(task.name for task in tasks).items() if count > 1])
             if dupplicated_tasks:
                 dupes = ", ".join('"%s"' % t for t in dupplicated_tasks)
-                print("WARNING: Command%s found in the list of commands more than once. " %
+                print(
+                    "WARNING: Command%s found in the list of commands more than once. " %
                     ("s %s are" % dupes if len(dupplicated_tasks) > 1 else " %s is" % dupes) +
                     "Only first occurence will be executed.")
 
@@ -499,9 +510,9 @@ def init_cli(verbose_output=None):
                         default = () if option.multiple else option.default
 
                         if global_value != default and local_value != default and global_value != local_value:
-                            raise FatalError('Option "%s" provided for "%s" is already defined to a different value. '
+                            raise FatalError(
-                                             "This option can appear at most once in the command line." %
+                                'Option "%s" provided for "%s" is already defined to a different value. '
-                                             (key, task.name))
+                                "This option can appear at most once in the command line." % (key, task.name))
                         if local_value != default:
                             global_args[key] = local_value
 
@@ -537,7 +548,8 @@ def init_cli(verbose_output=None):
                         # Otherwise invoke it with default set of options
                         # and put to the front of the list of unprocessed tasks
                         else:
-                            print('Adding "%s"\'s dependency "%s" to list of commands with default set of options.' %
+                            print(
+                                'Adding "%s"\'s dependency "%s" to list of commands with default set of options.' %
                                 (task.name, dep))
                             dep_task = ctx.invoke(ctx.command.get_command(ctx, dep))
 
@@ -631,7 +643,11 @@ def init_cli(verbose_output=None):
         except NameError:
             pass
 
-    return CLI(help="ESP-IDF build management", verbose_output=verbose_output, all_actions=all_actions)
+    cli_help = (
+        "ESP-IDF CLI build management tool. "
+        "For commands that are not known to idf.py an attempt to execute it as a build system target will be made.")
+
+    return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions)
 
 
 def main():
@@ -709,7 +725,8 @@ if __name__ == "__main__":
             # Trying to find best utf-8 locale available on the system and restart python with it
             best_locale = _find_usable_locale()
 
-            print("Your environment is not configured to handle unicode filenames outside of ASCII range."
+            print(
+                "Your environment is not configured to handle unicode filenames outside of ASCII range."
                 " Environment variable LC_ALL is temporary set to %s for unicode support." % best_locale)
 
             os.environ["LC_ALL"] = best_locale

tools/idf_py_actions/constants.py

@@ -16,17 +16,23 @@ else:
     MAKE_CMD = "make"
     MAKE_GENERATOR = "Unix Makefiles"
 
-GENERATORS = [
+GENERATORS = {
-    # ('generator name', 'build command line', 'version command line', 'verbose flag')
+    # - command: build command line
-    ("Ninja", ["ninja"], ["ninja", "--version"], "-v"),
+    # - version: version command line
-    (
+    # - dry_run: command to run in dry run mode
-        MAKE_GENERATOR,
+    # - verbose_flag: verbose flag
-        [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)],
+    "Ninja": {
-        [MAKE_CMD, "--version"],
+        "command": ["ninja"],
-        "VERBOSE=1",
+        "version": ["ninja", "--version"],
-    ),
+        "dry_run": ["ninja", "-n"],
-]
+        "verbose_flag": "-v"
-GENERATOR_CMDS = dict((a[0], a[1]) for a in GENERATORS)
+    },
-GENERATOR_VERBOSE = dict((a[0], a[3]) for a in GENERATORS)
+    MAKE_GENERATOR: {
+        "command": [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)],
+        "version": [MAKE_CMD, "--version"],
+        "dry_run": [MAKE_CMD, "-n"],
+        "verbose_flag": "VERBOSE=1",
+    }
+}
 
 SUPPORTED_TARGETS = ["esp32", "esp32s2beta"]

tools/idf_py_actions/core_ext.py

@@ -1,16 +1,25 @@
 import os
 import shutil
+import subprocess
 import sys
 
 import click
 
-from idf_py_actions.constants import GENERATOR_CMDS, GENERATOR_VERBOSE, SUPPORTED_TARGETS
+from idf_py_actions.constants import GENERATORS, SUPPORTED_TARGETS
 from idf_py_actions.errors import FatalError
 from idf_py_actions.global_options import global_options
 from idf_py_actions.tools import ensure_build_directory, idf_version, merge_action_lists, realpath, run_tool
 
 
 def action_extensions(base_actions, project_path):
+    def run_target(target_name, args):
+        generator_cmd = GENERATORS[args.generator]["command"]
+
+        if args.verbose:
+            generator_cmd += [GENERATORS[args.generator]["verbose_flag"]]
+
+        run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir)
+
     def build_target(target_name, ctx, args):
         """
         Execute the target build system to build target 'target_name'
@@ -19,12 +28,23 @@ def action_extensions(base_actions, project_path):
         directory (with the specified generator) as needed.
         """
         ensure_build_directory(args, ctx.info_name)
-        generator_cmd = GENERATOR_CMDS[args.generator]
+        run_target(target_name, args)
 
-        if args.verbose:
+    def fallback_target(target_name, ctx, args):
-            generator_cmd += [GENERATOR_VERBOSE[args.generator]]
+        """
+        Execute targets that are not explicitly known to idf.py
+        """
+        ensure_build_directory(args, ctx.info_name)
 
-        run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir)
+        try:
+            subprocess.check_output(GENERATORS[args.generator]["dry_run"] + [target_name], cwd=args.cwd)
+
+        except Exception:
+            raise FatalError(
+                'command "%s" is not known to idf.py and is not a %s target' %
+                (target_name, GENERATORS[args.generator].command))
+
+        run_target(target_name, args)
 
     def verbose_callback(ctx, param, value):
         if not value or ctx.resilient_parsing:
@@ -69,7 +89,8 @@ def action_extensions(base_actions, project_path):
             return
 
         if not os.path.exists(os.path.join(build_dir, "CMakeCache.txt")):
-            raise FatalError("Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically "
+            raise FatalError(
+                "Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically "
                 "delete files in this directory. Delete the directory manually to 'clean' it." % build_dir)
         red_flags = ["CMakeLists.txt", ".git", ".svn"]
         for red in red_flags:
@@ -111,7 +132,8 @@ def action_extensions(base_actions, project_path):
     def validate_root_options(ctx, args, tasks):
         args.project_dir = realpath(args.project_dir)
         if args.build_dir is not None and args.project_dir == realpath(args.build_dir):
-            raise FatalError("Setting the build directory to the project directory is not supported. Suggest dropping "
+            raise FatalError(
+                "Setting the build directory to the project directory is not supported. Suggest dropping "
                 "--build-dir option, the default is a 'build' subdirectory inside the project directory.")
         if args.build_dir is None:
             args.build_dir = os.path.join(args.project_dir, "build")
@@ -165,7 +187,8 @@ def action_extensions(base_actions, project_path):
             },
             {
                 "names": ["--ccache/--no-ccache"],
-                "help": ("Use ccache in build. Disabled by default, unless "
+                "help": (
+                    "Use ccache in build. Disabled by default, unless "
                     "IDF_CCACHE_ENABLE environment variable is set to a non-zero value."),
                 "is_flag": True,
                 "default": os.getenv("IDF_CCACHE_ENABLE") not in [None, "", "0"],
@@ -173,7 +196,7 @@ def action_extensions(base_actions, project_path):
             {
                 "names": ["-G", "--generator"],
                 "help": "CMake generator.",
-                "type": click.Choice(GENERATOR_CMDS.keys()),
+                "type": click.Choice(GENERATORS.keys()),
             },
             {
                 "names": ["--dry-run"],
@@ -192,7 +215,8 @@ def action_extensions(base_actions, project_path):
                 "aliases": ["build"],
                 "callback": build_target,
                 "short_help": "Build the project.",
-                "help": ("Build the project. This can involve multiple steps:\n\n"
+                "help": (
+                    "Build the project. This can involve multiple steps:\n\n"
                     "1. Create the build directory if needed. "
                     "The sub-directory 'build' is used to hold build output, "
                     "although this can be changed with the -B option.\n\n"
@@ -282,6 +306,11 @@ def action_extensions(base_actions, project_path):
                 "help": "Read otadata partition.",
                 "options": global_options,
             },
+            "fallback": {
+                "callback": fallback_target,
+                "help": "Handle for targets not known for idf.py.",
+                "hidden": True
+            }
         }
     }
 
@@ -290,7 +319,8 @@ def action_extensions(base_actions, project_path):
             "reconfigure": {
                 "callback": reconfigure,
                 "short_help": "Re-run CMake.",
-                "help": ("Re-run CMake even if it doesn't seem to need re-running. "
+                "help": (
+                    "Re-run CMake even if it doesn't seem to need re-running. "
                     "This isn't necessary during normal usage, "
                     "but can be useful after adding/removing files from the source tree, "
                     "or when modifying CMake cache variables. "
@@ -302,7 +332,8 @@ def action_extensions(base_actions, project_path):
             "set-target": {
                 "callback": set_target,
                 "short_help": "Set the chip target to build.",
-                "help": ("Set the chip target to build. This will remove the "
+                "help": (
+                    "Set the chip target to build. This will remove the "
                     "existing sdkconfig file and corresponding CMakeCache and "
                     "create new ones according to the new target.\nFor example, "
                     "\"idf.py set-target esp32\" will select esp32 as the new chip "
@@ -319,7 +350,8 @@ def action_extensions(base_actions, project_path):
             "clean": {
                 "callback": clean,
                 "short_help": "Delete build output files from the build directory.",
-                "help": ("Delete build output files from the build directory, "
+                "help": (
+                    "Delete build output files from the build directory, "
                     "forcing a 'full rebuild' the next time "
                     "the project is built. Cleaning doesn't delete "
                     "CMake configuration output and some other files"),
@@ -328,7 +360,8 @@ def action_extensions(base_actions, project_path):
             "fullclean": {
                 "callback": fullclean,
                 "short_help": "Delete the entire build directory contents.",
-                "help": ("Delete the entire build directory contents. "
+                "help": (
+                    "Delete the entire build directory contents. "
                     "This includes all CMake configuration output."
                     "The next time the project is built, "
                     "CMake will configure it from scratch. "

tools/idf_py_actions/tools.py

@@ -126,9 +126,9 @@ def _detect_cmake_generator(prog_name):
     """
     Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found.
     """
-    for (generator, _, version_check, _) in GENERATORS:
+    for (generator_name,  generator) in GENERATORS.items():
-        if executable_exists(version_check):
+        if executable_exists(generator["version"]):
-            return generator
+            return generator_name
     raise FatalError("To use %s, either the 'ninja' or 'GNU make' build tool must be available in the PATH" % prog_name)