idf.py build & flash tool

Generate flasher args files & .json project info file as part of cmake build

Kconfig

@@ -20,6 +20,9 @@ config PYTHON
         The executable name/path that is used to run python. On some systems Python 2.x
         may need to be invoked as python2.
 
+        (Note: This option is used with the GNU Make build system only, not idf.py
+        or CMake-based builds.)
+
 config MAKE_WARN_UNDEFINED_VARIABLES
     bool "'make' warns on undefined variables"
     default "y"

components/esp32/CMakeLists.txt

@@ -41,6 +41,7 @@ else()
 
   target_link_libraries(esp32 "${CMAKE_CURRENT_SOURCE_DIR}/libhal.a")
   target_link_libraries(esp32 gcc)
+  target_link_libraries(esp32 "-u call_user_start_cpu0")
 
   #ld_include_panic_highint_hdl is added as an undefined symbol because otherwise the
 #linker will ignore panic_highint_hdl.S as it has no other files depending on any
@@ -69,6 +70,7 @@ else()
       COMMAND ${CMAKE_OBJCOPY} -O binary phy_init_data.obj ${PHY_INIT_DATA_BIN}
       )
     add_custom_target(phy_init_data ALL DEPENDS ${PHY_INIT_DATA_BIN})
+    add_dependencies(flash phy_init_data)
 
   endif(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
 

components/esptool_py/CMakeLists.txt

@@ -1,10 +1,9 @@
 register_config_only_component()
 
+# Generate pre-canned flasher args files suitable for passing to esptool.py
+foreach(part project app bootloader)
 configure_file(
-  "${CMAKE_CURRENT_LIST_DIR}/flash_project_args.in"
+  "${CMAKE_CURRENT_LIST_DIR}/flash_${part}_args.in"
-  "${CMAKE_BINARY_DIR}/flash_project_args"
+  "${CMAKE_BINARY_DIR}/flash_${part}_args"
   )
-if(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
+endforeach()
-  file_append_line(${CMAKE_BINARY_DIR}/flash_project_args
-    "${CONFIG_PHY_DATA_OFFSET} esp32/phy_init_data.bin")
-endif()

components/esptool_py/flash_app_args.in

@@ -0,0 +1,4 @@
+--flash_mode ${ESPFLASHMODE}
+--flash_size ${ESPFLASHSIZE}
+--flash_freq ${ESPFLASHFREQ}
+${CONFIG_APP_OFFSET} ${PROJECT_NAME}.bin

components/esptool_py/flash_bootloader_args.in

@@ -0,0 +1,4 @@
+--flash_mode ${ESPFLASHMODE}
+--flash_size ${ESPFLASHSIZE}
+--flash_freq ${ESPFLASHFREQ}
+0x1000 bootloader/bootloader.bin

components/esptool_py/flash_partition_table_args.in

@@ -0,0 +1,4 @@
+--flash_mode ${ESPFLASHMODE}
+--flash_size ${ESPFLASHSIZE}
+--flash_freq ${ESPFLASHFREQ}
+0x8000 partition_table/partition-table.bin

components/esptool_py/flash_project_args.in

@@ -1,3 +1,7 @@
+--flash_mode ${ESPFLASHMODE}
+--flash_size ${ESPFLASHSIZE}
+--flash_freq ${ESPFLASHFREQ}
 0x1000 bootloader/bootloader.bin
 0x8000 partition_table/partition-table.bin
-${CONFIG_APP_OFFSET} ${PROJECT_NAME}.elf
+${CONFIG_APP_OFFSET} ${PROJECT_NAME}.bin
+${PHY_PARTITION_OFFSET} ${PHY_PARTITION_BIN_FILE}

components/esptool_py/project_include.cmake

@@ -1,48 +1,49 @@
 # Set some global esptool.py variables
+#
+# Many of these are read when generating flash_app_args & flash_project_args
 set(ESPTOOLPY "${PYTHON}" "${CMAKE_CURRENT_LIST_DIR}/esptool/esptool.py" --chip esp32)
 set(ESPSECUREPY "${PYTHON}" "${CMAKE_CURRENT_LIST_DIR}/esptool/espsecure.py")
 
-set(ESPPORT $ENV{ESPPORT})
-if(NOT ESPPORT)
-  set(ESPPORT ${CONFIG_ESPTOOLPY_PORT})
-endif()
-
-set(ESPBAUD $ENV{ESPPORT})
-if(NOT ESPBAUD)
-  set(ESPPORT ${CONFIG_ESPTOOLPY_PORT})
-endif()
-
 set(ESPFLASHMODE ${CONFIG_ESPTOOLPY_FLASHMODE})
 set(ESPFLASHFREQ ${CONFIG_ESPTOOLPY_FLASHFREQ})
 set(ESPFLASHSIZE ${CONFIG_ESPTOOLPY_FLASHSIZE})
 
 set(ESPTOOLPY_SERIAL "${ESPTOOLPY}" --port "${ESPPORT}" --baud ${ESPBAUD})
 
-if(CONFIG_ESPTOOLPY_FLASHSIZE_DETECT)
-  set(flashsize_arg detect)
-else()
-  set(flashsize_arg ${ESPFLASHSIZE})
-endif()
-
 set(ESPTOOLPY_ELF2IMAGE_FLASH_OPTIONS --flash_mode ${ESPFLASHMODE} --flash_freq ${ESPFLASHFREQ} --flash_size ${ESPFLASHSIZE})
 
-set(ESPTOOLPY_WRITE_FLASH_OPTIONS --flash_mode ${ESPFLASHMODE} --flash_freq ${ESPFLASHFREQ} --flash_size ${flashsize_Arg})
+if(CONFIG_ESPTOOLPY_FLASHSIZE_DETECT)
+  # Set ESPFLASHSIZE to 'detect' *after* elf2image options are generated,
+  # as elf2image can't have 'detect' as an option...
+  set(ESPFLASHSIZE detect)
+endif()
 
-set(ESPTOOLPY_WRITE_FLASH ${ESPTOOLPY_SERIAL} write_flash ${ESPTOOLPY_WRITE_FLASH_OPTIONS})
+# Set variables if the PHY data partition is in the flash
+if (CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
+  set(PHY_PARTITION_OFFSET   ${CONFIG_PHY_DATA_OFFSET})
+  set(PHY_PARTITION_BIN_FILE "esp32/phy_init_data.bin")
+endif()
 
 #
-# Add 'binary' target - generates with elf2image
+# Add 'app.bin' target - generates with elf2image
 #
 add_custom_command(OUTPUT "${PROJECT_NAME}.bin"
   COMMAND ${ESPTOOLPY} elf2image ${ESPTOOLPY_ELF2IMAGE_FLASH_OPTIONS} -o "${PROJECT_NAME}.bin" "${PROJECT_NAME}.elf"
   DEPENDS ${PROJECT_NAME}.elf
   VERBATIM
   )
-add_custom_target(binary ALL DEPENDS "${PROJECT_NAME}.bin")
+add_custom_target(app ALL DEPENDS "${PROJECT_NAME}.bin")
 
 #
 # Add 'flash' target - not all build systems can run this directly
 #
-add_custom_target(flash DEPENDS binary partition_table bootloader_subproject)
+function(esptool_py_custom_target target_name flasher_filename dependencies)
-# TODO: this target should call "idftool" not esptool directly, so it can
+  add_custom_target(${target_name} DEPENDS ${dependencies}
-# override things (port, baud, etc) at runtime not configure time
+    COMMAND ${ESPTOOLPY} -p ${CONFIG_ESPTOOLPY_PORT} -b ${CONFIG_ESPTOOLPY_BAUD} write_flash @flash_${flasher_filename}_args
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    )
+endfunction()
+
+esptool_py_custom_target(flash project "app;partition_table;bootloader")
+esptool_py_custom_target(app-flash app "app")
+esptool_py_custom_target(bootloader-flash bootloader "bootloader")

toolchain.cmake

@@ -3,6 +3,5 @@ set(CMAKE_SYSTEM_NAME Generic)
 set(CMAKE_C_COMPILER xtensa-esp32-elf-gcc)
 set(CMAKE_CXX_COMPILER xtensa-esp32-elf-g++)
 set(CMAKE_ASM_COMPILER xtensa-esp32-elf-gcc)
-set(CMAKE_OBJCOPY xtensa-esp32-elf-objcopy)
 
 set(CMAKE_EXE_LINKER_FLAGS "-nostdlib" CACHE STRING "Linker Base Flags")

tools/cmake/components.cmake

@@ -65,7 +65,7 @@ function(register_component)
         message(FATAL_ERROR "${CMAKE_CURRENT_LIST_FILE}: COMPONENT_SRCDIRS entry '${dir}' does not exist")
       endif()
 
-      file(GLOB matches "${abs_dir}/*.[c|S]" "${abs_dir}/*.cpp")
+      file(GLOB matches "${abs_dir}/*.c" "${abs_dir}/*.cpp" "${abs_dir}/*.S")
       if(matches)
         list(SORT matches)
         set(COMPONENT_SRCS "${COMPONENT_SRCS};${matches}")

tools/cmake/idf_functions.cmake

@@ -22,6 +22,12 @@ macro(idf_set_global_variables)
   # Tell cmake to drop executables in the top-level build dir
   set(EXECUTABLE_OUTPUT_PATH "${CMAKE_BINARY_DIR}")
 
+  # cmake cross-toolchain doesn't include any gcc binutils names
+  set(CMAKE_OBJCOPY xtensa-esp32-elf-objcopy)
+
+  # path to idf.py tool
+  set(IDFTOOL ${PYTHON} "${IDF_PATH}/tools/idf.py")
+
 endmacro()
 
 # Add all the IDF global compiler & preprocessor options
@@ -117,6 +123,8 @@ function(idf_add_executable)
 
   add_map_file(${exe_target})
 
+  add_flasher_argfile(${exe_target})
+
 endfunction(idf_add_executable)
 
 
@@ -144,3 +152,12 @@ function(add_map_file exe_target)
     )
 
 endfunction(add_map_file)
+
+# add_flasher_argfile
+#
+# Add argument file(s) for flashing the app, and for flashing the whole system
+function(add_flasher_argfile)
+
+  
+
+endfunction(add_flasher_argfile)

tools/cmake/kconfig.cmake

@@ -6,19 +6,20 @@ macro(kconfig_set_variables)
   set_default(SDKCONFIG ${PROJECT_PATH}/sdkconfig)
   set(SDKCONFIG_HEADER ${CMAKE_BINARY_DIR}/sdkconfig.h)
   set(SDKCONFIG_CMAKE ${CMAKE_BINARY_DIR}/sdkconfig.cmake)
+  set(SDKCONFIG_JSON ${CMAKE_BINARY_DIR}/sdkconfig.json)
 
   set(ROOT_KCONFIG ${IDF_PATH}/Kconfig)
 endmacro()
 
 # Use the existing Makefile to build mconf (out of tree) when needed
 #
-# TODO: Find a solution on Windows
+# TODO: Download(?) a prebuilt mingw mconf on Windows
 ExternalProject_Add(mconf
   SOURCE_DIR ${IDF_PATH}/tools/kconfig
   CONFIGURE_COMMAND ""
   BINARY_DIR "kconfig_bin"
   BUILD_COMMAND make -f ${IDF_PATH}/tools/kconfig/Makefile mconf
-  BUILD_BYPRODUCTS "kconfig_bin" ${MCONF}
+  BUILD_BYPRODUCTS ${MCONF}
   INSTALL_COMMAND ""
   EXCLUDE_FROM_ALL 1
   )
@@ -73,21 +74,28 @@ function(kconfig_process_config)
   # Generate configuration output via confgen.py
   # makes sdkconfig.h and skdconfig.cmake
   #
-  # This happens at cmake runtime not during the build
+  # This happens during the cmake run not during the build
   execute_process(COMMAND ${confgen_basecommand}
     --output header ${SDKCONFIG_HEADER}
-    --output cmake ${SDKCONFIG_CMAKE})
+    --output cmake ${SDKCONFIG_CMAKE}
+    --output json ${SDKCONFIG_JSON}
+	RESULT_VARIABLE config_result)
+  if(config_result)
+	message(FATAL_ERROR "Failed to run confgen.py (${confgen_basecommand}). Error ${config_result}")
+  endif()
 
   # When sdkconfig file changes in the future, trigger a cmake run
-  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SDKCONFIG})
+  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${SDKCONFIG}")
 
   # Ditto if either of the generated files are missing/modified (this is a bit irritating as it means
   # you can't edit these manually without them being regenerated, but I don't know of a better way...)
-  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SDKCONFIG_HEADER})
+  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${SDKCONFIG_HEADER}")
-  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SDKCONFIG_CMAKE})
+  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${SDKCONFIG_CMAKE}")
 
   # Or if the config generation tool changes
-  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${IDF_PATH}/tools/kconfig_new/confgen.py)
+  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${IDF_PATH}/tools/kconfig_new/confgen.py")
-  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${IDF_PATH}/tools/kconfig_new/kconfiglib.py)
+  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${IDF_PATH}/tools/kconfig_new/kconfiglib.py")
+
+  set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${SDKCONFIG_HEADER}" "${SDKCONFIG_CMAKE}")
 
 endfunction()

tools/cmake/project.cmake

@@ -22,6 +22,8 @@ include(kconfig)
 include(git_submodules)
 include(idf_functions)
 
+set_default(PYTHON "python")
+
 # Verify the environment is configured correctly
 idf_verify_environment()
 
@@ -38,8 +40,6 @@ kconfig_process_config()
 # Include sdkconfig.cmake so rest of the build knows the configuration
 include(${SDKCONFIG_CMAKE})
 
-set(PYTHON "${CONFIG_PYTHON}")
-
 # Add some idf-wide definitions
 idf_set_global_compiler_options()
 
@@ -49,7 +49,6 @@ git_describe(GIT_REVISION)
 add_definitions(-DIDF_VER=\"${GIT_REVISION}\")
 git_submodule_check("${IDF_PATH}")
 
-
 # Include any top-level project_include.cmake files from components
 foreach(component ${COMPONENT_PATHS})
   include_if_exists("${component}/project_include.cmake")
@@ -68,6 +67,10 @@ endforeach()
 #
 idf_add_executable()
 
+# Write project description JSON file
+configure_file("${CMAKE_CURRENT_LIST_DIR}/project_description.json.in"
+  "${CMAKE_BINARY_DIR}/project_description.json")
+
 #
 # Finish component registration (add cross-dependencies, make
 # executable dependent on all components)

tools/cmake/project_description.json.in

@@ -0,0 +1,17 @@
+{
+    "project_name":       "${PROJECT_NAME}",
+    "project_path":       "${PROJECT_PATH}",
+    "build_dir":          "${CMAKE_BINARY_DIR}",
+    "config_file":        "${SDKCONFIG}",
+    "app_elf":            "${PROJECT_NAME}.elf",
+    "app_bin":            "${PROJECT_NAME}.bin",
+    "git_revision":       "${GIT_REVISION}",
+    "project_binaries":  {
+        "bootloader":        [ "0x1000", "bootloader/bootloader.bin" ],
+        "partition_table":   [ "0x8000", "partition_table/partition-table.bin" ],
+        "app":               [ "${CONFIG_APP_OFFSET}", "${PROJECT_NAME}.bin" ],
+        "phy_data":          [ "${PHY_PARTITION_OFFSET}", "${PHY_PARTITION_BIN_FILE}" ]
+    },
+    "phy_data_partition": "${CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION}",
+    "monitor_baud" : "${CONFIG_MONITOR_BAUD}"
+}

tools/idf.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+#
+# 'idf.py' is a top-level config/build command line tool for ESP-IDF
+#
+# You don't have to use idf.py, you can use cmake directly
+# (or use cmake in an IDE)
+#
+#
+#
+# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys
+import argparse
+import os
+import os.path
+import subprocess
+import multiprocessing
+import re
+import shutil
+import json
+
+# Use this Python interpreter for any subprocesses we launch
+PYTHON=sys.executable
+
+# note: os.environ changes don't automatically propagate to child processes,
+# you have to pass this in explicitly
+os.environ["PYTHON"]=sys.executable
+
+# Make flavors, across the various kinds of Windows environments & POSIX...
+if "MSYSTEM" in os.environ:  # MSYS
+    MAKE_CMD = "make"
+    MAKE_GENERATOR = "MSYS Makefiles"
+elif os.name == 'nt':  # other Windows
+    MAKE_CMD = "mingw32-make"
+    MAKE_GENERATOR = "MinGW Makefiles"
+else:
+    MAKE_CMD = "make"
+    MAKE_GENERATOR = "Unix Makefiles"
+
+GENERATORS = [
+    # ('generator name', 'build command line', 'version command line')
+    ("Ninja", [ "ninja" ], [ "ninja", "--version" ]),
+    (MAKE_GENERATOR, [ MAKE_CMD, "-j", str(multiprocessing.cpu_count()+2) ], [ "make", "--version" ]),
+    ]
+GENERATOR_CMDS = dict( (a[0], a[1]) for a in GENERATORS )
+
+def check_environment():
+    """
+    Verify the environment contains the top-level tools we need to operate
+
+    (cmake will check a lot of other things)
+    """
+    if not executable_exists(["cmake", "--version"]):
+        raise RuntimeError("'cmake' must be available on the PATH to use idf.py")
+    # find the directory idf.py is in, then the parent directory of this, and assume this is IDF_PATH
+    detected_idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+    if "IDF_PATH" in os.environ:
+        set_idf_path = os.path.realpath(os.environ["IDF_PATH"])
+        if set_idf_path != detected_idf_path:
+            print("WARNING: IDF_PATH environment variable is set to %s but idf.py path indicates IDF directory %s. Using the environment variable directory, but results may be unexpected..."
+                  % (set_idf_path, detected_idf_path))
+    else:
+        os.environ["IDF_PATH"] = detected_idf_path
+
+def executable_exists(args):
+    try:
+        subprocess.check_output(args)
+        return True
+    except:
+        return False
+
+def detect_cmake_generator():
+    """
+    Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found.
+    """
+    for (generator, _, version_check) in GENERATORS:
+        if executable_exists(version_check):
+            return generator
+    raise RuntimeError("To use idf.py, either the 'ninja' or 'GNU make' build tool must be available in the PATH")
+
+def _ensure_build_directory(args):
+    """Check the build directory exists and that cmake has been run there.
+
+    If this isn't the case, create the build directory (if necessary) and
+    do an initial cmake run to configure it.
+
+    This function will also check args.generator parameter. If the parameter is incompatible with
+    the build directory, an error is raised. If the parameter is None, this function will set it to
+    an auto-detected default generator or to the value already configured in the build directory.
+    """
+    project_dir = args.project_dir
+    # Verify the project directory
+    if not os.path.isdir(project_dir):
+        if not os.path.exists(project_dir):
+            raise RuntimeError("Project directory %s does not exist")
+        else:
+            raise RuntimeError("%s must be a project directory")
+    if not os.path.exists(os.path.join(project_dir, "CMakeLists.txt")):
+        raise RuntimeError("CMakeLists.txt not found in project directory %s" % project_dir)
+
+    # Verify/create the build directory
+    build_dir = args.build_dir
+    if not os.path.isdir(build_dir):
+        os.mkdir(build_dir)
+    cache_path = os.path.join(build_dir, "CMakeCache.txt")
+    if not os.path.exists(cache_path):
+        if args.generator is None:
+            args.generator = detect_cmake_generator()
+        print("Running cmake...")
+        # Note: we explicitly pass in os.environ here, as we may have set IDF_PATH there during startup
+        try:
+            subprocess.check_call(["cmake", "-G", args.generator, project_dir], env=os.environ, cwd=args.build_dir)
+        except:
+            if os.path.exists(cache_path):  # don't allow partial CMakeCache.txt files, to keep the "should I run cmake?" logic simple
+                os.remove(cache_path)
+            raise
+
+    # Learn some things from the CMakeCache.txt file in the build directory
+    cache = parse_cmakecache(cache_path)
+    try:
+        generator = cache["CMAKE_GENERATOR"]
+    except KeyError:
+        generator = detect_cmake_generator()
+    if args.generator is None:
+        args.generator = generator  # reuse the previously configured generator, if none was given
+    if generator != args.generator:
+        raise RuntimeError("Build is configured for generator '%s' not '%s'. Run 'idf.py fullclean' to start again."
+                           % (generator, args.generator))
+
+    try:
+        home_dir = cache["CMAKE_HOME_DIRECTORY"]
+        if os.path.realpath(home_dir) != os.path.realpath(project_dir):
+            raise RuntimeError("Build directory '%s' configured for project '%s' not '%s'. Run 'idf.py fullclean' to start again."
+                            % (build_dir, os.path.realpath(home_dir), os.path.realpath(project_dir)))
+    except KeyError:
+        pass  # if cmake failed part way, CMAKE_HOME_DIRECTORY may not be set yet
+
+
+def parse_cmakecache(path):
+    """
+    Parse the CMakeCache file at 'path'.
+
+    Returns a dict of name:value.
+
+    CMakeCache entries also each have a "type", but this is currently ignored.
+    """
+    result = {}
+    with open(path) as f:
+        for line in f:
+            # cmake cache lines look like: CMAKE_CXX_FLAGS_DEBUG:STRING=-g
+            # groups are name, type, value
+            m = re.match(r"^([^#/:=]+):([^:=]+)=(.+)\n$", line)
+            if m:
+               result[m.group(1)] = m.group(3)
+    return result
+
+def build_target(target_name, args):
+    """
+    Execute the target build system to build target 'target_name'
+
+    Calls _ensure_build_directory() which will run cmake to generate a build
+    directory (with the specified generator) as needed.
+    """
+    _ensure_build_directory(args)
+    generator_cmd = GENERATOR_CMDS[args.generator]
+    print("Running '%s %s' in %s..." % (generator_cmd, target_name, args.build_dir))
+    subprocess.check_call(generator_cmd + [target_name], cwd=args.build_dir)
+
+def _get_esptool_args(args):
+    esptool_path = os.path.join(os.environ["IDF_PATH"], "components/esptool_py/esptool/esptool.py")
+    result = [ PYTHON, esptool_path ]
+    if args.port is not None:
+        result += [ "-p", args.port ]
+    result += [ "-b", str(args.baud) ]
+    return result
+
+def flash(action, args):
+    """
+    Run esptool to flash the entire project, from an argfile generated by the build system
+    """
+    flasher_args_path = {
+        "bootloader-flash":      "flash_bootloader_args",
+        "partition_table-flash": "flash_partition_table_args",
+        "app-flash":             "flash_app_args",
+        "flash":                 "flash_project_args",
+    }[action]
+    esptool_args = _get_esptool_args(args)
+    esptool_args += [ "write_flash", "@"+flasher_args_path ]
+    subprocess.check_call(esptool_args, cwd=args.build_dir)
+
+def erase_flash(action, args):
+    esptool_args = _get_esptool_args(args)
+    esptool_args += [ "erase_flash" ]
+    subprocess.check_call(esptool_args, cwd=args.build_dir)
+
+def monitor(action, args):
+    """
+    Run idf_monitor.py to watch build output
+    """
+    desc_path = os.path.join(args.build_dir, "project_description.json")
+    if not os.path.exists(desc_path):
+        _ensure_build_directory(args)
+    with open(desc_path, "r") as f:
+        project_desc = json.load(f)
+
+    elf_file = os.path.join(args.build_dir, project_desc["app_elf"])
+    if not os.path.exists(elf_file):
+        raise RuntimeError("ELF file '%s' not found. You need to build & flash the project before running 'monitor', and the binary on the device must match the one in the build directory exactly. Try 'idf.py flash monitor'." % elf_file)
+    idf_monitor = os.path.join(os.environ["IDF_PATH"], "tools/idf_monitor.py")
+    monitor_args = [PYTHON, idf_monitor ]
+    if args.port is not None:
+        monitor_args += [ "-p", args.port ]
+    monitor_args += [ "-b", project_desc["monitor_baud"] ]
+    monitor_args += [ elf_file ]
+    subprocess.check_call(monitor_args, cwd=args.build_dir)
+
+def clean(action, args):
+    if not os.path.isdir(args.build_dir):
+        print("Build directory '%s' not found. Nothing to clean." % args.build_dir)
+        return
+    build_target("clean", args)
+
+def fullclean(action, args):
+    build_dir = args.build_dir
+    if not os.path.isdir(build_dir):
+        print("Build directory '%s' not found. Nothing to clean." % build_dir)
+        return
+    if len(os.listdir(build_dir)) == 0:
+        print("Build directory '%s' is empty. Nothing to clean." % build_dir)
+        return
+
+    if not os.path.exists(os.path.join(build_dir, "CMakeCache.txt")):
+        raise RuntimeError("Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically delete files in this directory. Delete the directory manually to 'clean' it." % build_dir)
+    red_flags = [ "CMakeLists.txt", ".git", ".svn" ]
+    for red in red_flags:
+        red = os.path.join(build_dir, red)
+        if os.path.exists(red):
+            raise RuntimeError("Refusing to automatically delete files in directory containing '%s'. Delete files manually if you're sure." % red)
+    # OK, delete everything in the build directory...
+    for f in os.listdir(build_dir):  # TODO: once we are Python 3 only, this can be os.scandir()
+        f = os.path.join(build_dir, f)
+        if os.path.isdir(f):
+            shutil.rmtree(f)
+        else:
+            os.remove(f)
+
+ACTIONS = {
+    # action name : ( function (or alias), dependencies, order-only dependencies )
+    "all" :                  ( build_target, [], [ "menuconfig", "clean", "fullclean" ] ),
+    "build":                 ( "all",        [], [] ),  # build is same as 'all' target
+    "clean":                 ( clean,        [], [ "fullclean" ] ),
+    "fullclean":             ( fullclean,    [], [] ),
+    "menuconfig":            ( build_target, [], [] ),
+    "size":                  ( build_target, [], [ "app" ] ),
+    "size-components":       ( build_target, [], [ "app" ] ),
+    "size-files":            ( build_target, [], [ "app" ] ),
+    "bootloader":            ( build_target, [], [] ),
+    "bootloader-clean":      ( build_target, [], [] ),
+    "bootloader-flash":      ( flash,        [ "bootloader" ], [] ),
+    "app":                   ( build_target, [], [] ),
+    "app-flash":             ( flash,        [], [ "app" ]),
+    "partition_table":       ( build_target, [], [] ),
+    "partition_table-flash": ( flash,        [ "partition_table" ], []),
+    "flash":                 ( flash,        [ "all" ], [ ] ),
+    "erase_flash":           ( erase_flash,  [], []),
+    "monitor":               ( monitor,      [], [ "flash", "partition_table-flash", "bootloader-flash", "app-flash" ]),
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(description='ESP-IDF build management tool')
+    parser.add_argument('-p', '--port', help="Serial port", default=None)
+    parser.add_argument('-b', '--baud', help="Baud rate", default=460800)
+    parser.add_argument('-C', '--project-dir', help="Project directory", default=os.getcwd())
+    parser.add_argument('-B', '--build-dir', help="Build directory", default=None)
+    parser.add_argument('-G', '--generator', help="Cmake generator", choices=GENERATOR_CMDS.keys())
+    parser.add_argument('actions', help="Actions (build targets or other operations)", nargs='+',
+                        choices=ACTIONS.keys())
+
+    args = parser.parse_args()
+
+    check_environment()
+
+    # Advanced parameter checks
+    if args.build_dir is not None and os.path.realpath(args.project_dir) == os.path.realpath(args.build_dir):
+        raise RuntimeError("Setting the build directory to the project directory is not supported. Suggest dropping --build-dir option, the default is a 'build' subdirectory inside the project directory.")
+    if args.build_dir is None:
+        args.build_dir = os.path.join(args.project_dir, "build")
+    args.build_dir = os.path.realpath(args.build_dir)
+
+    completed_actions = set()
+    def execute_action(action, remaining_actions):
+        ( function, dependencies, order_dependencies ) = ACTIONS[action]
+        # very simple dependency management, build a set of completed actions and make sure
+        # all dependencies are in it
+        for dep in dependencies:
+            if not dep in completed_actions:
+                execute_action(dep, remaining_actions)
+        for dep in order_dependencies:
+            if dep in remaining_actions and not dep in completed_actions:
+                execute_action(dep, remaining_actions)
+
+        if action in completed_actions:
+            pass  # we've already done this, don't do it twice...
+        elif function in ACTIONS:  # alias of another action
+            execute_action(function, remaining_actions)
+        else:
+            function(action, args)
+
+        completed_actions.add(action)
+
+    while len(args.actions) > 0:
+        execute_action(args.actions[0], args.actions[1:])
+        args.actions.pop(0)
+
+
+if __name__ == "__main__":
+    main()
+

tools/kconfig_new/confgen.py

@@ -25,6 +25,7 @@ import sys
 import os
 import os.path
 import tempfile
+import json
 
 import gen_kconfig_doc
 import kconfiglib
@@ -149,6 +150,26 @@ def write_cmake(config, filename):
                     prefix, sym.name, val))
         config.walk_menu(write_node)
 
+def write_json(config, filename):
+    config_dict = {}
+
+    def write_node(node):
+        sym = node.item
+        if not isinstance(sym, kconfiglib.Symbol):
+            return
+
+        val = sym.str_value  # this calculates _write_to_conf, due to kconfiglib magic
+        if sym._write_to_conf:
+            if sym.type in [ kconfiglib.BOOL, kconfiglib.TRISTATE ]:
+                val = (val != "n")
+            elif sym.type == kconfiglib.HEX:
+                val = int(val, 16)
+            elif sym.type == kconfiglib.INT:
+                val = int(val)
+            config_dict[sym.name] = val
+    config.walk_menu(write_node)
+    with open(filename, "w") as f:
+        json.dump(config_dict, f, indent=4, sort_keys=True)
 
 def update_if_changed(source, destination):
     with open(source, "r") as f:
@@ -167,7 +188,9 @@ OUTPUT_FORMATS = {
     "config" : write_config,
     "header" : write_header,
     "cmake" : write_cmake,
-    "docs" : gen_kconfig_doc.write_docs }
+    "docs" : gen_kconfig_doc.write_docs,
+    "json" : write_json,
+    }
 
 class FatalError(RuntimeError):
     """