Merge branch 'feature/component_discovery' into 'master'

build system: Improvements to component discovery

See merge request !902

components/bootloader/Makefile.projbuild

@@ -1,15 +1,13 @@
-#
+# Bootloader component (top-level project parts)
-# Bootloader component
 #
 # The bootloader is not a real component that gets linked into the project.
-# Instead it is an entire standalone project ( in src/) that gets built in 
+# Instead it is an entire standalone project (in subproject/) that gets
-# the upper projects build directory. This Makefile.projbuild provides the 
+# built in the upper project's build directory. This Makefile.projbuild provides
-# glue to  build the bootloader project from the original project. It 
+# the glue to build the bootloader project from the original project. It
-# basically runs Make in the src/ directory but it needs to zero some variables
+# basically runs Make in the subproject/ directory but it needs to
-# the ESP-IDF project.mk makefile exports first, to not let them interfere.
+# zero some variables the ESP-IDF project.mk makefile exports first, to not
+# let them interfere.
 #
-ifndef IS_BOOTLOADER_BUILD
-
 BOOTLOADER_COMPONENT_PATH := $(COMPONENT_PATH)
 BOOTLOADER_BUILD_DIR=$(abspath $(BUILD_DIR_BASE)/bootloader)
 BOOTLOADER_BIN=$(BOOTLOADER_BUILD_DIR)/bootloader.bin
@@ -22,16 +20,29 @@ export SECURE_BOOT_SIGNING_KEY  # used by bootloader_support component
 BOOTLOADER_OFFSET := 0x1000
 
 # Custom recursive make for bootloader sub-project
-BOOTLOADER_MAKE=+$(MAKE) -C $(BOOTLOADER_COMPONENT_PATH)/src \
+#
-	V=$(V) BUILD_DIR_BASE=$(BOOTLOADER_BUILD_DIR) TEST_COMPONENTS= TESTS_ALL=
+# NB: Some variables are cleared in the environment, not
+# overriden, because they need to be re-defined in the child
+# project.
+BOOTLOADER_MAKE= +\
+	PROJECT_PATH= \
+	COMPONENT_DIRS= \
+	$(MAKE) -C $(BOOTLOADER_COMPONENT_PATH)/subproject \
+	V=$(V) \
+	BUILD_DIR_BASE=$(BOOTLOADER_BUILD_DIR) \
+	TEST_COMPONENTS= \
+	TESTS_ALL=
 
-.PHONY: bootloader-clean bootloader-flash bootloader $(BOOTLOADER_BIN)
+.PHONY: bootloader-clean bootloader-flash bootloader-list-components bootloader $(BOOTLOADER_BIN)
 
 $(BOOTLOADER_BIN): $(SDKCONFIG_MAKEFILE)
 	$(BOOTLOADER_MAKE) $@
 
 clean: bootloader-clean
 
+bootloader-list-components:
+	$(BOOTLOADER_MAKE) list-components
+
 ifndef CONFIG_SECURE_BOOT_ENABLED
 # If secure boot disabled, bootloader flashing is integrated
 # with 'make flash' and no warnings are printed.
@@ -110,11 +121,3 @@ endif
 bootloader-clean:
 	$(BOOTLOADER_MAKE) app-clean
 	rm -f $(SECURE_BOOTLOADER_KEY) $(BOOTLOADER_DIGEST_BIN)
-
-$(BOOTLOADER_BUILD_DIR):
-	mkdir -p $@
-
-else
-CFLAGS += -D BOOTLOADER_BUILD=1 -I $(IDF_PATH)/components/esp32/include
-
-endif

components/bootloader/component.mk

@@ -0,0 +1,7 @@
+# bootloader component is special, as bootloader is also a project.
+#
+# This top-level component is only configuration files for the IDF project.
+#
+# See Makefile.projbuild for the targets which actually build the bootloader.
+COMPONENT_CONFIG_ONLY := 1
+

components/bootloader/subproject/Makefile

@@ -2,12 +2,16 @@
 # This is a project Makefile. It is assumed the directory this Makefile resides in is a
 # project subdirectory.
 #
+ifeq ("$(MAKELEVEL)","0")
+$(error Bootloader makefile expects to be run as part of 'make bootloader' from a top-level project.)
+endif
 
 PROJECT_NAME := bootloader
 
+COMPONENTS := esptool_py bootloader_support log spi_flash micro-ecc soc main
+
 #We cannot include the esp32 component directly but we need its includes.
-#This is fixed by adding CFLAGS from Makefile.projbuild
+CFLAGS += -I $(IDF_PATH)/components/esp32/include
-COMPONENTS := esptool_py bootloader bootloader_support log spi_flash micro-ecc soc
 
 # The bootloader pseudo-component is also included in this build, for its Kconfig.projbuild to be included.
 #
@@ -15,6 +19,9 @@ COMPONENTS := esptool_py bootloader bootloader_support log spi_flash micro-ecc s
 IS_BOOTLOADER_BUILD := 1
 export IS_BOOTLOADER_BUILD
 
+# BOOTLOADER_BUILD macro is the same, for source file changes
+CFLAGS += -D BOOTLOADER_BUILD=1
+
 # include the top-level "project" include directory, for sdkconfig.h
 CFLAGS += -I$(BUILD_DIR_BASE)/../include
 

components/bootloader/subproject/main/component.mk

@@ -15,6 +15,6 @@ ifndef CONFIG_SPI_FLASH_ROM_DRIVER_PATCH
 LINKER_SCRIPTS += $(IDF_PATH)/components/esp32/ld/esp32.rom.spiflash.ld
 endif
 
-COMPONENT_ADD_LDFLAGS := -L $(COMPONENT_PATH) -lmain $(addprefix -T ,$(LINKER_SCRIPTS))
+COMPONENT_ADD_LDFLAGS += -L $(COMPONENT_PATH) $(addprefix -T ,$(LINKER_SCRIPTS))
 
 COMPONENT_ADD_LINKER_DEPS := $(LINKER_SCRIPTS)

components/cxx/component.mk

@@ -1,3 +1,3 @@
 # Mark __cxa_guard_dummy as undefined so that implementation of static guards 
 # is taken from cxx_guards.o instead of libstdc++.a
-COMPONENT_ADD_LDFLAGS := -l$(COMPONENT_NAME) -u __cxa_guard_dummy
+COMPONENT_ADD_LDFLAGS += -u __cxa_guard_dummy

components/esp32/component.mk

@@ -28,8 +28,7 @@ endif
 #ld_include_panic_highint_hdl is added as an undefined symbol because otherwise the 
 #linker will ignore panic_highint_hdl.S as it has no other files depending on any
 #symbols in it.
-COMPONENT_ADD_LDFLAGS := -lesp32 \
+COMPONENT_ADD_LDFLAGS += $(COMPONENT_PATH)/libhal.a \
-                         $(COMPONENT_PATH)/libhal.a \
                          -L$(COMPONENT_PATH)/lib \
                          $(addprefix -l,$(LIBS)) \
                          -L $(COMPONENT_PATH)/ld \

components/esptool_py/component.mk

@@ -0,0 +1,5 @@
+# esptool_py component is special, because it doesn't contain any
+# IDF source files. It only adds steps via Makefile.projbuild &
+# Kconfig.projbuild
+COMPONENT_CONFIG_ONLY := 1
+

components/freertos/component.mk

@@ -2,7 +2,7 @@
 # Component Makefile
 #
 
-COMPONENT_ADD_LDFLAGS = -l$(COMPONENT_NAME) -Wl,--undefined=uxTopUsedPriority
+COMPONENT_ADD_LDFLAGS += -Wl,--undefined=uxTopUsedPriority
 COMPONENT_ADD_INCLUDEDIRS := include
 COMPONENT_PRIV_INCLUDEDIRS := include/freertos
 

components/newlib/component.mk

@@ -7,7 +7,7 @@ endif
 
 LIBM_PATH := $(COMPONENT_PATH)/lib/libm.a
 
-COMPONENT_ADD_LDFLAGS := $(LIBC_PATH) $(LIBM_PATH) -lnewlib
+COMPONENT_ADD_LDFLAGS += $(LIBC_PATH) $(LIBM_PATH)
 
 COMPONENT_ADD_LINKER_DEPS := $(LIBC_PATH) $(LIBM_PATH)
 

components/partition_table/Makefile.projbuild

@@ -14,6 +14,9 @@ GEN_ESP32PART := $(PYTHON) $(COMPONENT_PATH)/gen_esp32part.py -q
 # Has a matching value in bootloader_support esp_flash_partitions.h
 PARTITION_TABLE_OFFSET := 0x8000
 
+# if CONFIG_PARTITION_TABLE_FILENAME is unset, means we haven't re-generated config yet...
+ifneq ("$(CONFIG_PARTITION_TABLE_FILENAME)","")
+
 ifndef PARTITION_TABLE_CSV_PATH
 # Path to partition CSV file is relative to project path for custom
 # partition CSV files, but relative to component dir otherwise.$
@@ -21,7 +24,9 @@ PARTITION_TABLE_ROOT := $(call dequote,$(if $(CONFIG_PARTITION_TABLE_CUSTOM),$(P
 PARTITION_TABLE_CSV_PATH := $(call dequote,$(abspath $(PARTITION_TABLE_ROOT)/$(subst $(quote),,$(CONFIG_PARTITION_TABLE_FILENAME))))
 endif
 
-PARTITION_TABLE_BIN := $(BUILD_DIR_BASE)/$(notdir $(PARTITION_TABLE_CSV_PATH:.csv=.bin))
+PARTITION_TABLE_CSV_NAME := $(notdir $(PARTITION_TABLE_CSV_PATH))
+
+PARTITION_TABLE_BIN := $(BUILD_DIR_BASE)/$(PARTITION_TABLE_CSV_NAME:.csv=.bin)
 
 ifdef CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES
 PARTITION_TABLE_BIN_UNSIGNED := $(PARTITION_TABLE_BIN:.bin=-unsigned.bin)
@@ -58,3 +63,5 @@ partition_table-clean:
 	rm -f $(PARTITION_TABLE_BIN)
 
 clean: partition_table-clean
+
+endif

components/partition_table/component.mk

@@ -0,0 +1,5 @@
+# partition table component is special, because it doesn't contain any
+# IDF source files. It only adds steps via Makefile.projbuild &
+# Kconfig.projbuild
+COMPONENT_CONFIG_ONLY := 1
+

docs/api-guides/build-system.rst

@@ -84,7 +84,7 @@ This example "myProject" contains the following elements:
 
 - Optional "components" directory contains components that are part of the project. A project does not have to contain custom components of this kind, but it can be useful for structuring reusable code or including third party components that aren't part of ESP-IDF.
 
-- "main" directory is a special "pseudo-component" that contains source code for the project itself. "main" is a default name, the Makefile variable ``SRCDIRS`` defaults to this but can be set to look for pseudo-components in other directories.
+- "main" directory is a special "pseudo-component" that contains source code for the project itself. "main" is a default name, the Makefile variable ``COMPONENT_DIRS`` includes this component but you can modify this variable (or set ``EXTRA_COMPONENT_DIRS``) to look for components in other places.
 
 - "build" directory is where build output is created. After the make process is run, this directory will contain interim object files and libraries as well as final binary output files. This directory is usually not added to source control or distributed with the project source code.
 
@@ -120,17 +120,27 @@ These variables all have default values that can be overridden for custom behavi
 
 - ``PROJECT_PATH``: Top-level project directory. Defaults to the directory containing the Makefile. Many other project variables are based on this variable. The project path cannot contain spaces.
 - ``BUILD_DIR_BASE``: The build directory for all objects/libraries/binaries. Defaults to ``$(PROJECT_PATH)/build``.
-- ``COMPONENT_DIRS``: Directories to search for components. Defaults to `$(IDF_PATH)/components`, `$(PROJECT_PATH)/components` and ``EXTRA_COMPONENT_DIRS``. Override this variable if you don't want to search for components in the esp-idf & project ``components`` directories.
+- ``COMPONENT_DIRS``: Directories to search for components. Defaults to `$(IDF_PATH)/components`, `$(PROJECT_PATH)/components`, ``$(PROJECT_PATH)/main`` and ``EXTRA_COMPONENT_DIRS``. Override this variable if you don't want to search for components in these places.
-- ``EXTRA_COMPONENT_DIRS``: Optional list of additional directories to search for components. Components themselves are in sub-directories of these directories, this is a top-level directory containing the component directories.
+- ``EXTRA_COMPONENT_DIRS``: Optional list of additional directories to search for components.
 - ``COMPONENTS``: A list of component names to build into the project. Defaults to all components found in the COMPONENT_DIRS directories.
-- ``SRCDIRS``: Directories under the main project directory which contain project-specific "pseudo-components". Defaults to 'main'. The difference between specifying a directory here and specifying it under ``EXTRA_COMPONENT_DIRS`` is that a directory in ``SRCDIRS`` is a component itself (contains a file "component.mk"), whereas a directory in ``EXTRA_COMPONENT_DIRS`` contains component directories which contain a file "component.mk". See the `Example Project` for a concrete case of this.
+
+Any paths in these Makefile variables should be absolute paths. You can convert relative paths using ``$(PROJECT_PATH)/xxx``, ``$(IDF_PATH)/xxx``, or use the Make function ``$(abspath xxx)``.
+
+These variables should all be set before the line ``include $(IDF_PATH)/make/project.mk`` in the Makefile.
 
 Component Makefiles
 -------------------
 
 Each project contains one or more components, which can either be part of esp-idf or added from other component directories.
 
-A component is any sub-directory that contains a `component.mk` file [#f1]_.
+A component is any directory that contains a ``component.mk`` file.
+
+Searching for Components
+------------------------
+
+The list of directories in ``COMPONENT_DIRS`` is searched for the project's components. Directories in this list can either be components themselves (ie they contain a `component.mk` file), or they can be top-level directories whose subdirectories are components.
+
+Running the ``make list-components`` target dumps many of these variables and can help debug the discovery of component directories.
 
 Multiple components with the same name
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -238,10 +248,15 @@ The following variables can be set inside ``component.mk`` to control the build
   directory, of any files that are generated using custom make rules
   in the component.mk file and which need to be removed as part of
   ``make clean``. See `Source Code Generation` for an example.
-- ``COMPONENT_OWNBUILDTARGET`` & `COMPONENT_OWNCLEANTARGET`: These
+- ``COMPONENT_OWNBUILDTARGET`` & ``COMPONENT_OWNCLEANTARGET``: These
   targets allow you to fully override the default build behaviour for
   the component. See `Fully Overriding The Component Makefile` for
   more details.
+- ``COMPONENT_CONFIG_ONLY``: If set, this flag indicates that the component
+  produces no built output at all (ie ``COMPONENT_LIBRARY`` is not built),
+  and most other component variables are ignored. This flag is used for IDF
+  internal components which contain only ``KConfig.projbuild`` and/or
+  ``Makefile.projbuild`` files to configure the project, but no source files.
 - ``CFLAGS``: Flags passed to the C compiler. A default set of
   ``CFLAGS`` is defined based on project settings. Component-specific
   additions can be made via ``CFLAGS +=``. It is also possible
@@ -359,11 +374,15 @@ Take care when setting variables or targets in this file. As the values are incl
 KConfig.projbuild
 ^^^^^^^^^^^^^^^^^
 
-This is an equivalent to `Makefile.projbuild` for `component configuration` KConfig files. If you want to include
+This is an equivalent to ``Makefile.projbuild`` for `component configuration` KConfig files. If you want to include
 configuration options at the top-level of menuconfig, rather than inside the "Component Configuration" sub-menu, then these can be defined in the KConfig.projbuild file alongside the ``component.mk`` file.
 
 Take care when adding configuration values in this file, as they will be included across the entire project configuration. Where possible, it's generally better to create a KConfig file for `component configuration`.
 
+Configuration-Only Components
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Some special components which contain no source files, only ``Kconfig.projbuild`` and ``Makefile.projbuild``, may set the flag ``COMPONENT_CONFIG_ONLY`` in the component.mk file. If this flag is set, most other component variables are ignored and no build step is run for the component.
 
 Example Component Makefiles
 ---------------------------
@@ -542,12 +561,11 @@ target. The build target can do anything as long as it creates
 $(COMPONENT_LIBRARY) for the project make process to link into the app binary.
 
 (Actually, even this is not strictly necessary - if the COMPONENT_ADD_LDFLAGS variable
-is set then the component can instruct the linker to link other binaries instead.)
+is overridden then the component can instruct the linker to link other binaries instead.)
 
 
 .. _esp-idf-template: https://github.com/espressif/esp-idf-template
 .. _GNU Make Manual: https://www.gnu.org/software/make/manual/make.html
-.. [#f1] Actually, some components in esp-idf are "pure configuration" components that don't have a component.mk file, only a Makefile.projbuild and/or Kconfig.projbuild file. However, these components are unusual and most components have a component.mk file.
 
 
 Custom sdkconfig defaults
@@ -565,7 +583,7 @@ There're some scenarios that we want to flash the target board without IDF. For
 
     print_flash_cmd:
         echo $(ESPTOOL_WRITE_FLASH_OPTIONS) $(ESPTOOL_ALL_FLASH_ARGS) | sed -e 's:'$(PWD)/build/'::g'
-    
+
 the original ESPTOOL_ALL_FLASH_ARGS are absolute file name. Usually we want to save relative file name so we can move the bin folder to somewhere else. For this case we can use ``sed`` to convert to relative file name, like what we did in the example above.
 
 When running ``make print_flash_cmd``, it will print the flash arguments::
@@ -576,3 +594,11 @@ Then use flash arguments as the arguemnts for esptool write_flash arguments::
 
     python esptool.py --chip esp32 --port /dev/ttyUSB0 --baud 921600 --before default_reset --after hard_reset write_flash -z --flash_mode dio --flash_freq 40m --flash_size detect 0x1000 bootloader/bootloader.bin 0x10000 example_app.bin 0x8000 partition_table_unit_test_app.bin
 
+Building the Bootloader
+=======================
+
+The bootloader is built by default as part of "make all", or can be built standalone via "make bootloader-clean". There is also "make bootloader-list-components" to see the components included in the bootloader build.
+
+The component in IDF components/bootloader is special, as the second stage bootloader is a separate .ELF and .BIN file to the main project. However it shares its configuration and build directory with the main project.
+
+This is accomplished by adding a subproject under components/bootloader/subproject. This subproject has its own Makefile, but it expects to be called from the project's own Makefile via some glue in the components/bootloader/Makefile.projectbuild file. See these files for more details.

make/component_wrapper.mk

@@ -71,11 +71,12 @@ endef
 
 include $(COMPONENT_MAKEFILE)
 
-
 ################################################################################
 # 3) Set variables that depend on values that may changed by component.mk
 ################################################################################
 
+ifndef COMPONENT_CONFIG_ONLY  # Skip steps 3-5 if COMPONENT_CONFIG_ONLY is set
+
 # Object files which need to be linked into the library
 # By default we take all .c, .cpp & .S files in COMPONENT_SRCDIRS.
 ifndef COMPONENT_OBJS
@@ -112,6 +113,8 @@ COMPONENT_INCLUDES := $(OWN_INCLUDES) $(filter-out $(OWN_INCLUDES),$(COMPONENT_I
 # 4) Define a target to generate component_project_vars.mk Makefile which
 # contains common per-component settings which are included directly in the
 # top-level project make
+#
+# (Skipped if COMPONENT_CONFIG_ONLY is set.)
 ################################################################################
 
 # macro to generate variable-relative paths inside component_project_vars.mk, whenever possible
@@ -140,23 +143,23 @@ component_project_vars.mk::
 	$(details) "Building component project variables list $(abspath $@)"
 	@echo '# Automatically generated build file. Do not edit.' > $@
 	@echo 'COMPONENT_INCLUDES += $(call MakeVariablePath,$(addprefix $(COMPONENT_PATH)/,$(COMPONENT_ADD_INCLUDEDIRS)))' >> $@
-	@echo 'COMPONENT_LDFLAGS += $(call MakeVariablePath,$(COMPONENT_ADD_LDFLAGS))' >> $@
+	@echo 'COMPONENT_LDFLAGS += $(call MakeVariablePath,-L$(COMPONENT_BUILD_DIR) $(COMPONENT_ADD_LDFLAGS))' >> $@
 	@echo 'COMPONENT_LINKER_DEPS += $(call MakeVariablePath,$(call resolvepath,$(COMPONENT_ADD_LINKER_DEPS),$(COMPONENT_PATH)))' >> $@
 	@echo 'COMPONENT_SUBMODULES += $(call MakeVariablePath,$(addprefix $(COMPONENT_PATH)/,$(COMPONENT_SUBMODULES)))' >> $@
+	@echo 'COMPONENT_LIBRARIES += $(COMPONENT_NAME)' >> $@
 	@echo '$(COMPONENT_NAME)-build: $(addsuffix -build,$(COMPONENT_DEPENDS))' >> $@
 
-
 ################################################################################
-# 5) If COMPONENT_OWNBUILDTARGET / COMPONENT_OWNCLEANTARGET is not set by component.mk,
+# 5) Where COMPONENT_OWNBUILDTARGET / COMPONENT_OWNCLEANTARGET
-#	define default build, clean, etc. targets
+# is not set by component.mk, define default build, clean, etc. targets
+#
+# (Skipped if COMPONENT_CONFIG_ONLY is set.)
 ################################################################################
 
-# If COMPONENT_OWNBUILDTARGET is not set, define a phony build target and
+# Default build behaviour: define a phony build target and a COMPONENT_LIBRARY link target.
-# a COMPONENT_LIBRARY link target.
 ifndef COMPONENT_OWNBUILDTARGET
 .PHONY: build
 build: $(COMPONENT_LIBRARY)
-	@mkdir -p $(COMPONENT_SRCDIRS)
 
 # Build the archive. We remove the archive first, otherwise ar will get confused if we update
 # an archive when multiple filenames have the same name (src1/test.o and src2/test.o)
@@ -178,6 +181,18 @@ endif
 # Include all dependency files already generated
 -include $(COMPONENT_OBJS:.o=.d)
 
+# This is a fix for situation where the project or IDF dir moves, and instead
+# of rebuilding the target the build fails until make clean is run
+#
+# It adds an empty dependency rule for the (possibly non-existent) source file itself,
+# which prevents it not being found from failing the build
+#
+# $1 == Source File, $2 == .o file used for .d file name
+define AppendSourceToDependencies
+echo "$1:" >> $$(patsubst %.o,%.d,$2)
+endef
+
+
 # This pattern is generated for each COMPONENT_SRCDIR to compile the files in it.
 define GenerateCompileTargets
 # $(1) - directory containing source files, relative to $(COMPONENT_PATH) - one of $(COMPONENT_SRCDIRS)
@@ -185,14 +200,17 @@ define GenerateCompileTargets
 $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.c $(COMMON_MAKEFILES) $(COMPONENT_MAKEFILE) | $(COMPONENT_SRCDIRS)
 	$$(summary) CC $$@
 	$$(CC) $$(CFLAGS) $$(CPPFLAGS) $$(addprefix -I ,$$(COMPONENT_INCLUDES)) $$(addprefix -I ,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@
+	$(call AppendSourceToDependencies,$$<,$$@)
 
 $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.cpp $(COMMON_MAKEFILES) $(COMPONENT_MAKEFILE) | $(COMPONENT_SRCDIRS)
 	$$(summary) CXX $$@
 	$$(CXX) $$(CXXFLAGS) $$(CPPFLAGS) $$(addprefix -I,$$(COMPONENT_INCLUDES)) $$(addprefix -I,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@
+	$(call AppendSourceToDependencies,$$<,$$@)
 
 $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.S $(COMMON_MAKEFILES) $(COMPONENT_MAKEFILE) | $(COMPONENT_SRCDIRS)
 	$$(summary) AS $$@
 	$$(CC) $$(CPPFLAGS) $$(DEBUG_FLAGS) $$(addprefix -I ,$$(COMPONENT_INCLUDES)) $$(addprefix -I ,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@
+	$(call AppendSourceToDependencies,$$<,$$@)
 
 # CWD is build dir, create the build subdirectory if it doesn't exist
 #
@@ -250,3 +268,19 @@ $(foreach txtfile,$(COMPONENT_EMBED_TXTFILES), $(eval $(call GenerateEmbedTarget
 
 # generate targets to create binary embed directories
 $(foreach bindir,$(sort $(dir $(COMPONENT_EMBED_FILES))), $(eval $(call GenerateBuildDirTarget,$(bindir))))
+
+
+else   # COMPONENT_CONFIG_ONLY is set
+
+build:
+	$(details) "No build needed for $(COMPONENT_NAME) (COMPONENT_CONFIG_ONLY)"
+
+clean:
+	$(summary) RM component_project_vars.mk
+	rm -f component_project_vars.mk
+
+component_project_vars.mk::  # no need to add variables via component.mk
+	@echo '# COMPONENT_CONFIG_ONLY target sets no variables here' > $@
+
+endif  # COMPONENT_CONFIG_ONLY
+

make/project.mk

@@ -10,7 +10,7 @@
 # where this file is located.
 #
 
-.PHONY: build-components menuconfig defconfig all build clean all_binaries check-submodules size size-components size-files
+.PHONY: build-components menuconfig defconfig all build clean all_binaries check-submodules size size-components size-files list-components
 all: all_binaries
 # see below for recipe of 'all' target
 #
@@ -33,6 +33,7 @@ help:
 	@echo "make erase_flash - Erase entire flash contents"
 	@echo "make monitor - Run idf_monitor tool to monitor serial output from app"
 	@echo "make simple_monitor - Monitor serial output on terminal console"
+	@echo "make list-components - List all components in the project"
 	@echo ""
 	@echo "make app - Build just the app"
 	@echo "make app-flash - Flash just the app"
@@ -95,24 +96,35 @@ export COMMON_MAKEFILES
 
 # The directory where we put all objects/libraries/binaries. The project Makefile can
 # configure this if needed.
-BUILD_DIR_BASE ?= $(PROJECT_PATH)/build
+ifndef BUILD_DIR_BASE
+BUILD_DIR_BASE := $(PROJECT_PATH)/build
+endif
 export BUILD_DIR_BASE
 
-# Component directories. These directories are searched for components.
+# Component directories. These directories are searched for components (either the directory is a component,
-# The project Makefile can override these component dirs, or define extra component directories.
+# or the directory contains subdirectories which are components.)
-COMPONENT_DIRS ?= $(PROJECT_PATH)/components $(EXTRA_COMPONENT_DIRS) $(IDF_PATH)/components
+# The project Makefile can override these component dirs, or add extras via EXTRA_COMPONENT_DIRS
+ifndef COMPONENT_DIRS
+COMPONENT_DIRS := $(PROJECT_PATH)/components $(EXTRA_COMPONENT_DIRS) $(IDF_PATH)/components $(PROJECT_PATH)/main
+endif
 export COMPONENT_DIRS
 
-# Source directories of the project itself (a special, project-specific component.) Defaults to only "main".
+ifdef SRCDIRS
-SRCDIRS ?= main
+$(warning SRCDIRS variable is deprecated. These paths can be added to EXTRA_COMPONENT_DIRS or COMPONENT_DIRS instead.)
+COMPONENT_DIRS += $(abspath $(SRCDIRS))
+endif
 
-# The project Makefile can define a list of components, but if it does not do this we just take
+# The project Makefile can define a list of components, but if it does not do this we just take all available components
-# all available components in the component dirs.
+# in the component dirs. A component is COMPONENT_DIRS directory, or immediate subdirectory, 
+# which contains a component.mk file.
+#
+# Use the "make list-components" target to debug this step.
 ifndef COMPONENTS
 # Find all component names. The component names are the same as the
-# directories they're in, so /bla/components/mycomponent/ -> mycomponent. We then use
+# directories they're in, so /bla/components/mycomponent/component.mk -> mycomponent.
-# COMPONENT_DIRS to build COMPONENT_PATHS with the full path to each component.
+COMPONENTS := $(dir $(foreach cd,$(COMPONENT_DIRS),                           \
-COMPONENTS := $(foreach dir,$(COMPONENT_DIRS),$(wildcard $(dir)/*))
+					$(wildcard $(cd)/*/component.mk) $(wildcard $(cd)/component.mk) \
+				))
 COMPONENTS := $(sort $(foreach comp,$(COMPONENTS),$(lastword $(subst /, ,$(comp)))))
 endif
 export COMPONENTS
@@ -123,11 +135,7 @@ export COMPONENTS
 #
 # NOTE: These paths must be generated WITHOUT a trailing / so we
 # can use $(notdir x) to get the component name.
-COMPONENT_PATHS := $(foreach comp,$(COMPONENTS),$(firstword $(foreach dir,$(COMPONENT_DIRS),$(wildcard $(dir)/$(comp)))))
+COMPONENT_PATHS := $(foreach comp,$(COMPONENTS),$(firstword $(foreach cd,$(COMPONENT_DIRS),$(wildcard $(dir $(cd))$(comp) $(cd)/$(comp)))))
-COMPONENT_PATHS += $(abspath $(SRCDIRS))
-
-# A component is buildable if it has a component.mk makefile in it
-COMPONENT_PATHS_BUILDABLE := $(foreach cp,$(COMPONENT_PATHS),$(if $(wildcard $(cp)/component.mk),$(cp)))
 
 # If TESTS_ALL set to 1, set TEST_COMPONENTS_LIST to all components
 ifeq ($(TESTS_ALL),1)
@@ -159,7 +167,7 @@ COMPONENT_SUBMODULES :=
 # dependencies.
 #
 # See the component_project_vars.mk target in component_wrapper.mk
-COMPONENT_PROJECT_VARS := $(addsuffix /component_project_vars.mk,$(notdir $(COMPONENT_PATHS_BUILDABLE) ) $(TEST_COMPONENT_NAMES))
+COMPONENT_PROJECT_VARS := $(addsuffix /component_project_vars.mk,$(notdir $(COMPONENT_PATHS) ) $(TEST_COMPONENT_NAMES))
 COMPONENT_PROJECT_VARS := $(addprefix $(BUILD_DIR_BASE)/,$(COMPONENT_PROJECT_VARS))
 # this line is -include instead of include to prevent a spurious error message on make 3.81
 -include $(COMPONENT_PROJECT_VARS)
@@ -190,9 +198,7 @@ endif
 IDF_VER := $(shell cd ${IDF_PATH} && git describe --always --tags --dirty)
 
 # Set default LDFLAGS
-SRCDIRS_COMPONENT_NAMES := $(sort $(foreach comp,$(SRCDIRS),$(lastword $(subst /, ,$(comp)))))
 LDFLAGS ?= -nostdlib \
-	$(addprefix -L$(BUILD_DIR_BASE)/,$(COMPONENTS) $(TEST_COMPONENT_NAMES) $(SRCDIRS_COMPONENT_NAMES) ) \
 	-u call_user_start_cpu0	\
 	$(EXTRA_LDFLAGS) \
 	-Wl,--gc-sections	\
@@ -299,11 +305,13 @@ APP_BIN:=$(APP_ELF:.elf=.bin)
 # Include any Makefile.projbuild file letting components add
 # configuration at the project level
 define includeProjBuildMakefile
-$(if $(V),$(if $(wildcard $(1)/Makefile.projbuild),$(info including $(1)/Makefile.projbuild...)))
+$(if $(V),$(info including $(1)/Makefile.projbuild...))
 COMPONENT_PATH := $(1)
--include $(1)/Makefile.projbuild
+include $(1)/Makefile.projbuild
 endef
-$(foreach componentpath,$(COMPONENT_PATHS),$(eval $(call includeProjBuildMakefile,$(componentpath))))
+$(foreach componentpath,$(COMPONENT_PATHS), \
+	$(if $(wildcard $(componentpath)/Makefile.projbuild), \
+		$(eval $(call includeProjBuildMakefile,$(componentpath)))))
 
 # once we know component paths, we can include the config generation targets
 #
@@ -312,22 +320,15 @@ ifndef IS_BOOTLOADER_BUILD
 include $(IDF_PATH)/make/project_config.mk
 endif
 
-# A "component" library is any library in the LDFLAGS where
-# the name of the library is also a name of the component
-APP_LIBRARIES = $(patsubst -l%,%,$(filter -l%,$(LDFLAGS)))
-COMPONENT_LIBRARIES = $(filter $(notdir $(COMPONENT_PATHS_BUILDABLE)) $(TEST_COMPONENT_NAMES),$(APP_LIBRARIES))
-
 # ELF depends on the library archive files for COMPONENT_LIBRARIES
 # the rules to build these are emitted as part of GenerateComponentTarget below
 #
 # also depends on additional dependencies (linker scripts & binary libraries)
 # stored in COMPONENT_LINKER_DEPS, built via component.mk files' COMPONENT_ADD_LINKER_DEPS variable
-$(APP_ELF): $(foreach libcomp,$(COMPONENT_LIBRARIES),$(BUILD_DIR_BASE)/$(libcomp)/lib$(libcomp).a) $(COMPONENT_LINKER_DEPS)
+$(APP_ELF): $(foreach libcomp,$(COMPONENT_LIBRARIES),$(BUILD_DIR_BASE)/$(libcomp)/lib$(libcomp).a) $(COMPONENT_LINKER_DEPS) $(COMPONENT_PROJECT_VARS)
 	$(summary) LD $(notdir $@)
 	$(CC) $(LDFLAGS) -o $@ -Wl,-Map=$(APP_MAP)
 
-# Generation of $(APP_BIN) from $(APP_ELF) is added by the esptool
-# component's Makefile.projbuild
 app: $(APP_BIN)
 ifeq ("$(CONFIG_SECURE_BOOT_ENABLED)$(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)","y") # secure boot enabled, but remote sign app image
 	@echo "App built but not signed. Signing step via espsecure.py:"
@@ -358,12 +359,12 @@ endef
 # $(2) - name of component
 #
 define GenerateComponentTargets
-.PHONY: $(2)-build $(2)-clean
+.PHONY: component-$(2)-build component-$(2)-clean
 
-$(2)-build: check-submodules
+component-$(2)-build: check-submodules
 	$(call ComponentMake,$(1),$(2)) build
 
-$(2)-clean:
+component-$(2)-clean:
 	$(call ComponentMake,$(1),$(2)) clean
 
 $(BUILD_DIR_BASE)/$(2):
@@ -373,7 +374,7 @@ $(BUILD_DIR_BASE)/$(2):
 # (this target exists for all components even ones which don't build libraries, but it's
 # only invoked for the targets whose libraries appear in COMPONENT_LIBRARIES and hence the
 # APP_ELF dependencies.)
-$(BUILD_DIR_BASE)/$(2)/lib$(2).a: $(2)-build
+$(BUILD_DIR_BASE)/$(2)/lib$(2).a: component-$(2)-build
 	$(details) "Target '$$^' responsible for '$$@'" # echo which build target built this file
 
 # add a target to generate the component_project_vars.mk files that
@@ -387,10 +388,10 @@ $(BUILD_DIR_BASE)/$(2)/component_project_vars.mk: $(1)/component.mk $(COMMON_MAK
 	$(call ComponentMake,$(1),$(2)) component_project_vars.mk
 endef
 
-$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentTargets,$(component),$(notdir $(component)))))
+$(foreach component,$(COMPONENT_PATHS),$(eval $(call GenerateComponentTargets,$(component),$(notdir $(component)))))
 $(foreach component,$(TEST_COMPONENT_PATHS),$(eval $(call GenerateComponentTargets,$(component),$(lastword $(subst /, ,$(dir $(component))))_test)))
 
-app-clean: $(addsuffix -clean,$(notdir $(COMPONENT_PATHS_BUILDABLE)))
+app-clean: $(addprefix component-,$(addsuffix -clean,$(notdir $(COMPONENT_PATHS))))
 	$(summary) RM $(APP_ELF)
 	rm -f $(APP_ELF) $(APP_BIN) $(APP_MAP)
 
@@ -439,6 +440,18 @@ endef
 $(foreach submodule,$(subst $(IDF_PATH)/,,$(filter $(IDF_PATH)/%,$(COMPONENT_SUBMODULES))),$(eval $(call GenerateSubmoduleCheckTarget,$(submodule))))
 
 
+# PHONY target to list components in the build and their paths
+list-components:
+	$(info $(call dequote,$(SEPARATOR)))
+	$(info COMPONENT_DIRS (components searched for here))
+	$(foreach cd,$(COMPONENT_DIRS),$(info $(cd)))
+	$(info $(call dequote,$(SEPARATOR)))
+	$(info COMPONENTS (list of component names))
+	$(info $(COMPONENTS))
+	$(info $(call dequote,$(SEPARATOR)))
+	$(info COMPONENT_PATHS (paths to all components):)
+	$(foreach cp,$(COMPONENT_PATHS),$(info $(cp)))
+
 # Check toolchain version using the output of xtensa-esp32-elf-gcc --version command.
 # The output normally looks as follows
 #     xtensa-esp32-elf-gcc (crosstool-NG crosstool-ng-1.22.0-59-ga194053) 4.8.5

tools/ci/test_build_system.sh

@@ -71,7 +71,7 @@ function run_tests()
 
     print_status "Bootloader source file rebuilds bootloader"
     take_build_snapshot
-    touch ${IDF_PATH}/components/bootloader/src/main/bootloader_start.c
+    touch ${IDF_PATH}/components/bootloader/subproject/main/bootloader_start.c
     make bootloader || failure "Failed to partial build bootloader"
     assert_rebuilt ${BOOTLOADER_BINS} bootloader/main/bootloader_start.o
     assert_not_rebuilt ${APP_BINS} partitions_singleapp.bin