build system: Refactor SubmoduleCheck to work project-wide

Required at project level because some components use header files in
other components' submodules, and one component with a
submodule (esptool.py) doesn't have or need a component.mk.

components/bt/component.mk

@@ -15,5 +15,4 @@ COMPONENT_ADD_LDFLAGS := -lbt -L $(COMPONENT_PATH)/lib \
 ALL_LIB_FILES := $(patsubst %,$(COMPONENT_PATH)/lib/lib%.a,$(LIBS))
 $(COMPONENT_LIBRARY): $(ALL_LIB_FILES)
 
-# automatically trigger a git submodule update if BT library is missing
+COMPONENT_SUBMODULES += lib
-$(eval $(call SubmoduleCheck,$(ALL_LIB_FILES),$(COMPONENT_PATH)/lib))

components/esp32/component.mk

@@ -17,9 +17,7 @@ COMPONENT_ADD_LDFLAGS := -lesp32 \
 
 ALL_LIB_FILES := $(patsubst %,$(COMPONENT_PATH)/lib/lib%.a,$(LIBS))
 
-# automatically trigger a git submodule update
+COMPONENT_SUBMODULES += lib
-# if any libraries are missing
-$(eval $(call SubmoduleCheck,$(ALL_LIB_FILES),$(COMPONENT_PATH)/lib))
 
 # this is a hack to make sure the app is re-linked if the binary
 # libraries change or are updated. If they change, the main esp32

components/esptool_py/Makefile.projbuild

@@ -54,4 +54,6 @@ app-flash: $(APP_BIN) $(ESPTOOLPY_SRC)
 	@echo "Flashing app to serial port $(ESPPORT), offset $(CONFIG_APP_OFFSET)..."
 	$(ESPTOOLPY_WRITE_FLASH) $(CONFIG_APP_OFFSET) $(APP_BIN)
 
-$(eval $(call SubmoduleCheck,$(ESPTOOLPY_SRC),$(COMPONENT_PATH)/esptool))
+# Submodules normally added in component.mk, but can be added
+# at the project level as long as qualified path
+COMPONENT_SUBMODULES += $(COMPONENT_PATH)/esptool

docs/build_system.rst

@@ -186,6 +186,14 @@ The following variables can be set inside ``component.mk`` to control build sett
   generates an include file which you then want to include in another
   component. Most components do not need to set this variable.
 
+The following variable only works for components that are part of esp-idf itself:
+
+- ``COMPONENT_SUBMODULES``: Optional list of git submodule paths
+  (relative to COMPONENT_PATH) used by the component. These will be
+  checked (and initialised if necessary) by the build process. This
+  variable is ignored if the component is outside the IDF_PATH
+  directory.
+
 
 Optional Component-Specific Variables
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

make/common.mk

@@ -26,31 +26,6 @@ details := @true
 MAKEFLAGS += --silent
 endif
 
-# Pseudo-target to check a git submodule has been properly initialised
-#
-# $(eval $(call SubmoduleCheck,FILENAMES,SUBMODULE_PATH)) to create a target that
-# automatically runs 'git submodule update --init SUBMODULE_PATH' if any of
-# the files in FILENAMES are missing, and fails if this is not possible.
-#
-# Will also print a WARNING if the submodule at SUBMODULE_PATH appears
-# to require an update.
-define SubmoduleCheck
-$(1):
-	@echo "WARNING: Missing submodule $(2) for $$@..."
-	[ -d ${IDF_PATH}/.git ] || ( echo "ERROR: esp-idf must be cloned from git to work."; exit 1)
-	[ -x $(which git) ] || ( echo "ERROR: Need to run 'git submodule --init' in esp-idf root directory."; exit 1)
-	@echo "Attempting 'git submodule update --init' in esp-idf root directory..."
-	cd ${IDF_PATH} && git submodule update --init $(2)
-
-# Parse 'git submodule status' output for out-of-date submodule.
-# Status output prefixes status line with '+' if the submodule commit doesn't match
-ifneq ("$(shell cd ${IDF_PATH} && git submodule status $(2) | grep '^+')","")
-$$(info WARNING: git submodule $2 may be out of date. Run 'git submodule update' to update.)
-endif
-endef
-
-
-
 # General make utilities
 
 # convenience variable for printing an 80 asterisk wide separator line

make/component_wrapper.mk

@@ -103,8 +103,8 @@ endef
 
 # component_project_vars.mk target for the component. This is used to
 # take component.mk variables COMPONENT_ADD_INCLUDEDIRS,
-# COMPONENT_ADD_LDFLAGS and COMPONENT_DEPENDS and inject those into
+# COMPONENT_ADD_LDFLAGS, COMPONENT_DEPENDS and COMPONENT_SUBMODULES
-# the project make pass.
+# and inject those into the project make pass.
 #
 # The target here has no dependencies, as the parent target in
 # project.mk evaluates dependencies before calling down to here. See
@@ -119,6 +119,7 @@ component_project_vars.mk::
 	@echo '# Automatically generated build file. Do not edit.' > $@
 	@echo 'COMPONENT_INCLUDES += $(call MakeVariablePath,$(addprefix $(COMPONENT_PATH)/,$(COMPONENT_ADD_INCLUDEDIRS)))' >> $@
 	@echo 'COMPONENT_LDFLAGS += $(call MakeVariablePath,$(COMPONENT_ADD_LDFLAGS))' >> $@
+	@echo 'COMPONENT_SUBMODULES += $(call MakeVariablePath,$(addprefix $(COMPONENT_PATH)/,$(COMPONENT_SUBMODULES)))' >> $@
 	@echo '$(COMPONENT_NAME)-build: $(addsuffix -build,$(COMPONENT_DEPENDS))' >> $@
 
 

make/project.mk

@@ -10,7 +10,7 @@
 # where this file is located.
 #
 
-.PHONY: build-components menuconfig defconfig all build clean all_binaries
+.PHONY: build-components menuconfig defconfig all build clean all_binaries check-submodules
 all: all_binaries
 # see below for recipe of 'all' target
 #
@@ -94,13 +94,16 @@ COMPONENT_PATHS += $(abspath $(SRCDIRS))
 # A component is buildable if it has a component.mk makefile in it
 COMPONENT_PATHS_BUILDABLE := $(foreach cp,$(COMPONENT_PATHS),$(if $(wildcard $(cp)/component.mk),$(cp)))
 
-# Initialise a project-wide list of include dirs (COMPONENT_INCLUDES),
+# Initialise project-wide variables which can be added to by
-# and LDFLAGS args (COMPONENT_LDFLAGS) supplied by each component.
+# each component.
 #
 # These variables are built up via the component_project_vars.mk
 # generated makefiles (one per component).
+#
+# See docs/build-system.rst for more details.
 COMPONENT_INCLUDES :=
 COMPONENT_LDFLAGS :=
+COMPONENT_SUBMODULES :=
 
 # COMPONENT_PROJECT_VARS is the list of component_project_vars.mk generated makefiles
 # for each component.
@@ -289,7 +292,7 @@ endef
 define GenerateComponentTargets
 .PHONY: $(2)-build $(2)-clean
 
-$(2)-build:
+$(2)-build: check-submodules
 	$(call ComponentMake,$(1),$(2)) build
 
 $(2)-clean:
@@ -332,4 +335,30 @@ app-clean: $(addsuffix -clean,$(notdir $(COMPONENT_PATHS_BUILDABLE)))
 config-clean: app-clean
 clean: config-clean
 
+# phony target to check if any git submodule listed in COMPONENT_SUBMODULES are missing
+# or out of date, and exit if so. Components can add paths to this variable.
+#
+# This only works for components inside IDF_PATH
+check-submodules:
 
+# Generate a target to check this submodule
+# $(1) - submodule directory, relative to IDF_PATH
+define GenerateSubmoduleCheckTarget
+check-submodules: $(IDF_PATH)/$(1)/.git
+$(IDF_PATH)/$(1)/.git:
+	@echo "WARNING: Missing submodule $(1)..."
+	[ -d ${IDF_PATH}/.git ] || ( echo "ERROR: esp-idf must be cloned from git to work."; exit 1)
+	[ -x $(which git) ] || ( echo "ERROR: Need to run 'git submodule init $(1)' in esp-idf root directory."; exit 1)
+	@echo "Attempting 'git submodule update --init $(1)' in esp-idf root directory..."
+	cd ${IDF_PATH} && git submodule update --init $(1)
+
+# Parse 'git submodule status' output for out-of-date submodule.
+# Status output prefixes status line with '+' if the submodule commit doesn't match
+ifneq ("$(shell cd ${IDF_PATH} && git submodule status $(1) | grep '^+')","")
+$$(info WARNING: git submodule $(1) may be out of date. Run 'git submodule update' to update.)
+endif
+endef
+
+# filter/subst in expression ensures all submodule paths begin with $(IDF_PATH), and then strips that prefix
+# so the argument is suitable for use with 'git submodule' commands
+$(foreach submodule,$(subst $(IDF_PATH)/,,$(filter $(IDF_PATH)/%,$(COMPONENT_SUBMODULES))),$(eval $(call GenerateSubmoduleCheckTarget,$(submodule))))