build system: Replace get_variable target w/ component_project_vars.mk generated makefiles

Reduces number of make invocations, allows variables exported in project
to be seen in all component make processes, not just the main ones.

Also makes a no-op build about 3x faster than it was.

components/bootloader/src/main/component.mk

@@ -7,6 +7,6 @@
 # please read the esp-idf build system document if you need to do this.
 #
 
-COMPONENT_ADD_LDFLAGS := -L $(abspath .) -lmain -T esp32.bootloader.ld -T $(IDF_PATH)/components/esp32/ld/esp32.rom.ld 
+COMPONENT_ADD_LDFLAGS := -L $(COMPONENT_PATH) -lmain -T esp32.bootloader.ld -T $(IDF_PATH)/components/esp32/ld/esp32.rom.ld
 
 include $(IDF_PATH)/make/component_common.mk

components/bt/component.mk

@@ -2,15 +2,13 @@
 # Component Makefile
 #
 
-#COMPONENT_ADD_INCLUDEDIRS := 
-
 COMPONENT_ADD_INCLUDEDIRS := include
 
 CFLAGS += -Wno-error=unused-label -Wno-error=return-type -Wno-error=missing-braces -Wno-error=pointer-sign -Wno-error=parentheses
 
 LIBS := btdm_app
 
-COMPONENT_ADD_LDFLAGS := -lbt -L$(abspath lib) \
+COMPONENT_ADD_LDFLAGS := -lbt -L $(COMPONENT_PATH)/lib \
                            $(addprefix -l,$(LIBS)) \
                           $(LINKER_SCRIPTS)
 

components/esp32/component.mk

@@ -15,10 +15,10 @@ LIBS := core net80211 phy rtc pp wpa smartconfig coexist
 LINKER_SCRIPTS += -T esp32_out.ld -T esp32.common.ld -T esp32.rom.ld -T esp32.peripherals.ld
 
 COMPONENT_ADD_LDFLAGS := -lesp32 \
-                           $(abspath libhal.a) \
+                           $(COMPONENT_PATH)/libhal.a \
-                           -L$(abspath lib) \
+                           -L$(COMPONENT_PATH)/lib \
                            $(addprefix -l,$(LIBS)) \
-                          -L $(abspath ld) \
+                          -L $(COMPONENT_PATH)/ld \
                           $(LINKER_SCRIPTS)
 
 include $(IDF_PATH)/make/component_common.mk

components/newlib/component.mk

@@ -1,4 +1,4 @@
-COMPONENT_ADD_LDFLAGS := $(abspath lib/libc.a) $(abspath lib/libm.a) -lnewlib
+COMPONENT_ADD_LDFLAGS := $(COMPONENT_PATH)/lib/libc.a $(COMPONENT_PATH)/lib/libm.a -lnewlib
 
 COMPONENT_ADD_INCLUDEDIRS := include platform_include
 

make/common.mk

@@ -2,12 +2,6 @@
 # and component makefiles
 #
 
-# Include project config file, if it exists.
-#
-# (Note that we only rebuild auto.conf automatically for some targets,
-# see project_config.mk for details.)
--include $(BUILD_DIR_BASE)/include/config/auto.conf
-
 #Handling of V=1/VERBOSE=1 flag
 #
 # if V=1, $(summary) does nothing and $(details) will echo extra details

make/component_common.mk

@@ -58,10 +58,19 @@ COMPONENT_ADD_LDFLAGS ?= -l$(COMPONENT_NAME)
 OWN_INCLUDES:=$(abspath $(addprefix $(COMPONENT_PATH)/,$(COMPONENT_ADD_INCLUDEDIRS) $(COMPONENT_PRIV_INCLUDEDIRS)))
 COMPONENT_INCLUDES := $(OWN_INCLUDES) $(filter-out $(OWN_INCLUDES),$(COMPONENT_INCLUDES))
 
-#This target is used to collect variable values from inside project.mk
+# This target is used to take component.mk variables COMPONENT_ADD_INCLUDEDIRS,
-# see project.mk GetVariable macro for details.
+# COMPONENT_ADD_LDFLAGS and COMPONENT_DEPENDS and inject them into the project
-get_variable:
+# makefile level.
-	@echo "$(GET_VARIABLE)=$(call $(GET_VARIABLE)) "
+#
+# The target here has no dependencies, as the parent target in
+# project.mk evaluates dependencies before calling down to here. See
+# GenerateProjectVarsTarget in project.mk.
+component_project_vars.mk::
+	$(details) "Rebuilding component project variables list $(abspath $@)"
+	@echo "# Automatically generated build file. Do not edit." > $@
+	@echo "COMPONENT_INCLUDES += $(addprefix $(COMPONENT_PATH)/,$(COMPONENT_ADD_INCLUDEDIRS))" >> $@
+	@echo "COMPONENT_LDFLAGS += $(COMPONENT_ADD_LDFLAGS)" >> $@
+	@echo "$(COMPONENT_NAME)-build: $(addsuffix -build,$(COMPONENT_DEPENDS))" >> $@
 
 #Targets for build/clean. Use builtin recipe if component Makefile
 #hasn't defined its own.
@@ -77,10 +86,12 @@ $(COMPONENT_LIBRARY): $(COMPONENT_OBJS)
 	$(Q) $(AR) cru $@ $(COMPONENT_OBJS)
 endif
 
+CLEAN_FILES = $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EXTRA_CLEAN) component_project_vars.mk
+
 ifeq ("$(COMPONENT_OWNCLEANTARGET)", "")
 clean:
-	$(summary) RM $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EXTRA_CLEAN)
+	$(summary) RM $(CLEAN_FILES)
-	$(Q) rm -f $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EXTRA_CLEAN)
+	$(Q) rm -f $(CLEAN_FILES)
 endif
 
 #Include all dependency files already generated

make/project.mk

@@ -48,11 +48,22 @@ PROJECT_PATH := $(abspath $(dir $(firstword $(MAKEFILE_LIST))))
 export PROJECT_PATH
 endif
 
+COMMON_MAKEFILES := $(abspath $(IDF_PATH)/make/project.mk $(IDF_PATH)/make/common.mk $(IDF_PATH)/make/component_common.mk)
+export COMMON_MAKEFILES
+
 #The directory where we put all objects/libraries/binaries. The project Makefile can
 #configure this if needed.
 BUILD_DIR_BASE ?= $(PROJECT_PATH)/build
 export BUILD_DIR_BASE
 
+# Include project config file, if it exists.
+#
+# (Note that we only rebuild auto.conf automatically for some targets,
+# see project_config.mk for details.)
+SDKCONFIG_MAKEFILE := $(BUILD_DIR_BASE)/include/config/auto.conf
+-include $(SDKCONFIG_MAKEFILE)
+export $(filter CONFIG_%,$(.VARIABLES))
+
 #Component directories. These directories are searched for components.
 #The project Makefile can override these component dirs, or define extra component directories.
 COMPONENT_DIRS ?= $(PROJECT_PATH)/components $(EXTRA_COMPONENT_DIRS) $(IDF_PATH)/components
@@ -87,48 +98,33 @@ COMPONENT_PATHS_BUILDABLE := $(foreach cp,$(COMPONENT_PATHS),$(if $(wildcard $(c
 # LDFLAGS args (COMPONENT_LDFLAGS) supplied by each component.
 COMPONENT_INCLUDES :=
 COMPONENT_LDFLAGS :=
-#
-# Also add any inter-component dependencies for each component.
 
-# Extract a variable from a child make process
+# include paths for generated "component project variables" targets with
+# COMPONENT_INCLUDES, COMPONENT_LDFLAGS & dependency targets
 #
-# $(1) - path to directory to invoke make in
+# See component_project_vars.mk target in component_common.mk
-# $(2) - name of variable to print via the get_variable target (passed in GET_VARIABLE)
+COMPONENT_PROJECT_VARS := $(addsuffix /component_project_vars.mk,$(notdir $(COMPONENT_PATHS_BUILDABLE)))
+COMPONENT_PROJECT_VARS := $(addprefix $(BUILD_DIR_BASE)/,$(COMPONENT_PROJECT_VARS))
+include $(COMPONENT_PROJECT_VARS)
+
+
+# Generate a target to rebuild component_project_vars.mk for a component
+# $(1) - component directory
+# $(2) - component name only
 #
-# needs 'sed' processing of stdout because make sometimes echoes other stuff on stdout,
+# Rebuilds if component.mk, makefiles or sdkconfig changes.
-# even if asked not to.
+define GenerateProjectVarsTarget
-#
+$(BUILD_DIR_BASE)/$(2)/component_project_vars.mk: $(1)/component.mk $(COMMON_MAKEFILES) $(if $(MAKE_RESTARTS),,$(SDKCONFIG_MAKEFILE)) $(BUILD_DIR_BASE)/$(2)
-# Debugging this? Replace $(shell with $(error and you'll see the full command as-run.
+	$(Q) +$(MAKE) -C $(BUILD_DIR_BASE)/$(2) -f $(1)/component.mk component_project_vars.mk COMPONENT_PATH=$(1)
-define GetVariable
-$(shell "$(MAKE)" -s --no-print-directory -C $(1) -f component.mk get_variable PROJECT_PATH=$(PROJECT_PATH) GET_VARIABLE=$(2) | sed -En "s/^$(2)=(.+)/\1/p" )
 endef
+$(foreach comp,$(COMPONENT_PATHS_BUILDABLE), $(eval $(call GenerateProjectVarsTarget,$(comp),$(notdir $(comp)))))
 
-COMPONENT_INCLUDES := $(abspath $(foreach comp,$(COMPONENT_PATHS_BUILDABLE),$(addprefix $(comp)/, \
+#Also add project include path, for top-level includes
-	$(call GetVariable,$(comp),COMPONENT_ADD_INCLUDEDIRS))))
-
-#Also add project include path, for sdk includes
 COMPONENT_INCLUDES += $(abspath $(BUILD_DIR_BASE)/include/)
+
 export COMPONENT_INCLUDES
-
-#COMPONENT_LDFLAGS has a list of all flags that are needed to link the components together. It's collected
-#in the same way as COMPONENT_INCLUDES is.
-COMPONENT_LDFLAGS := $(foreach comp,$(COMPONENT_PATHS_BUILDABLE), \
-	$(call GetVariable,$(comp),COMPONENT_ADD_LDFLAGS))
 export COMPONENT_LDFLAGS
 
-# Generate component dependency targets from dependencies lists
-# each component gains a target of its own <name>-build with dependencies
-# of the names of any other components (-build) that need building first
-#
-# the actual targets (that invoke submakes) are generated below by
-# GenerateComponentTarget macro.
-define GenerateComponentDependencies
-# $(1) = component path
-.PHONY: $$(notdir $(1))
-$$(notdir $(1))-build: $(addsuffix -build,$(call GetVariable,$(1),COMPONENT_DEPENDS))
-endef
-$(foreach comp,$(COMPONENT_PATHS_BUILDABLE), $(eval $(call GenerateComponentDependencies,$(comp))))
-
 #Make sure submakes can also use this.
 export PROJECT_PATH
 
@@ -242,7 +238,7 @@ COMPONENT_PATH := $(1)
 endef
 $(foreach componentpath,$(COMPONENT_PATHS),$(eval $(call includeProjBuildMakefile,$(componentpath))))
 
-# once we know component paths, we can include the config
+# once we know component paths, we can include the config generation targets
 include $(IDF_PATH)/make/project_config.mk
 
 # A "component" library is any library in the LDFLAGS where
@@ -269,29 +265,31 @@ $(BUILD_DIR_BASE):
 
 define GenerateComponentPhonyTarget
 # $(1) - path to component dir
-# $(2) - target to generate (build, clean)
+# $(2) - name of component
-.PHONY: $(notdir $(1))-$(2)
+# $(3) - target to generate (build, clean)
-$(notdir $(1))-$(2): | $(BUILD_DIR_BASE)/$(notdir $(1))
+.PHONY: $(2)-$(3)
-	$(Q) +$(MAKE) -C $(BUILD_DIR_BASE)/$(notdir $(1)) -f $(1)/component.mk COMPONENT_BUILD_DIR=$(BUILD_DIR_BASE)/$(notdir $(1)) $(2)
+$(2)-$(3): | $(BUILD_DIR_BASE)/$(2)
+	$(Q) +$(MAKE) -C $(BUILD_DIR_BASE)/$(2) -f $(1)/component.mk COMPONENT_PATH=$(1) COMPONENT_BUILD_DIR=$(BUILD_DIR_BASE)/$(2) $(3)
 endef
 
 define GenerateComponentTargets
 # $(1) - path to component dir
-$(BUILD_DIR_BASE)/$(notdir $(1)):
+# $(2) - name of component
-	@mkdir -p $(BUILD_DIR_BASE)/$(notdir $(1))
+$(BUILD_DIR_BASE)/$(2):
+	@mkdir -p $(BUILD_DIR_BASE)/$(2)
 
 # tell make it can build any component's library by invoking the recursive -build target
 # (this target exists for all components even ones which don't build libraries, but it's
 # only invoked for the targets whose libraries appear in COMPONENT_LIBRARIES and hence the
 # APP_ELF dependencies.)
-$(BUILD_DIR_BASE)/$(notdir $(1))/lib$(notdir $(1)).a: $(notdir $(1))-build
+$(BUILD_DIR_BASE)/$(2)/lib$(2).a: $(2)-build
 	$(details) "Target '$$^' responsible for '$$@'" # echo which build target built this file
 endef
 
-$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentTargets,$(component))))
+$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentTargets,$(component),$(notdir $(component)))))
 
-$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentPhonyTarget,$(component),build)))
+$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentPhonyTarget,$(component),$(notdir $(component)),build)))
-$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentPhonyTarget,$(component),clean)))
+$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentPhonyTarget,$(component),$(notdir $(component)),clean)))
 
 app-clean: $(addsuffix -clean,$(notdir $(COMPONENT_PATHS_BUILDABLE)))
 	$(summary) RM $(APP_ELF)

make/project_config.mk

@@ -37,7 +37,7 @@ defconfig: $(KCONFIG_TOOL_DIR)/mconf $(IDF_PATH)/Kconfig $(BUILD_DIR_BASE)
 
 # Work out of whether we have to build the Kconfig makefile
 # (auto.conf), or if we're in a situation where we don't need it
-NON_CONFIG_TARGETS := clean %-clean get_variable help menuconfig defconfig
+NON_CONFIG_TARGETS := clean %-clean help menuconfig defconfig
 AUTO_CONF_REGEN_TARGET := $(BUILD_DIR_BASE)/include/config/auto.conf
 
 # disable AUTO_CONF_REGEN_TARGET if all targets are non-config targets