Merge branch 'bugfix/secure_boot_fixes' into 'master'

Secure boot related fixes

Fix some issues (mostly build system) from the secure boot implementation

Also refactor the way submodule checks are applied to make them more reliable.

See merge request !207

components/bt/component.mk

@@ -15,5 +15,4 @@ COMPONENT_ADD_LDFLAGS := -lbt -L $(COMPONENT_PATH)/lib \
 ALL_LIB_FILES := $(patsubst %,$(COMPONENT_PATH)/lib/lib%.a,$(LIBS))
 $(COMPONENT_LIBRARY): $(ALL_LIB_FILES)
 
-# automatically trigger a git submodule update if BT library is missing
+COMPONENT_SUBMODULES += lib
-$(eval $(call SubmoduleCheck,$(ALL_LIB_FILES),$(COMPONENT_PATH)/lib))

components/esp32/component.mk

@@ -17,9 +17,7 @@ COMPONENT_ADD_LDFLAGS := -lesp32 \
 
 ALL_LIB_FILES := $(patsubst %,$(COMPONENT_PATH)/lib/lib%.a,$(LIBS))
 
-# automatically trigger a git submodule update
+COMPONENT_SUBMODULES += lib
-# if any libraries are missing
-$(eval $(call SubmoduleCheck,$(ALL_LIB_FILES),$(COMPONENT_PATH)/lib))
 
 # this is a hack to make sure the app is re-linked if the binary
 # libraries change or are updated. If they change, the main esp32

components/esptool_py/Makefile.projbuild

@@ -54,4 +54,6 @@ app-flash: $(APP_BIN) $(ESPTOOLPY_SRC)
 	@echo "Flashing app to serial port $(ESPPORT), offset $(CONFIG_APP_OFFSET)..."
 	$(ESPTOOLPY_WRITE_FLASH) $(CONFIG_APP_OFFSET) $(APP_BIN)
 
-$(eval $(call SubmoduleCheck,$(ESPTOOLPY_SRC),$(COMPONENT_PATH)/esptool))
+# Submodules normally added in component.mk, but can be added
+# at the project level as long as qualified path
+COMPONENT_SUBMODULES += $(COMPONENT_PATH)/esptool

components/micro-ecc/component.mk

@@ -4,3 +4,5 @@ COMPONENT_SRCDIRS := micro-ecc
 COMPONENT_OBJS := micro-ecc/uECC.o
 
 COMPONENT_ADD_INCLUDEDIRS := micro-ecc
+
+COMPONENT_SUBMODULES := micro-ecc

docs/build_system.rst

@@ -186,6 +186,14 @@ The following variables can be set inside ``component.mk`` to control build sett
   generates an include file which you then want to include in another
   component. Most components do not need to set this variable.
 
+The following variable only works for components that are part of esp-idf itself:
+
+- ``COMPONENT_SUBMODULES``: Optional list of git submodule paths
+  (relative to COMPONENT_PATH) used by the component. These will be
+  checked (and initialised if necessary) by the build process. This
+  variable is ignored if the component is outside the IDF_PATH
+  directory.
+
 
 Optional Component-Specific Variables
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

make/common.mk

@@ -26,31 +26,6 @@ details := @true
 MAKEFLAGS += --silent
 endif
 
-# Pseudo-target to check a git submodule has been properly initialised
-#
-# $(eval $(call SubmoduleCheck,FILENAMES,SUBMODULE_PATH)) to create a target that
-# automatically runs 'git submodule update --init SUBMODULE_PATH' if any of
-# the files in FILENAMES are missing, and fails if this is not possible.
-#
-# Will also print a WARNING if the submodule at SUBMODULE_PATH appears
-# to require an update.
-define SubmoduleCheck
-$(1):
-	@echo "WARNING: Missing submodule $(2) for $$@..."
-	[ -d ${IDF_PATH}/.git ] || ( echo "ERROR: esp-idf must be cloned from git to work."; exit 1)
-	[ -x $(which git) ] || ( echo "ERROR: Need to run 'git submodule --init' in esp-idf root directory."; exit 1)
-	@echo "Attempting 'git submodule update --init' in esp-idf root directory..."
-	cd ${IDF_PATH} && git submodule update --init $(2)
-
-# Parse 'git submodule status' output for out-of-date submodule.
-# Status output prefixes status line with '+' if the submodule commit doesn't match
-ifneq ("$(shell cd ${IDF_PATH} && git submodule status $(2) | grep '^+')","")
-$$(info WARNING: git submodule $2 may be out of date. Run 'git submodule update' to update.)
-endif
-endef
-
-
-
 # General make utilities
 
 # convenience variable for printing an 80 asterisk wide separator line

make/component_wrapper.mk

@@ -103,8 +103,8 @@ endef
 
 # component_project_vars.mk target for the component. This is used to
 # take component.mk variables COMPONENT_ADD_INCLUDEDIRS,
-# COMPONENT_ADD_LDFLAGS and COMPONENT_DEPENDS and inject those into
+# COMPONENT_ADD_LDFLAGS, COMPONENT_DEPENDS and COMPONENT_SUBMODULES
-# the project make pass.
+# and inject those into the project make pass.
 #
 # The target here has no dependencies, as the parent target in
 # project.mk evaluates dependencies before calling down to here. See
@@ -119,6 +119,7 @@ component_project_vars.mk::
 	@echo '# Automatically generated build file. Do not edit.' > $@
 	@echo 'COMPONENT_INCLUDES += $(call MakeVariablePath,$(addprefix $(COMPONENT_PATH)/,$(COMPONENT_ADD_INCLUDEDIRS)))' >> $@
 	@echo 'COMPONENT_LDFLAGS += $(call MakeVariablePath,$(COMPONENT_ADD_LDFLAGS))' >> $@
+	@echo 'COMPONENT_SUBMODULES += $(call MakeVariablePath,$(addprefix $(COMPONENT_PATH)/,$(COMPONENT_SUBMODULES)))' >> $@
 	@echo '$(COMPONENT_NAME)-build: $(addsuffix -build,$(COMPONENT_DEPENDS))' >> $@
 
 
@@ -179,7 +180,7 @@ $(foreach srcdir,$(COMPONENT_SRCDIRS), $(eval $(call GenerateCompileTargets,$(sr
 
 ## Support for embedding binary files into the ELF as symbols
 
-OBJCOPY_EMBED_ARGS := --input binary --output elf32-xtensa-le --binary-architecture xtensa --rename-section .data=.rodata.embedded
+OBJCOPY_EMBED_ARGS := --input-target binary --output-target elf32-xtensa-le --binary-architecture xtensa --rename-section .data=.rodata.embedded
 
 # Generate pattern for embedding text or binary files into the app
 # $(1) is name of file (as relative path inside component)
@@ -188,18 +189,29 @@ OBJCOPY_EMBED_ARGS := --input binary --output elf32-xtensa-le --binary-architect
 # txt files are null-terminated before being embedded (otherwise
 # identical behaviour.)
 #
-# Files are temporarily copied to the build directory before objcopy,
-# because objcopy generates the symbol name from the full command line
-# path to the input file.
 define GenerateEmbedTarget
-$(1).$(2).o: $(call resolvepath,$(1),$(COMPONENT_PATH)) | $$(dir $(1))
+
+# copy the input file into the build dir (using a subdirectory
+# in case the file already exists elsewhere in the build dir)
+embed_bin/$$(notdir $(1)): $(call resolvepath,$(1),$(COMPONENT_PATH)) | embed_bin
+	cp $$< $$@
+
+embed_txt/$$(notdir $(1)): $(call resolvepath,$(1),$(COMPONENT_PATH)) | embed_txt
+	cp $$< $$@
+	echo -ne '\0' >> $$@  # null-terminate text files
+
+# messing about with the embed_X subdirectory then using 'cd' for objcopy is because the
+# full path passed to OBJCOPY makes it into the name of the symbols in the .o file
+$(1).$(2).o: embed_$(2)/$$(notdir $(1)) | $$(dir $(1))
 	$(summary) EMBED $$@
-	$$(if $$(filter-out $$(notdir $$(abspath $$<)),$$(abspath $$(notdir $$<))), cp $$< $$(notdir $$<) )  # copy input file to build dir, unless already in build dir
+	cd embed_$(2); $(OBJCOPY) $(OBJCOPY_EMBED_ARGS) $$(notdir $$<) ../$$@
-	$$(if $$(subst bin,,$(2)),echo -ne '\0' >> $$(notdir $$<) )  # trailing NUL byte on text output
+
-	$(OBJCOPY) $(OBJCOPY_EMBED_ARGS) $$(notdir $$<) $$@
+CLEAN_FILES += embed_$(2)/$$(notdir $(1))
-	rm $$(notdir $$<)
 endef
 
+embed_txt embed_bin:
+	mkdir -p $@
+
 # generate targets to embed binary & text files
 $(foreach binfile,$(COMPONENT_EMBED_FILES), $(eval $(call GenerateEmbedTarget,$(binfile),bin)))
 

make/project.mk

@@ -10,7 +10,7 @@
 # where this file is located.
 #
 
-.PHONY: build-components menuconfig defconfig all build clean all_binaries
+.PHONY: build-components menuconfig defconfig all build clean all_binaries check-submodules
 all: all_binaries
 # see below for recipe of 'all' target
 #
@@ -94,13 +94,16 @@ COMPONENT_PATHS += $(abspath $(SRCDIRS))
 # A component is buildable if it has a component.mk makefile in it
 COMPONENT_PATHS_BUILDABLE := $(foreach cp,$(COMPONENT_PATHS),$(if $(wildcard $(cp)/component.mk),$(cp)))
 
-# Initialise a project-wide list of include dirs (COMPONENT_INCLUDES),
+# Initialise project-wide variables which can be added to by
-# and LDFLAGS args (COMPONENT_LDFLAGS) supplied by each component.
+# each component.
 #
 # These variables are built up via the component_project_vars.mk
 # generated makefiles (one per component).
+#
+# See docs/build-system.rst for more details.
 COMPONENT_INCLUDES :=
 COMPONENT_LDFLAGS :=
+COMPONENT_SUBMODULES :=
 
 # COMPONENT_PROJECT_VARS is the list of component_project_vars.mk generated makefiles
 # for each component.
@@ -289,7 +292,7 @@ endef
 define GenerateComponentTargets
 .PHONY: $(2)-build $(2)-clean
 
-$(2)-build:
+$(2)-build: check-submodules
 	$(call ComponentMake,$(1),$(2)) build
 
 $(2)-clean:
@@ -332,4 +335,30 @@ app-clean: $(addsuffix -clean,$(notdir $(COMPONENT_PATHS_BUILDABLE)))
 config-clean: app-clean
 clean: config-clean
 
+# phony target to check if any git submodule listed in COMPONENT_SUBMODULES are missing
+# or out of date, and exit if so. Components can add paths to this variable.
+#
+# This only works for components inside IDF_PATH
+check-submodules:
 
+# Generate a target to check this submodule
+# $(1) - submodule directory, relative to IDF_PATH
+define GenerateSubmoduleCheckTarget
+check-submodules: $(IDF_PATH)/$(1)/.git
+$(IDF_PATH)/$(1)/.git:
+	@echo "WARNING: Missing submodule $(1)..."
+	[ -d ${IDF_PATH}/.git ] || ( echo "ERROR: esp-idf must be cloned from git to work."; exit 1)
+	[ -x $(which git) ] || ( echo "ERROR: Need to run 'git submodule init $(1)' in esp-idf root directory."; exit 1)
+	@echo "Attempting 'git submodule update --init $(1)' in esp-idf root directory..."
+	cd ${IDF_PATH} && git submodule update --init $(1)
+
+# Parse 'git submodule status' output for out-of-date submodule.
+# Status output prefixes status line with '+' if the submodule commit doesn't match
+ifneq ("$(shell cd ${IDF_PATH} && git submodule status $(1) | grep '^+')","")
+$$(info WARNING: git submodule $(1) may be out of date. Run 'git submodule update' to update.)
+endif
+endef
+
+# filter/subst in expression ensures all submodule paths begin with $(IDF_PATH), and then strips that prefix
+# so the argument is suitable for use with 'git submodule' commands
+$(foreach submodule,$(subst $(IDF_PATH)/,,$(filter $(IDF_PATH)/%,$(COMPONENT_SUBMODULES))),$(eval $(call GenerateSubmoduleCheckTarget,$(submodule))))