From 548ea1bdd523f894c3b86f0395776887c88566e1 Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Fri, 11 Dec 2020 16:28:11 +0100 Subject: [PATCH] tools: Wrap flash binaries into a UF2 file for flashing through USB MSC @mmoskal This commit adds basic support for UF2 into ESP-IDF. --- components/bootloader/Makefile.projbuild | 1 + components/esptool_py/Makefile.projbuild | 16 ++ components/partition_table/Makefile.projbuild | 1 + tools/ci/config/host-test.yml | 6 + tools/ci/executable-list.txt | 2 + tools/ci/test_build_system.sh | 9 + tools/ci/test_build_system_cmake.sh | 15 ++ tools/cmake/idf.cmake | 1 + tools/cmake/project.cmake | 3 + tools/cmake/uf2.cmake | 41 ++++ tools/idf.py | 2 +- tools/idf_py_actions/uf2_ext.py | 24 ++ tools/mkuf2.py | 212 +++++++++++++++++ tools/test_mkuf2/test_mkuf2.py | 225 ++++++++++++++++++ 14 files changed, 557 insertions(+), 1 deletion(-) create mode 100644 tools/cmake/uf2.cmake create mode 100644 tools/idf_py_actions/uf2_ext.py create mode 100755 tools/mkuf2.py create mode 100755 tools/test_mkuf2/test_mkuf2.py diff --git a/components/bootloader/Makefile.projbuild b/components/bootloader/Makefile.projbuild index e624ee7499..0b2f1ef20c 100644 --- a/components/bootloader/Makefile.projbuild +++ b/components/bootloader/Makefile.projbuild @@ -55,6 +55,7 @@ bootloader: $(BOOTLOADER_BIN) | check_python_dependencies @echo "$(ESPTOOLPY_WRITE_FLASH) $(BOOTLOADER_OFFSET) $^" ESPTOOL_ALL_FLASH_ARGS += $(BOOTLOADER_OFFSET) $(BOOTLOADER_BIN) +UF2_ADD_BINARIES += $(BOOTLOADER_OFFSET) $(BOOTLOADER_BIN) bootloader-flash: $(BOOTLOADER_BIN) $(call prereq_if_explicit,erase_flash) | check_python_dependencies $(ESPTOOLPY_WRITE_FLASH) 0x1000 $^ diff --git a/components/esptool_py/Makefile.projbuild b/components/esptool_py/Makefile.projbuild index 1d3007b820..2311d7497d 100644 --- a/components/esptool_py/Makefile.projbuild +++ b/components/esptool_py/Makefile.projbuild @@ -66,6 +66,7 @@ ESPTOOLPY_WRITE_FLASH_ENCRYPT=$(ESPTOOLPY_SERIAL) write_flash --encrypt $(if $(C endif ESPTOOL_ALL_FLASH_ARGS += $(APP_OFFSET) $(APP_BIN) +UF2_ADD_BINARIES += $(APP_OFFSET) $(APP_BIN) ifdef CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES ifndef IS_BOOTLOADER_BUILD @@ -164,4 +165,19 @@ monitor: $(call prereq_if_explicit,%flash) | check_python_dependencies [ -f $(APP_ELF) ] || exit 1 $(MONITOR_PYTHON) $(IDF_PATH)/tools/idf_monitor.py $(MONITOR_OPTS) $(APP_ELF) +# Make supports ESP32 only +UF2_CHIP_ID = "0x1c5f21b0" + +uf2-app: $(APP_BIN) partition_table_get_info + $(PYTHON) $(IDF_PATH)/tools/mkuf2.py write \ + -o "$(BUILD_DIR_BASE)/uf2-app.bin" \ + --chip-id "$(UF2_CHIP_ID)" \ + $(APP_OFFSET) $(APP_BIN) + +uf2: all_binaries + $(PYTHON) $(IDF_PATH)/tools/mkuf2.py write \ + -o "$(BUILD_DIR_BASE)/uf2.bin" \ + --chip-id "$(UF2_CHIP_ID)" \ + $(UF2_ADD_BINARIES) + .PHONY: erase_flash diff --git a/components/partition_table/Makefile.projbuild b/components/partition_table/Makefile.projbuild index a6585f8353..7b85adaa0f 100644 --- a/components/partition_table/Makefile.projbuild +++ b/components/partition_table/Makefile.projbuild @@ -96,6 +96,7 @@ check_table_contents: partition_table_get_info PARTITION_TABLE_FLASH_CMD = $(ESPTOOLPY_SERIAL) write_flash $(PARTITION_TABLE_OFFSET) $(PARTITION_TABLE_BIN) ESPTOOL_ALL_FLASH_ARGS += $(PARTITION_TABLE_OFFSET) $(PARTITION_TABLE_BIN) +UF2_ADD_BINARIES += $(PARTITION_TABLE_OFFSET) $(PARTITION_TABLE_BIN) partition_table: $(PARTITION_TABLE_BIN) partition_table_get_info | check_python_dependencies @echo "Partition table binary generated. Contents:" diff --git a/tools/ci/config/host-test.yml b/tools/ci/config/host-test.yml index 19789d6570..645aeb306b 100644 --- a/tools/ci/config/host-test.yml +++ b/tools/ci/config/host-test.yml @@ -312,6 +312,12 @@ test_mkdfu: - cd ${IDF_PATH}/tools/test_mkdfu - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py +test_mkuf2: + extends: .host_test_template + script: + - cd ${IDF_PATH}/tools/test_mkuf2 + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkuf2.py + test_docs: extends: .host_test_template image: $ESP_IDF_DOC_ENV_IMAGE diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 5cea2cbb28..1ce46dea38 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -92,12 +92,14 @@ tools/ldgen/test/test_fragments.py tools/ldgen/test/test_generation.py tools/mass_mfg/mfg_gen.py tools/mkdfu.py +tools/mkuf2.py tools/set-submodules-to-github.sh tools/test_idf_monitor/run_test_idf_monitor.py tools/test_idf_py/test_idf_py.py tools/test_idf_size/test.sh tools/test_idf_tools/test_idf_tools.py tools/test_mkdfu/test_mkdfu.py +tools/test_mkuf2/test_mkuf2.py tools/unit-test-app/tools/get_available_configs.sh tools/unit-test-app/unit_test.py tools/windows/eclipse_make.sh diff --git a/tools/ci/test_build_system.sh b/tools/ci/test_build_system.sh index 780bda49f6..f66c935f7b 100755 --- a/tools/ci/test_build_system.sh +++ b/tools/ci/test_build_system.sh @@ -333,6 +333,15 @@ function run_tests() rm sdkconfig.defaults make defconfig + print_status "UF2 build works" + rm -f -r build sdkconfig + make defconfig + make uf2 + assert_built ${APP_BINS} "uf2.bin" + make uf2-app + assert_built "uf2-app.bin" + rm -f -r build sdkconfig + print_status "Empty directory not treated as a component" mkdir -p components/esp32 make || failure "Failed to build with empty esp32 directory in components" diff --git a/tools/ci/test_build_system_cmake.sh b/tools/ci/test_build_system_cmake.sh index 10794abcb6..3d014a5159 100755 --- a/tools/ci/test_build_system_cmake.sh +++ b/tools/ci/test_build_system_cmake.sh @@ -743,6 +743,21 @@ endmenu\n" >> ${IDF_PATH}/Kconfig assert_built ${APP_BINS} ${BOOTLOADER_BINS} ${PARTITION_BIN} "dfu.bin" rm -rf build sdkconfig + print_status "UF2 build works" + rm -f -r build sdkconfig + idf.py uf2 &> tmp.log + grep "build/uf2.bin\" has been written." tmp.log || (tail -n 100 tmp.log ; failure "UF2 build works for esp32") + assert_built ${APP_BINS} ${BOOTLOADER_BINS} ${PARTITION_BIN} "uf2.bin" + idf.py uf2-app &> tmp.log + grep "build/uf2-app.bin\" has been written." tmp.log || (tail -n 100 tmp.log ; failure "UF2 build works for application binary") + assert_built "uf2-app.bin" + idf.py set-target esp32s2 + idf.py uf2 &> tmp.log + grep "build/uf2.bin\" has been written." tmp.log || (tail -n 100 tmp.log ; failure "UF2 build works for esp32s2") + rm tmp.log + assert_built ${APP_BINS} ${BOOTLOADER_BINS} ${PARTITION_BIN} "uf2.bin" + rm -rf build sdkconfig + print_status "Loadable ELF build works" echo "CONFIG_APP_BUILD_TYPE_ELF_RAM=y" > sdkconfig idf.py reconfigure || failure "Couldn't configure for loadable ELF file" diff --git a/tools/cmake/idf.cmake b/tools/cmake/idf.cmake index 82468355e9..547473f22b 100644 --- a/tools/cmake/idf.cmake +++ b/tools/cmake/idf.cmake @@ -44,6 +44,7 @@ if(NOT __idf_env_set) include(targets) include(ldgen) include(dfu) + include(uf2) include(version) __build_init("${idf_path}") diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index 8b47268f54..3060dccdcb 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -494,6 +494,9 @@ macro(project project_name) # Add DFU build and flash targets __add_dfu_targets() + # Add UF2 build targets + __add_uf2_targets() + idf_build_executable(${project_elf}) __project_info("${test_components}") diff --git a/tools/cmake/uf2.cmake b/tools/cmake/uf2.cmake new file mode 100644 index 0000000000..1543f7bf0f --- /dev/null +++ b/tools/cmake/uf2.cmake @@ -0,0 +1,41 @@ +# Add UF2 build target + +function(__add_uf2_targets) + idf_build_get_property(target IDF_TARGET) + if("${target}" STREQUAL "esp32") + set(uf2_family_id "0x1c5f21b0") + elseif("${target}" STREQUAL "esp32s2") + set(uf2_family_id "0xbfdd4eee") + elseif("${target}" STREQUAL "esp32c3") + set(uf2_family_id "0xd42ba06c") + elseif("${target}" STREQUAL "esp32s3") + set(uf2_family_id "0xc47e5767") + elseif("${target}" STREQUAL "linux") + return() + else() + message(FATAL_ERROR "UF2 family identificator is unknown for ${target}") + # Generate an ID and submit a pull request as described here: https://github.com/microsoft/uf2 + endif() + + idf_build_get_property(python PYTHON) + idf_build_get_property(idf_path IDF_PATH) + + add_custom_target(uf2-app + COMMAND ${python} ${idf_path}/tools/mkuf2.py write + -o "${CMAKE_CURRENT_BINARY_DIR}/uf2-app.bin" + --json "${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json" + --chip-id "${uf2_family_id}" + --bin app + DEPENDS gen_project_binary + VERBATIM + USES_TERMINAL) + + add_custom_target(uf2 + COMMAND ${python} ${idf_path}/tools/mkuf2.py write + -o "${CMAKE_CURRENT_BINARY_DIR}/uf2.bin" + --json "${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json" + --chip-id "${uf2_family_id}" + DEPENDS gen_project_binary bootloader + VERBATIM + USES_TERMINAL) +endfunction() diff --git a/tools/idf.py b/tools/idf.py index 04c4d05036..c44520227b 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -465,7 +465,7 @@ def init_cli(verbose_output=None): def _print_closing_message(self, args, actions): # print a closing message of some kind # - if "flash" in str(actions) or "dfu" in str(actions): + if any(t in str(actions) for t in ("flash", "dfu", "uf2", "uf2-app")): print("Done") return diff --git a/tools/idf_py_actions/uf2_ext.py b/tools/idf_py_actions/uf2_ext.py new file mode 100644 index 0000000000..6635c860a0 --- /dev/null +++ b/tools/idf_py_actions/uf2_ext.py @@ -0,0 +1,24 @@ +from idf_py_actions.tools import ensure_build_directory, run_target + + +def action_extensions(base_actions, project_path): + def uf2_target(target_name, ctx, args): + ensure_build_directory(args, ctx.info_name) + run_target(target_name, args) + + uf2_actions = { + "actions": { + "uf2": { + "callback": uf2_target, + "short_help": "Generate the UF2 binary with all the binaries included", + "dependencies": ["all"], + }, + "uf2-app": { + "callback": uf2_target, + "short_help": "Generate an UF2 binary for the application only", + "dependencies": ["all"], + }, + } + } + + return uf2_actions diff --git a/tools/mkuf2.py b/tools/mkuf2.py new file mode 100755 index 0000000000..c672af15e1 --- /dev/null +++ b/tools/mkuf2.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# +# Copyright 2020 Espressif Systems (Shanghai) CO LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +import argparse +import hashlib +import json +import os +import struct + +from functools import partial +from future.utils import iteritems + +try: + from itertools import izip as zip +except ImportError: + # Python 3 + pass + + +def round_up_int_div(n, d): + # equivalent to math.ceil(n / d) + return (n + d - 1) // d + + +class UF2Writer(object): + + # The UF2 format is described here: https://github.com/microsoft/uf2 + UF2_BLOCK_SIZE = 512 + UF2_DATA_SIZE = 476 # max value of CHUNK_SIZE reduced by optional parts. Currently, MD5_PART only. + UF2_MD5_PART_SIZE = 24 + UF2_FIRST_MAGIC = 0x0A324655 + UF2_SECOND_MAGIC = 0x9E5D5157 + UF2_FINAL_MAGIC = 0x0AB16F30 + UF2_FLAG_FAMILYID_PRESENT = 0x00002000 + UF2_FLAG_MD5_PRESENT = 0x00004000 + + def __init__(self, chip_id, output_file, chunk_size): + self.chip_id = chip_id + self.CHUNK_SIZE = self.UF2_DATA_SIZE - self.UF2_MD5_PART_SIZE if chunk_size is None else chunk_size + self.f = open(output_file, 'wb') + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.f: + self.f.close() + + @staticmethod + def _to_uint32(num): + return struct.pack(' 0 + assert len_chunk <= self.CHUNK_SIZE + assert block_no < blocks + block = self._to_uint32(self.UF2_FIRST_MAGIC) + block += self._to_uint32(self.UF2_SECOND_MAGIC) + block += self._to_uint32(self.UF2_FLAG_FAMILYID_PRESENT | self.UF2_FLAG_MD5_PRESENT) + block += self._to_uint32(addr) + block += self._to_uint32(len_chunk) + block += self._to_uint32(block_no) + block += self._to_uint32(blocks) + block += self._to_uint32(self.chip_id) + block += chunk + + md5_part = self._to_uint32(addr) + md5_part += self._to_uint32(len_chunk) + md5_part += hashlib.md5(chunk).digest() + assert(len(md5_part) == self.UF2_MD5_PART_SIZE) + + block += md5_part + block += b'\x00' * (self.UF2_DATA_SIZE - self.UF2_MD5_PART_SIZE - len_chunk) + block += self._to_uint32(self.UF2_FINAL_MAGIC) + assert len(block) == self.UF2_BLOCK_SIZE + self.f.write(block) + + def add_file(self, addr, f_path): + blocks = round_up_int_div(os.path.getsize(f_path), self.CHUNK_SIZE) + with open(f_path, 'rb') as fin: + a = addr + for i, chunk in enumerate(iter(partial(fin.read, self.CHUNK_SIZE), b'')): + len_chunk = len(chunk) + self._write_block(a, chunk, len_chunk, i, blocks) + a += len_chunk + + +def action_write(args): + with UF2Writer(args['chip_id'], args['output_file'], args['chunk_size']) as writer: + for addr, f in args['files']: + print('Adding {} at {:#x}'.format(f, addr)) + writer.add_file(addr, f) + print('"{}" has been written.'.format(args['output_file'])) + + +def main(): + parser = argparse.ArgumentParser() + + def four_byte_aligned(integer): + return integer & 3 == 0 + + def parse_chunk_size(string): + num = int(string, 0) + if not four_byte_aligned(num): + raise argparse.ArgumentTypeError('Chunk size should be a 4-byte aligned number') + return num + + def parse_chip_id(string): + num = int(string, 16) + if num < 0 or num > 0xFFFFFFFF: + raise argparse.ArgumentTypeError('Chip ID should be a 4-byte unsigned integer') + return num + + # Provision to add "info" command + subparsers = parser.add_subparsers(dest="command") + write_parser = subparsers.add_parser("write") + write_parser.add_argument("-o", "--output-file", + help='Filename for storing the output UF2 image', + required=True) + write_parser.add_argument("--chip-id", + required=True, + type=parse_chip_id, + help='Hexa-decimal chip identificator') + write_parser.add_argument("--chunk-size", + required=False, + type=parse_chunk_size, + default=None, + help='Specify the used data part of the 512 byte UF2 block. A common value is 256. By ' + 'default the largest possible value will be used.') + write_parser.add_argument("--json", + help='Optional file for loading "flash_files" dictionary with
items') + write_parser.add_argument("--bin", + help='Use only a subset of binaries from the JSON file, e.g. "partition_table ' + 'bootloader app"', + nargs='*') + write_parser.add_argument("files", + metavar="
", help='Add at
', + nargs="*") + + args = parser.parse_args() + + def check_file(file_name): + if not os.path.isfile(file_name): + raise RuntimeError('{} is not a regular file!'.format(file_name)) + return file_name + + def parse_addr(string): + num = int(string, 0) + if not four_byte_aligned(num): + raise RuntimeError('{} is not a 4-byte aligned valid address'.format(string)) + return num + + files = [] + if args.files: + files += [(parse_addr(addr), check_file(f_name)) for addr, f_name in zip(args.files[::2], args.files[1::2])] + + if args.json: + json_dir = os.path.dirname(os.path.abspath(args.json)) + + def process_json_file(path): + ''' + The input path is relative to json_dir. This function makes it relative to the current working + directory. + ''' + return check_file(os.path.relpath(os.path.join(json_dir, path), start=os.curdir)) + + with open(args.json) as f: + json_content = json.load(f) + + if args.bin: + try: + bin_selection = [json_content[b] for b in args.bin] + flash_dic = dict((x['offset'], x['file']) for x in bin_selection) + except KeyError: + print('Invalid binary was selected.') + valid = [k if all(x in v for x in ('offset', 'file')) else None for k, v in iteritems(json_content)] + print('Valid ones:', ' '.join(x for x in valid if x)) + exit(1) + else: + flash_dic = json_content['flash_files'] + + files += [(parse_addr(addr), process_json_file(f_name)) for addr, f_name in iteritems(flash_dic)] + + files = sorted([(addr, f_name) for addr, f_name in iteritems(dict(files))], + key=lambda x: x[0]) # remove possible duplicates and sort based on the address + + cmd_args = {'output_file': args.output_file, + 'files': files, + 'chip_id': args.chip_id, + 'chunk_size': args.chunk_size, + } + + {'write': action_write + }[args.command](cmd_args) + + +if __name__ == "__main__": + main() diff --git a/tools/test_mkuf2/test_mkuf2.py b/tools/test_mkuf2/test_mkuf2.py new file mode 100755 index 0000000000..b13cf5ea41 --- /dev/null +++ b/tools/test_mkuf2/test_mkuf2.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2020 Espressif Systems (Shanghai) CO LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import unicode_literals + +import filecmp +import hashlib +import os +import pexpect +import random +import struct +import sys +import tempfile +import time +import unittest + +from functools import partial +from io import open +from itertools import chain + +try: + from itertools import izip as zip +except ImportError: + # Python 3 + pass + +current_dir = os.path.dirname(os.path.realpath(__file__)) +mkuf2_dir = os.path.abspath(os.path.join(current_dir, '..')) +mkuf2_path = os.path.join(mkuf2_dir, 'mkuf2.py') + +try: + import mkuf2 +except ImportError: + sys.path.append(mkuf2_dir) + import mkuf2 + + +class UF2Block(object): + def __init__(self, bs): + self.length = len(bs) + + # See https://github.com/microsoft/uf2 for the format + first_part = '<' + 'I' * 8 + # payload is between + last_part = '