diff --git a/components/partition_table/gen_esp32part.py b/components/partition_table/gen_esp32part.py index 396f6ff22f..53790846d5 100755 --- a/components/partition_table/gen_esp32part.py +++ b/components/partition_table/gen_esp32part.py @@ -11,6 +11,7 @@ # SPDX-License-Identifier: Apache-2.0 import argparse import binascii +import codecs import errno import hashlib import os @@ -175,21 +176,36 @@ def critical(msg): sys.stderr.write('\n') +def get_encoding(first_bytes): + """Detect the encoding by checking for BOM (Byte Order Mark)""" + BOMS = { + codecs.BOM_UTF8: 'utf-8-sig', + codecs.BOM_UTF16_LE: 'utf-16', + codecs.BOM_UTF16_BE: 'utf-16', + codecs.BOM_UTF32_LE: 'utf-32', + codecs.BOM_UTF32_BE: 'utf-32', + } + for bom, encoding in BOMS.items(): + if first_bytes.startswith(bom): + return encoding + return 'utf-8' + + class PartitionTable(list): def __init__(self): super(PartitionTable, self).__init__(self) @classmethod def from_file(cls, f): - data = f.read() - data_is_binary = data[0:2] == PartitionDefinition.MAGIC_BYTES + bin_data = f.read() + data_is_binary = bin_data[0:2] == PartitionDefinition.MAGIC_BYTES if data_is_binary: status('Parsing binary partition input...') - return cls.from_binary(data), True + return cls.from_binary(bin_data), True - data = data.decode() + str_data = bin_data.decode(get_encoding(bin_data)) status('Parsing CSV input...') - return cls.from_csv(data), False + return cls.from_csv(str_data), False @classmethod def from_csv(cls, csv_contents): diff --git a/components/partition_table/parttool.py b/components/partition_table/parttool.py index 6751e879e0..7d507b77eb 100755 --- a/components/partition_table/parttool.py +++ b/components/partition_table/parttool.py @@ -3,7 +3,7 @@ # parttool is used to perform partition level operations - reading, # writing, erasing and getting info about the partition. # -# SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import os @@ -30,8 +30,7 @@ def status(msg): print(msg) -class _PartitionId(): - +class _PartitionId: def __init__(self, name=None, p_type=None, subtype=None, part_list=None): self.name = name self.type = p_type @@ -40,13 +39,11 @@ class _PartitionId(): class PartitionName(_PartitionId): - def __init__(self, name): _PartitionId.__init__(self, name=name) class PartitionType(_PartitionId): - def __init__(self, p_type, subtype, part_list=None): _PartitionId.__init__(self, p_type=p_type, subtype=subtype, part_list=part_list) @@ -54,16 +51,27 @@ class PartitionType(_PartitionId): PARTITION_BOOT_DEFAULT = _PartitionId() -class ParttoolTarget(): - - def __init__(self, port=None, baud=None, partition_table_offset=PARTITION_TABLE_OFFSET, primary_bootloader_offset=None, recovery_bootloader_offset=None, - partition_table_file=None, esptool_args=[], esptool_write_args=[], esptool_read_args=[], esptool_erase_args=[]): +class ParttoolTarget: + def __init__( + self, + port=None, + baud=None, + partition_table_offset=PARTITION_TABLE_OFFSET, + primary_bootloader_offset=None, + recovery_bootloader_offset=None, + partition_table_file=None, + esptool_args=[], + esptool_write_args=[], + esptool_read_args=[], + esptool_erase_args=[], + ): self.port = port self.baud = baud gen.offset_part_table = partition_table_offset gen.primary_bootloader_offset = primary_bootloader_offset gen.recovery_bootloader_offset = recovery_bootloader_offset + gen.quiet = True def parse_esptool_args(esptool_args): results = list() @@ -84,23 +92,16 @@ class ParttoolTarget(): self.esptool_erase_args = parse_esptool_args(esptool_erase_args) if partition_table_file: - partition_table = None with open(partition_table_file, 'rb') as f: - input_is_binary = (f.read(2) == gen.PartitionDefinition.MAGIC_BYTES) - f.seek(0) - if input_is_binary: - partition_table = gen.PartitionTable.from_binary(f.read()) - - if partition_table is None: - with open(partition_table_file, 'r', encoding='utf-8') as f: - f.seek(0) - partition_table = gen.PartitionTable.from_csv(f.read()) + partition_table, _ = gen.PartitionTable.from_file(f) else: temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file.close() try: - self._call_esptool(['read_flash', str(partition_table_offset), str(gen.MAX_PARTITION_LENGTH), temp_file.name]) + self._call_esptool( + ['read_flash', str(partition_table_offset), str(gen.MAX_PARTITION_LENGTH), temp_file.name] + ) with open(temp_file.name, 'rb') as f: partition_table = gen.PartitionTable.from_binary(f.read()) finally: @@ -152,13 +153,13 @@ class ParttoolTarget(): def erase_partition(self, partition_id): partition = self.get_partition_info(partition_id) - self._call_esptool(['erase_region', str(partition.offset), str(partition.size)] + self.esptool_erase_args) + self._call_esptool(['erase_region', str(partition.offset), str(partition.size)] + self.esptool_erase_args) def read_partition(self, partition_id, output): partition = self.get_partition_info(partition_id) self._call_esptool(['read_flash', str(partition.offset), str(partition.size), output] + self.esptool_read_args) - def write_partition(self, partition_id, input, ignore_readonly=False): + def write_partition(self, partition_id, input, ignore_readonly=False): # noqa: A002 partition = self.get_partition_info(partition_id) if partition.readonly and not ignore_readonly: @@ -166,8 +167,8 @@ class ParttoolTarget(): self.erase_partition(partition_id) - with open(input, 'rb') as input_file: - content_len = len(input_file.read()) + with open(input, 'rb') as f: + content_len = len(f.read()) if content_len > partition.size: raise Exception('Input file size exceeds partition size') @@ -175,7 +176,7 @@ class ParttoolTarget(): self._call_esptool(['write_flash', str(partition.offset), input] + self.esptool_write_args) -def _write_partition(target, partition_id, input, ignore_readonly=False): +def _write_partition(target, partition_id, input, ignore_readonly=False): # noqa: A002 target.write_partition(partition_id, input, ignore_readonly) partition = target.get_partition_info(partition_id) status("Written contents of file '{}' at offset 0x{:x}".format(input, partition.offset)) @@ -184,8 +185,11 @@ def _write_partition(target, partition_id, input, ignore_readonly=False): def _read_partition(target, partition_id, output): target.read_partition(partition_id, output) partition = target.get_partition_info(partition_id) - status("Read partition '{}' contents from device at offset 0x{:x} to file '{}'" - .format(partition.name, partition.offset, output)) + status( + "Read partition '{}' contents from device at offset 0x{:x} to file '{}'".format( + partition.name, partition.offset, output + ) + ) def _erase_partition(target, partition_id): @@ -213,7 +217,7 @@ def _get_partition_info(target, partition_id, info): 'offset': '0x{:x}'.format(p.offset), 'size': '0x{:x}'.format(p.size), 'encrypted': '{}'.format(p.encrypted), - 'readonly': '{}'.format(p.readonly) + 'readonly': '{}'.format(p.readonly), } for i in info: infos += [info_dict[i]] @@ -232,19 +236,29 @@ def main(): parser.add_argument('--esptool-args', help='additional main arguments for esptool', nargs='+') parser.add_argument('--esptool-write-args', help='additional subcommand arguments when writing to flash', nargs='+') parser.add_argument('--esptool-read-args', help='additional subcommand arguments when reading flash', nargs='+') - parser.add_argument('--esptool-erase-args', help='additional subcommand arguments when erasing regions of flash', nargs='+') + parser.add_argument( + '--esptool-erase-args', help='additional subcommand arguments when erasing regions of flash', nargs='+' + ) # By default the device attached to the specified port is queried for the partition table. If a partition table file # is specified, that is used instead. - parser.add_argument('--port', '-p', help='port where the target device of the command is connected to; the partition table is sourced from this device \ - when the partition table file is not defined') + parser.add_argument( + '--port', + '-p', + help='port where the target device of the command is connected to; the partition table is sourced from ' + 'this device when the partition table file is not defined', + ) parser.add_argument('--baud', '-b', help='baudrate to use', type=int) parser.add_argument('--partition-table-offset', '-o', help='offset to read the partition table from', type=str) parser.add_argument('--primary-bootloader-offset', help='offset for primary bootloader', type=str) parser.add_argument('--recovery-bootloader-offset', help='offset for recovery bootloader', type=str) - parser.add_argument('--partition-table-file', '-f', help='file (CSV/binary) to read the partition table from; \ - overrides device attached to specified port as the partition table source when defined') + parser.add_argument( + '--partition-table-file', + '-f', + help='file (CSV/binary) to read the partition table from; ' + 'overrides device attached to specified port as the partition table source when defined', + ) partition_selection_parser = argparse.ArgumentParser(add_help=False) @@ -254,31 +268,54 @@ def main(): partition_selection_args.add_argument('--partition-name', '-n', help='name of the partition') partition_selection_args.add_argument('--partition-type', '-t', help='type of the partition') - partition_selection_args.add_argument('--partition-boot-default', '-d', help='select the default boot partition \ - using the same fallback logic as the IDF bootloader', action='store_true') + partition_selection_args.add_argument( + '--partition-boot-default', + '-d', + help='select the default boot partition \ + using the same fallback logic as the IDF bootloader', + action='store_true', + ) partition_selection_parser.add_argument('--partition-subtype', '-s', help='subtype of the partition') - partition_selection_parser.add_argument('--extra-partition-subtypes', help='Extra partition subtype entries', nargs='*') + partition_selection_parser.add_argument( + '--extra-partition-subtypes', help='Extra partition subtype entries', nargs='*' + ) subparsers = parser.add_subparsers(dest='operation', help='run parttool -h for additional help') # Specify the supported operations - read_part_subparser = subparsers.add_parser('read_partition', help='read partition from device and dump contents into a file', - parents=[partition_selection_parser]) + read_part_subparser = subparsers.add_parser( + 'read_partition', + help='read partition from device and dump contents into a file', + parents=[partition_selection_parser], + ) read_part_subparser.add_argument('--output', help='file to dump the read partition contents to') - write_part_subparser = subparsers.add_parser('write_partition', help='write contents of a binary file to partition on device', - parents=[partition_selection_parser]) + write_part_subparser = subparsers.add_parser( + 'write_partition', + help='write contents of a binary file to partition on device', + parents=[partition_selection_parser], + ) write_part_subparser.add_argument('--input', help='file whose contents are to be written to the partition offset') write_part_subparser.add_argument('--ignore-readonly', help='Ignore read-only attribute', action='store_true') - subparsers.add_parser('erase_partition', help='erase the contents of a partition on the device', parents=[partition_selection_parser]) + subparsers.add_parser( + 'erase_partition', help='erase the contents of a partition on the device', parents=[partition_selection_parser] + ) - print_partition_info_subparser = subparsers.add_parser('get_partition_info', help='get partition information', parents=[partition_selection_parser]) - print_partition_info_subparser.add_argument('--info', help='type of partition information to get', - choices=['name', 'type', 'subtype', 'offset', 'size', 'encrypted', 'readonly'], - default=['offset', 'size'], nargs='+') - print_partition_info_subparser.add_argument('--part_list', help='Get a list of partitions suitable for a given type', action='store_true') + print_partition_info_subparser = subparsers.add_parser( + 'get_partition_info', help='get partition information', parents=[partition_selection_parser] + ) + print_partition_info_subparser.add_argument( + '--info', + help='type of partition information to get', + choices=['name', 'type', 'subtype', 'offset', 'size', 'encrypted', 'readonly'], + default=['offset', 'size'], + nargs='+', + ) + print_partition_info_subparser.add_argument( + '--part_list', help='Get a list of partitions suitable for a given type', action='store_true' + ) args = parser.parse_args() quiet = args.quiet @@ -299,8 +336,10 @@ def main(): elif args.partition_boot_default: partition_id = PARTITION_BOOT_DEFAULT else: - raise RuntimeError('Partition to operate on should be defined using --partition-name OR \ - partition-type,--partition-subtype OR partition-boot-default') + raise RuntimeError( + 'Partition to operate on should be defined using --partition-name OR \ + partition-type,--partition-subtype OR partition-boot-default' + ) # Prepare the device to perform operation on target_args = {} @@ -341,18 +380,18 @@ def main(): target = ParttoolTarget(**target_args) # Create the operation table and execute the operation - common_args = {'target':target, 'partition_id':partition_id} + common_args = {'target': target, 'partition_id': partition_id} parttool_ops = { 'erase_partition': (_erase_partition, []), 'read_partition': (_read_partition, ['output']), 'write_partition': (_write_partition, ['input', 'ignore_readonly']), - 'get_partition_info': (_get_partition_info, ['info']) + 'get_partition_info': (_get_partition_info, ['info']), } (op, op_args) = parttool_ops[args.operation] for op_arg in op_args: - common_args.update({op_arg:vars(args)[op_arg]}) + common_args.update({op_arg: vars(args)[op_arg]}) if quiet: # If exceptions occur, suppress and exit quietly diff --git a/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py b/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py index 0c930a8c42..fb0ea91f4b 100755 --- a/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py +++ b/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import csv import io @@ -26,38 +26,32 @@ factory,0,2,65536,1048576, LONGER_BINARY_TABLE = b'' # type 0x00, subtype 0x00, # offset 64KB, size 1MB -LONGER_BINARY_TABLE += b'\xAA\x50\x00\x00' + \ - b'\x00\x00\x01\x00' + \ - b'\x00\x00\x10\x00' + \ - b'factory\0' + (b'\0' * 8) + \ - b'\x00\x00\x00\x00' +LONGER_BINARY_TABLE += ( + b'\xaa\x50\x00\x00' + b'\x00\x00\x01\x00' + b'\x00\x00\x10\x00' + b'factory\0' + (b'\0' * 8) + b'\x00\x00\x00\x00' +) # type 0x01, subtype 0x20, # offset 0x110000, size 128KB -LONGER_BINARY_TABLE += b'\xAA\x50\x01\x20' + \ - b'\x00\x00\x11\x00' + \ - b'\x00\x02\x00\x00' + \ - b'data' + (b'\0' * 12) + \ - b'\x00\x00\x00\x00' +LONGER_BINARY_TABLE += ( + b'\xaa\x50\x01\x20' + b'\x00\x00\x11\x00' + b'\x00\x02\x00\x00' + b'data' + (b'\0' * 12) + b'\x00\x00\x00\x00' +) # type 0x10, subtype 0x00, # offset 0x150000, size 1MB -LONGER_BINARY_TABLE += b'\xAA\x50\x10\x00' + \ - b'\x00\x00\x15\x00' + \ - b'\x00\x10\x00\x00' + \ - b'second' + (b'\0' * 10) + \ - b'\x00\x00\x00\x00' +LONGER_BINARY_TABLE += ( + b'\xaa\x50\x10\x00' + b'\x00\x00\x15\x00' + b'\x00\x10\x00\x00' + b'second' + (b'\0' * 10) + b'\x00\x00\x00\x00' +) # MD5 checksum -LONGER_BINARY_TABLE += b'\xEB\xEB' + b'\xFF' * 14 +LONGER_BINARY_TABLE += b'\xeb\xeb' + b'\xff' * 14 LONGER_BINARY_TABLE += b'\xf9\xbd\x06\x1b\x45\x68\x6f\x86\x57\x1a\x2c\xd5\x2a\x1d\xa6\x5b' # empty partition -LONGER_BINARY_TABLE += b'\xFF' * 32 +LONGER_BINARY_TABLE += b'\xff' * 32 def _strip_trailing_ffs(binary_table): """ Strip all FFs down to the last 32 bytes (terminating entry) """ - while binary_table.endswith(b'\xFF' * 64): - binary_table = binary_table[0:len(binary_table) - 32] + while binary_table.endswith(b'\xff' * 64): + binary_table = binary_table[0 : len(binary_table) - 32] return binary_table @@ -143,10 +137,10 @@ otherapp, app, factory,, 1M t = gen_esp32part.PartitionTable.from_csv(csv) # 'first' self.assertEqual(t[0].offset, 0x010000) # 64KB boundary as it's an app image - self.assertEqual(t[0].size, 0x100000) # Size specified in CSV + self.assertEqual(t[0].size, 0x100000) # Size specified in CSV # 'second' self.assertEqual(t[1].offset, 0x110000) # prev offset+size - self.assertEqual(t[1].size, 0x100000) # Size specified in CSV + self.assertEqual(t[1].size, 0x100000) # Size specified in CSV # 'minidata' self.assertEqual(t[2].offset, 0x210000) # 'otherapp' @@ -162,7 +156,7 @@ second, data, 0x15, , 1M t.verify() # 'first' self.assertEqual(t[0].offset, 0x10000) # in CSV - self.assertEqual(t[0].size, 0x200000 - t[0].offset) # Up to 2M + self.assertEqual(t[0].size, 0x200000 - t[0].offset) # Up to 2M # 'second' self.assertEqual(t[1].offset, 0x200000) # prev offset+size @@ -188,7 +182,9 @@ first, app, ota_0, 0x200000, 1M csv = """ bootloader, bootloader, primary, N/A, N/A """ - with self.assertRaisesRegex(gen_esp32part.InputError, 'Primary bootloader offset is not defined. Please use --primary-bootloader-offset'): + with self.assertRaisesRegex( + gen_esp32part.InputError, 'Primary bootloader offset is not defined. Please use --primary-bootloader-offset' + ): gen_esp32part.PartitionTable.from_csv(csv) def test_bootloader_and_part_table_partitions(self): @@ -234,12 +230,12 @@ first, 0x30, 0xEE, 0x100400, 0x300000 t = gen_esp32part.PartitionTable.from_csv(csv) tb = _strip_trailing_ffs(t.to_binary()) self.assertEqual(len(tb), 64 + 32) - self.assertEqual(b'\xAA\x50', tb[0:2]) # magic + self.assertEqual(b'\xaa\x50', tb[0:2]) # magic self.assertEqual(b'\x30\xee', tb[2:4]) # type, subtype eo, es = struct.unpack('