feedc0de/esp-idf
1
0
Fork 0
forked from espressif/esp-idf
Code Pull Requests Activity

Merge branch 'feature/parttable_tool_use_only_ascii_for_names' into 'master'

Browse Source 
fix(partition_table): Ignore UTF-8 BOM bytes in csv file

See merge request espressif/esp-idf!38954
This commit is contained in:
Konstantin Kondrashov
2025-05-15 18:19:05 +08:00
parent c707faa41d 179eb5a5aa
commit 9f6136d650
7 changed files with 300 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
components/partition_table/gen_esp32part.py
View File

@ -11,6 +11,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
import argparse import argparse
import binascii import binascii
import codecs
import errno import errno
import hashlib import hashlib
import os import os
@ -175,21 +176,36 @@ def critical(msg):
sys.stderr.write('\n') sys.stderr.write('\n')
def get_encoding(first_bytes):
"""Detect the encoding by checking for BOM (Byte Order Mark)"""
BOMS = {
codecs.BOM_UTF8: 'utf-8-sig',
codecs.BOM_UTF16_LE: 'utf-16',
codecs.BOM_UTF16_BE: 'utf-16',
codecs.BOM_UTF32_LE: 'utf-32',
codecs.BOM_UTF32_BE: 'utf-32',
}
for bom, encoding in BOMS.items():
if first_bytes.startswith(bom):
return encoding
return 'utf-8'
class PartitionTable(list): class PartitionTable(list):
def __init__(self): def __init__(self):
super(PartitionTable, self).__init__(self) super(PartitionTable, self).__init__(self)
@classmethod @classmethod
def from_file(cls, f): def from_file(cls, f):
data = f.read() bin_data = f.read()
data_is_binary = data[0:2] == PartitionDefinition.MAGIC_BYTES data_is_binary = bin_data[0:2] == PartitionDefinition.MAGIC_BYTES
if data_is_binary: if data_is_binary:
status('Parsing binary partition input...') status('Parsing binary partition input...')
return cls.from_binary(data), True return cls.from_binary(bin_data), True
data = data.decode() str_data = bin_data.decode(get_encoding(bin_data))
status('Parsing CSV input...') status('Parsing CSV input...')
return cls.from_csv(data), False return cls.from_csv(str_data), False
@classmethod @classmethod
def from_csv(cls, csv_contents): def from_csv(cls, csv_contents):

141
components/partition_table/parttool.py
View File

@ -3,7 +3,7 @@
# parttool is used to perform partition level operations - reading, # parttool is used to perform partition level operations - reading,
# writing, erasing and getting info about the partition. # writing, erasing and getting info about the partition.
# #
# SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD # SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
import argparse import argparse
import os import os
@ -30,8 +30,7 @@ def status(msg):
print(msg) print(msg)
class _PartitionId(): class _PartitionId:
def __init__(self, name=None, p_type=None, subtype=None, part_list=None): def __init__(self, name=None, p_type=None, subtype=None, part_list=None):
self.name = name self.name = name
self.type = p_type self.type = p_type
@ -40,13 +39,11 @@ class _PartitionId():
class PartitionName(_PartitionId): class PartitionName(_PartitionId):
def __init__(self, name): def __init__(self, name):
_PartitionId.__init__(self, name=name) _PartitionId.__init__(self, name=name)
class PartitionType(_PartitionId): class PartitionType(_PartitionId):
def __init__(self, p_type, subtype, part_list=None): def __init__(self, p_type, subtype, part_list=None):
_PartitionId.__init__(self, p_type=p_type, subtype=subtype, part_list=part_list) _PartitionId.__init__(self, p_type=p_type, subtype=subtype, part_list=part_list)
@ -54,16 +51,27 @@ class PartitionType(_PartitionId):
PARTITION_BOOT_DEFAULT = _PartitionId() PARTITION_BOOT_DEFAULT = _PartitionId()
class ParttoolTarget(): class ParttoolTarget:
def __init__(
def __init__(self, port=None, baud=None, partition_table_offset=PARTITION_TABLE_OFFSET, primary_bootloader_offset=None, recovery_bootloader_offset=None, self,
partition_table_file=None, esptool_args=[], esptool_write_args=[], esptool_read_args=[], esptool_erase_args=[]): port=None,
baud=None,
partition_table_offset=PARTITION_TABLE_OFFSET,
primary_bootloader_offset=None,
recovery_bootloader_offset=None,
partition_table_file=None,
esptool_args=[],
esptool_write_args=[],
esptool_read_args=[],
esptool_erase_args=[],
):
self.port = port self.port = port
self.baud = baud self.baud = baud
gen.offset_part_table = partition_table_offset gen.offset_part_table = partition_table_offset
gen.primary_bootloader_offset = primary_bootloader_offset gen.primary_bootloader_offset = primary_bootloader_offset
gen.recovery_bootloader_offset = recovery_bootloader_offset gen.recovery_bootloader_offset = recovery_bootloader_offset
gen.quiet = True
def parse_esptool_args(esptool_args): def parse_esptool_args(esptool_args):
results = list() results = list()
@ -84,23 +92,16 @@ class ParttoolTarget():
self.esptool_erase_args = parse_esptool_args(esptool_erase_args) self.esptool_erase_args = parse_esptool_args(esptool_erase_args)
if partition_table_file: if partition_table_file:
partition_table = None
with open(partition_table_file, 'rb') as f: with open(partition_table_file, 'rb') as f:
input_is_binary = (f.read(2) == gen.PartitionDefinition.MAGIC_BYTES) partition_table, _ = gen.PartitionTable.from_file(f)
f.seek(0)
if input_is_binary:
partition_table = gen.PartitionTable.from_binary(f.read())
if partition_table is None:
with open(partition_table_file, 'r', encoding='utf-8') as f:
f.seek(0)
partition_table = gen.PartitionTable.from_csv(f.read())
else: else:
temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file = tempfile.NamedTemporaryFile(delete=False)
temp_file.close() temp_file.close()
try: try:
self._call_esptool(['read_flash', str(partition_table_offset), str(gen.MAX_PARTITION_LENGTH), temp_file.name]) self._call_esptool(
['read_flash', str(partition_table_offset), str(gen.MAX_PARTITION_LENGTH), temp_file.name]
)
with open(temp_file.name, 'rb') as f: with open(temp_file.name, 'rb') as f:
partition_table = gen.PartitionTable.from_binary(f.read()) partition_table = gen.PartitionTable.from_binary(f.read())
finally: finally:
@ -152,13 +153,13 @@ class ParttoolTarget():
def erase_partition(self, partition_id): def erase_partition(self, partition_id):
partition = self.get_partition_info(partition_id) partition = self.get_partition_info(partition_id)
self._call_esptool(['erase_region', str(partition.offset), str(partition.size)] + self.esptool_erase_args) self._call_esptool(['erase_region', str(partition.offset), str(partition.size)] + self.esptool_erase_args)
def read_partition(self, partition_id, output): def read_partition(self, partition_id, output):
partition = self.get_partition_info(partition_id) partition = self.get_partition_info(partition_id)
self._call_esptool(['read_flash', str(partition.offset), str(partition.size), output] + self.esptool_read_args) self._call_esptool(['read_flash', str(partition.offset), str(partition.size), output] + self.esptool_read_args)
def write_partition(self, partition_id, input, ignore_readonly=False): def write_partition(self, partition_id, input, ignore_readonly=False): # noqa: A002
partition = self.get_partition_info(partition_id) partition = self.get_partition_info(partition_id)
if partition.readonly and not ignore_readonly: if partition.readonly and not ignore_readonly:
@ -166,8 +167,8 @@ class ParttoolTarget():
self.erase_partition(partition_id) self.erase_partition(partition_id)
with open(input, 'rb') as input_file: with open(input, 'rb') as f:
content_len = len(input_file.read()) content_len = len(f.read())
if content_len > partition.size: if content_len > partition.size:
raise Exception('Input file size exceeds partition size') raise Exception('Input file size exceeds partition size')
@ -175,7 +176,7 @@ class ParttoolTarget():
self._call_esptool(['write_flash', str(partition.offset), input] + self.esptool_write_args) self._call_esptool(['write_flash', str(partition.offset), input] + self.esptool_write_args)
def _write_partition(target, partition_id, input, ignore_readonly=False): def _write_partition(target, partition_id, input, ignore_readonly=False): # noqa: A002
target.write_partition(partition_id, input, ignore_readonly) target.write_partition(partition_id, input, ignore_readonly)
partition = target.get_partition_info(partition_id) partition = target.get_partition_info(partition_id)
status("Written contents of file '{}' at offset 0x{:x}".format(input, partition.offset)) status("Written contents of file '{}' at offset 0x{:x}".format(input, partition.offset))
@ -184,8 +185,11 @@ def _write_partition(target, partition_id, input, ignore_readonly=False):
def _read_partition(target, partition_id, output): def _read_partition(target, partition_id, output):
target.read_partition(partition_id, output) target.read_partition(partition_id, output)
partition = target.get_partition_info(partition_id) partition = target.get_partition_info(partition_id)
status("Read partition '{}' contents from device at offset 0x{:x} to file '{}'" status(
.format(partition.name, partition.offset, output)) "Read partition '{}' contents from device at offset 0x{:x} to file '{}'".format(
partition.name, partition.offset, output
)
)
def _erase_partition(target, partition_id): def _erase_partition(target, partition_id):
@ -213,7 +217,7 @@ def _get_partition_info(target, partition_id, info):
'offset': '0x{:x}'.format(p.offset), 'offset': '0x{:x}'.format(p.offset),
'size': '0x{:x}'.format(p.size), 'size': '0x{:x}'.format(p.size),
'encrypted': '{}'.format(p.encrypted), 'encrypted': '{}'.format(p.encrypted),
'readonly': '{}'.format(p.readonly) 'readonly': '{}'.format(p.readonly),
} }
for i in info: for i in info:
infos += [info_dict[i]] infos += [info_dict[i]]
@ -232,19 +236,29 @@ def main():
parser.add_argument('--esptool-args', help='additional main arguments for esptool', nargs='+') parser.add_argument('--esptool-args', help='additional main arguments for esptool', nargs='+')
parser.add_argument('--esptool-write-args', help='additional subcommand arguments when writing to flash', nargs='+') parser.add_argument('--esptool-write-args', help='additional subcommand arguments when writing to flash', nargs='+')
parser.add_argument('--esptool-read-args', help='additional subcommand arguments when reading flash', nargs='+') parser.add_argument('--esptool-read-args', help='additional subcommand arguments when reading flash', nargs='+')
parser.add_argument('--esptool-erase-args', help='additional subcommand arguments when erasing regions of flash', nargs='+') parser.add_argument(
'--esptool-erase-args', help='additional subcommand arguments when erasing regions of flash', nargs='+'
)
# By default the device attached to the specified port is queried for the partition table. If a partition table file # By default the device attached to the specified port is queried for the partition table. If a partition table file
# is specified, that is used instead. # is specified, that is used instead.
parser.add_argument('--port', '-p', help='port where the target device of the command is connected to; the partition table is sourced from this device \ parser.add_argument(
when the partition table file is not defined') '--port',
'-p',
help='port where the target device of the command is connected to; the partition table is sourced from '
'this device when the partition table file is not defined',
)
parser.add_argument('--baud', '-b', help='baudrate to use', type=int) parser.add_argument('--baud', '-b', help='baudrate to use', type=int)
parser.add_argument('--partition-table-offset', '-o', help='offset to read the partition table from', type=str) parser.add_argument('--partition-table-offset', '-o', help='offset to read the partition table from', type=str)
parser.add_argument('--primary-bootloader-offset', help='offset for primary bootloader', type=str) parser.add_argument('--primary-bootloader-offset', help='offset for primary bootloader', type=str)
parser.add_argument('--recovery-bootloader-offset', help='offset for recovery bootloader', type=str) parser.add_argument('--recovery-bootloader-offset', help='offset for recovery bootloader', type=str)
parser.add_argument('--partition-table-file', '-f', help='file (CSV/binary) to read the partition table from; \ parser.add_argument(
overrides device attached to specified port as the partition table source when defined') '--partition-table-file',
'-f',
help='file (CSV/binary) to read the partition table from; '
'overrides device attached to specified port as the partition table source when defined',
)
partition_selection_parser = argparse.ArgumentParser(add_help=False) partition_selection_parser = argparse.ArgumentParser(add_help=False)
@ -254,31 +268,54 @@ def main():
partition_selection_args.add_argument('--partition-name', '-n', help='name of the partition') partition_selection_args.add_argument('--partition-name', '-n', help='name of the partition')
partition_selection_args.add_argument('--partition-type', '-t', help='type of the partition') partition_selection_args.add_argument('--partition-type', '-t', help='type of the partition')
partition_selection_args.add_argument('--partition-boot-default', '-d', help='select the default boot partition \ partition_selection_args.add_argument(
using the same fallback logic as the IDF bootloader', action='store_true') '--partition-boot-default',
'-d',
help='select the default boot partition \
using the same fallback logic as the IDF bootloader',
action='store_true',
)
partition_selection_parser.add_argument('--partition-subtype', '-s', help='subtype of the partition') partition_selection_parser.add_argument('--partition-subtype', '-s', help='subtype of the partition')
partition_selection_parser.add_argument('--extra-partition-subtypes', help='Extra partition subtype entries', nargs='*') partition_selection_parser.add_argument(
'--extra-partition-subtypes', help='Extra partition subtype entries', nargs='*'
)
subparsers = parser.add_subparsers(dest='operation', help='run parttool -h for additional help') subparsers = parser.add_subparsers(dest='operation', help='run parttool -h for additional help')
# Specify the supported operations # Specify the supported operations
read_part_subparser = subparsers.add_parser('read_partition', help='read partition from device and dump contents into a file', read_part_subparser = subparsers.add_parser(
parents=[partition_selection_parser]) 'read_partition',
help='read partition from device and dump contents into a file',
parents=[partition_selection_parser],
)
read_part_subparser.add_argument('--output', help='file to dump the read partition contents to') read_part_subparser.add_argument('--output', help='file to dump the read partition contents to')
write_part_subparser = subparsers.add_parser('write_partition', help='write contents of a binary file to partition on device', write_part_subparser = subparsers.add_parser(
parents=[partition_selection_parser]) 'write_partition',
help='write contents of a binary file to partition on device',
parents=[partition_selection_parser],
)
write_part_subparser.add_argument('--input', help='file whose contents are to be written to the partition offset') write_part_subparser.add_argument('--input', help='file whose contents are to be written to the partition offset')
write_part_subparser.add_argument('--ignore-readonly', help='Ignore read-only attribute', action='store_true') write_part_subparser.add_argument('--ignore-readonly', help='Ignore read-only attribute', action='store_true')
subparsers.add_parser('erase_partition', help='erase the contents of a partition on the device', parents=[partition_selection_parser]) subparsers.add_parser(
'erase_partition', help='erase the contents of a partition on the device', parents=[partition_selection_parser]
)
print_partition_info_subparser = subparsers.add_parser('get_partition_info', help='get partition information', parents=[partition_selection_parser]) print_partition_info_subparser = subparsers.add_parser(
print_partition_info_subparser.add_argument('--info', help='type of partition information to get', 'get_partition_info', help='get partition information', parents=[partition_selection_parser]
choices=['name', 'type', 'subtype', 'offset', 'size', 'encrypted', 'readonly'], )
default=['offset', 'size'], nargs='+') print_partition_info_subparser.add_argument(
print_partition_info_subparser.add_argument('--part_list', help='Get a list of partitions suitable for a given type', action='store_true') '--info',
help='type of partition information to get',
choices=['name', 'type', 'subtype', 'offset', 'size', 'encrypted', 'readonly'],
default=['offset', 'size'],
nargs='+',
)
print_partition_info_subparser.add_argument(
'--part_list', help='Get a list of partitions suitable for a given type', action='store_true'
)
args = parser.parse_args() args = parser.parse_args()
quiet = args.quiet quiet = args.quiet
@ -299,8 +336,10 @@ def main():
elif args.partition_boot_default: elif args.partition_boot_default:
partition_id = PARTITION_BOOT_DEFAULT partition_id = PARTITION_BOOT_DEFAULT
else: else:
raise RuntimeError('Partition to operate on should be defined using --partition-name OR \ raise RuntimeError(
partition-type,--partition-subtype OR partition-boot-default') 'Partition to operate on should be defined using --partition-name OR \
partition-type,--partition-subtype OR partition-boot-default'
)
# Prepare the device to perform operation on # Prepare the device to perform operation on
target_args = {} target_args = {}
@ -341,18 +380,18 @@ def main():
target = ParttoolTarget(**target_args) target = ParttoolTarget(**target_args)
# Create the operation table and execute the operation # Create the operation table and execute the operation
common_args = {'target':target, 'partition_id':partition_id} common_args = {'target': target, 'partition_id': partition_id}
parttool_ops = { parttool_ops = {
'erase_partition': (_erase_partition, []), 'erase_partition': (_erase_partition, []),
'read_partition': (_read_partition, ['output']), 'read_partition': (_read_partition, ['output']),
'write_partition': (_write_partition, ['input', 'ignore_readonly']), 'write_partition': (_write_partition, ['input', 'ignore_readonly']),
'get_partition_info': (_get_partition_info, ['info']) 'get_partition_info': (_get_partition_info, ['info']),
} }
(op, op_args) = parttool_ops[args.operation] (op, op_args) = parttool_ops[args.operation]
for op_arg in op_args: for op_arg in op_args:
common_args.update({op_arg:vars(args)[op_arg]}) common_args.update({op_arg: vars(args)[op_arg]})
if quiet: if quiet:
# If exceptions occur, suppress and exit quietly # If exceptions occur, suppress and exit quietly

285
components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD # SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
import csv import csv
import io import io
@ -26,38 +26,32 @@ factory,0,2,65536,1048576,
LONGER_BINARY_TABLE = b'' LONGER_BINARY_TABLE = b''
# type 0x00, subtype 0x00, # type 0x00, subtype 0x00,
# offset 64KB, size 1MB # offset 64KB, size 1MB
LONGER_BINARY_TABLE += b'\xAA\x50\x00\x00' + \ LONGER_BINARY_TABLE += (
b'\x00\x00\x01\x00' + \ b'\xaa\x50\x00\x00' + b'\x00\x00\x01\x00' + b'\x00\x00\x10\x00' + b'factory\0' + (b'\0' * 8) + b'\x00\x00\x00\x00'
b'\x00\x00\x10\x00' + \ )
b'factory\0' + (b'\0' * 8) + \
b'\x00\x00\x00\x00'
# type 0x01, subtype 0x20, # type 0x01, subtype 0x20,
# offset 0x110000, size 128KB # offset 0x110000, size 128KB
LONGER_BINARY_TABLE += b'\xAA\x50\x01\x20' + \ LONGER_BINARY_TABLE += (
b'\x00\x00\x11\x00' + \ b'\xaa\x50\x01\x20' + b'\x00\x00\x11\x00' + b'\x00\x02\x00\x00' + b'data' + (b'\0' * 12) + b'\x00\x00\x00\x00'
b'\x00\x02\x00\x00' + \ )
b'data' + (b'\0' * 12) + \
b'\x00\x00\x00\x00'
# type 0x10, subtype 0x00, # type 0x10, subtype 0x00,
# offset 0x150000, size 1MB # offset 0x150000, size 1MB
LONGER_BINARY_TABLE += b'\xAA\x50\x10\x00' + \ LONGER_BINARY_TABLE += (
b'\x00\x00\x15\x00' + \ b'\xaa\x50\x10\x00' + b'\x00\x00\x15\x00' + b'\x00\x10\x00\x00' + b'second' + (b'\0' * 10) + b'\x00\x00\x00\x00'
b'\x00\x10\x00\x00' + \ )
b'second' + (b'\0' * 10) + \
b'\x00\x00\x00\x00'
# MD5 checksum # MD5 checksum
LONGER_BINARY_TABLE += b'\xEB\xEB' + b'\xFF' * 14 LONGER_BINARY_TABLE += b'\xeb\xeb' + b'\xff' * 14
LONGER_BINARY_TABLE += b'\xf9\xbd\x06\x1b\x45\x68\x6f\x86\x57\x1a\x2c\xd5\x2a\x1d\xa6\x5b' LONGER_BINARY_TABLE += b'\xf9\xbd\x06\x1b\x45\x68\x6f\x86\x57\x1a\x2c\xd5\x2a\x1d\xa6\x5b'
# empty partition # empty partition
LONGER_BINARY_TABLE += b'\xFF' * 32 LONGER_BINARY_TABLE += b'\xff' * 32
def _strip_trailing_ffs(binary_table): def _strip_trailing_ffs(binary_table):
""" """
Strip all FFs down to the last 32 bytes (terminating entry) Strip all FFs down to the last 32 bytes (terminating entry)
""" """
while binary_table.endswith(b'\xFF' * 64): while binary_table.endswith(b'\xff' * 64):
binary_table = binary_table[0:len(binary_table) - 32] binary_table = binary_table[0 : len(binary_table) - 32]
return binary_table return binary_table
@ -143,10 +137,10 @@ otherapp, app, factory,, 1M
t = gen_esp32part.PartitionTable.from_csv(csv) t = gen_esp32part.PartitionTable.from_csv(csv)
# 'first' # 'first'
self.assertEqual(t[0].offset, 0x010000) # 64KB boundary as it's an app image self.assertEqual(t[0].offset, 0x010000) # 64KB boundary as it's an app image
self.assertEqual(t[0].size, 0x100000) # Size specified in CSV self.assertEqual(t[0].size, 0x100000) # Size specified in CSV
# 'second' # 'second'
self.assertEqual(t[1].offset, 0x110000) # prev offset+size self.assertEqual(t[1].offset, 0x110000) # prev offset+size
self.assertEqual(t[1].size, 0x100000) # Size specified in CSV self.assertEqual(t[1].size, 0x100000) # Size specified in CSV
# 'minidata' # 'minidata'
self.assertEqual(t[2].offset, 0x210000) self.assertEqual(t[2].offset, 0x210000)
# 'otherapp' # 'otherapp'
@ -162,7 +156,7 @@ second, data, 0x15, , 1M
t.verify() t.verify()
# 'first' # 'first'
self.assertEqual(t[0].offset, 0x10000) # in CSV self.assertEqual(t[0].offset, 0x10000) # in CSV
self.assertEqual(t[0].size, 0x200000 - t[0].offset) # Up to 2M self.assertEqual(t[0].size, 0x200000 - t[0].offset) # Up to 2M
# 'second' # 'second'
self.assertEqual(t[1].offset, 0x200000) # prev offset+size self.assertEqual(t[1].offset, 0x200000) # prev offset+size
@ -188,7 +182,9 @@ first, app, ota_0, 0x200000, 1M
csv = """ csv = """
bootloader, bootloader, primary, N/A, N/A bootloader, bootloader, primary, N/A, N/A
""" """
with self.assertRaisesRegex(gen_esp32part.InputError, 'Primary bootloader offset is not defined. Please use --primary-bootloader-offset'): with self.assertRaisesRegex(
gen_esp32part.InputError, 'Primary bootloader offset is not defined. Please use --primary-bootloader-offset'
):
gen_esp32part.PartitionTable.from_csv(csv) gen_esp32part.PartitionTable.from_csv(csv)
def test_bootloader_and_part_table_partitions(self): def test_bootloader_and_part_table_partitions(self):
@ -234,12 +230,12 @@ first, 0x30, 0xEE, 0x100400, 0x300000
t = gen_esp32part.PartitionTable.from_csv(csv) t = gen_esp32part.PartitionTable.from_csv(csv)
tb = _strip_trailing_ffs(t.to_binary()) tb = _strip_trailing_ffs(t.to_binary())
self.assertEqual(len(tb), 64 + 32) self.assertEqual(len(tb), 64 + 32)
self.assertEqual(b'\xAA\x50', tb[0:2]) # magic self.assertEqual(b'\xaa\x50', tb[0:2]) # magic
self.assertEqual(b'\x30\xee', tb[2:4]) # type, subtype self.assertEqual(b'\x30\xee', tb[2:4]) # type, subtype
eo, es = struct.unpack('<LL', tb[4:12]) eo, es = struct.unpack('<LL', tb[4:12])
self.assertEqual(eo, 0x100400) # offset self.assertEqual(eo, 0x100400) # offset
self.assertEqual(es, 0x300000) # size self.assertEqual(es, 0x300000) # size
self.assertEqual(b'\xEB\xEB' + b'\xFF' * 14, tb[32:48]) self.assertEqual(b'\xeb\xeb' + b'\xff' * 14, tb[32:48])
self.assertEqual(b'\x43\x03\x3f\x33\x40\x87\x57\x51\x69\x83\x9b\x40\x61\xb1\x27\x26', tb[48:64]) self.assertEqual(b'\x43\x03\x3f\x33\x40\x87\x57\x51\x69\x83\x9b\x40\x61\xb1\x27\x26', tb[48:64])
def test_multiple_entries(self): def test_multiple_entries(self):
@ -250,8 +246,8 @@ second,0x31, 0xEF, , 0x100000
t = gen_esp32part.PartitionTable.from_csv(csv) t = gen_esp32part.PartitionTable.from_csv(csv)
tb = _strip_trailing_ffs(t.to_binary()) tb = _strip_trailing_ffs(t.to_binary())
self.assertEqual(len(tb), 96 + 32) self.assertEqual(len(tb), 96 + 32)
self.assertEqual(b'\xAA\x50', tb[0:2]) self.assertEqual(b'\xaa\x50', tb[0:2])
self.assertEqual(b'\xAA\x50', tb[32:34]) self.assertEqual(b'\xaa\x50', tb[32:34])
def test_encrypted_flag(self): def test_encrypted_flag(self):
csv = """ csv = """
@ -278,27 +274,79 @@ storage2, data, undefined, , 12k,
""" """
t = gen_esp32part.PartitionTable.from_csv(csv_txt) t = gen_esp32part.PartitionTable.from_csv(csv_txt)
t.verify() t.verify()
self.assertEqual(t[1].name, 'otadata') self.assertEqual(t[1].name, 'otadata')
self.assertEqual(t[1].type, 1) self.assertEqual(t[1].type, 1)
self.assertEqual(t[1].subtype, 0) self.assertEqual(t[1].subtype, 0)
self.assertEqual(t[6].name, 'storage') self.assertEqual(t[6].name, 'storage')
self.assertEqual(t[6].type, 1) self.assertEqual(t[6].type, 1)
self.assertEqual(t[6].subtype, 0x06) self.assertEqual(t[6].subtype, 0x06)
self.assertEqual(t[7].name, 'storage2') self.assertEqual(t[7].name, 'storage2')
self.assertEqual(t[7].type, 1) self.assertEqual(t[7].type, 1)
self.assertEqual(t[7].subtype, 0x06) self.assertEqual(t[7].subtype, 0x06)
class UTFCodingTests(Py23TestCase):
def test_utf8_bom_csv_file(self):
with open('partitions-utf8-bom.csv', 'rb') as csv_txt:
t, _ = gen_esp32part.PartitionTable.from_file(csv_txt)
t.verify()
self.assertEqual(t[0].name, 'nvs') # 3 BOM bytes are not part of the name
self.assertEqual(t[1].name, 'phy_инит_') # UTF-8 name is preserved
self.assertEqual(t[2].name, 'factory')
with open('partitions.bin', 'rb') as bin_file:
binary_content = bin_file.read()
self.assertEqual(_strip_trailing_ffs(t.to_binary()), _strip_trailing_ffs(binary_content))
def test_utf8_without_bom_csv_file(self):
with open('partitions-utf8_without-bom.csv', 'rb') as csv_txt:
t, _ = gen_esp32part.PartitionTable.from_file(csv_txt)
t.verify()
self.assertEqual(t[0].name, 'nvs')
self.assertEqual(t[1].name, 'phy_инит_') # UTF-8 name is preserved
self.assertEqual(t[2].name, 'factory')
with open('partitions.bin', 'rb') as bin_file:
binary_content = bin_file.read()
self.assertEqual(_strip_trailing_ffs(t.to_binary()), _strip_trailing_ffs(binary_content))
def test_utf8_bin_file(self):
with open('partitions.bin', 'rb') as bin_file:
t, _ = gen_esp32part.PartitionTable.from_file(bin_file)
t.verify()
self.assertEqual(t[0].name, 'nvs')
self.assertEqual(t[1].name, 'phy_инит_') # UTF-8 name is preserved
self.assertEqual(t[2].name, 'factory')
gen = t.to_csv()
self.assertIn('\nnvs,', gen)
self.assertIn('\nphy_инит_,', gen)
self.assertIn('\nfactory,', gen)
def test_utf8_without_bom_bin_file(self):
with open('partitions-utf8-bom.bin', 'rb') as bin_file:
t, _ = gen_esp32part.PartitionTable.from_file(bin_file)
t.verify()
# If the old tool grabbed the BOM bytes for the first name then
# we do not change the name. User needs to fix the CSV file.
self.assertEqual(t[0].name, '\ufeffnvs')
self.assertEqual(t[1].name, 'phy_инит_')
self.assertEqual(t[2].name, 'factory')
gen = t.to_csv()
self.assertIn('\ufeffnvs,', gen)
self.assertIn('\nphy_инит_,', gen)
self.assertIn('\nfactory,', gen)
class BinaryParserTests(Py23TestCase): class BinaryParserTests(Py23TestCase):
def test_parse_one_entry(self): def test_parse_one_entry(self):
# type 0x30, subtype 0xee, # type 0x30, subtype 0xee,
# offset 1MB, size 2MB # offset 1MB, size 2MB
entry = b'\xAA\x50\x30\xee' + \ entry = (
b'\x00\x00\x10\x00' + \ b'\xaa\x50\x30\xee'
b'\x00\x00\x20\x00' + \ + b'\x00\x00\x10\x00'
b'0123456789abc\0\0\0' + \ + b'\x00\x00\x20\x00'
b'\x00\x00\x00\x00' + \ + b'0123456789abc\0\0\0'
b'\xFF' * 32 + b'\x00\x00\x00\x00'
+ b'\xff' * 32
)
# verify that parsing 32 bytes as a table # verify that parsing 32 bytes as a table
# or as a single Definition are the same thing # or as a single Definition are the same thing
t = gen_esp32part.PartitionTable.from_binary(entry) t = gen_esp32part.PartitionTable.from_binary(entry)
@ -312,7 +360,7 @@ class BinaryParserTests(Py23TestCase):
self.assertEqual(e.type, 0x30) self.assertEqual(e.type, 0x30)
self.assertEqual(e.subtype, 0xEE) self.assertEqual(e.subtype, 0xEE)
self.assertEqual(e.offset, 0x100000) self.assertEqual(e.offset, 0x100000)
self.assertEqual(e.size, 0x200000) self.assertEqual(e.size, 0x200000)
self.assertEqual(e.name, '0123456789abc') self.assertEqual(e.name, '0123456789abc')
def test_multiple_entries(self): def test_multiple_entries(self):
@ -333,25 +381,17 @@ class BinaryParserTests(Py23TestCase):
self.assertEqual(round_trip, LONGER_BINARY_TABLE) self.assertEqual(round_trip, LONGER_BINARY_TABLE)
def test_bad_magic(self): def test_bad_magic(self):
bad_magic = b'OHAI' + \ bad_magic = b'OHAI' + b'\x00\x00\x10\x00' + b'\x00\x00\x20\x00' + b'0123456789abc\0\0\0' + b'\x00\x00\x00\x00'
b'\x00\x00\x10\x00' + \
b'\x00\x00\x20\x00' + \
b'0123456789abc\0\0\0' + \
b'\x00\x00\x00\x00'
with self.assertRaisesRegex(gen_esp32part.InputError, 'Invalid magic bytes'): with self.assertRaisesRegex(gen_esp32part.InputError, 'Invalid magic bytes'):
gen_esp32part.PartitionTable.from_binary(bad_magic) gen_esp32part.PartitionTable.from_binary(bad_magic)
def test_bad_length(self): def test_bad_length(self):
bad_length = b'OHAI' + \ bad_length = b'OHAI' + b'\x00\x00\x10\x00' + b'\x00\x00\x20\x00' + b'0123456789'
b'\x00\x00\x10\x00' + \
b'\x00\x00\x20\x00' + \
b'0123456789'
with self.assertRaisesRegex(gen_esp32part.InputError, '32 bytes'): with self.assertRaisesRegex(gen_esp32part.InputError, '32 bytes'):
gen_esp32part.PartitionTable.from_binary(bad_length) gen_esp32part.PartitionTable.from_binary(bad_length)
class CSVOutputTests(Py23TestCase): class CSVOutputTests(Py23TestCase):
def _readcsv(self, source_str): def _readcsv(self, source_str):
return list(csv.reader(source_str.split('\n'))) return list(csv.reader(source_str.split('\n')))
@ -393,7 +433,6 @@ class CSVOutputTests(Py23TestCase):
class CommandLineTests(Py23TestCase): class CommandLineTests(Py23TestCase):
def test_basic_cmdline(self): def test_basic_cmdline(self):
try: try:
binpath = tempfile.mktemp() binpath = tempfile.mktemp()
@ -404,8 +443,9 @@ class CommandLineTests(Py23TestCase):
f.write(LONGER_BINARY_TABLE) f.write(LONGER_BINARY_TABLE)
# run gen_esp32part.py to convert binary file to CSV # run gen_esp32part.py to convert binary file to CSV
output = subprocess.check_output([sys.executable, '../gen_esp32part.py', output = subprocess.check_output(
binpath, csvpath], stderr=subprocess.STDOUT) [sys.executable, '../gen_esp32part.py', binpath, csvpath], stderr=subprocess.STDOUT
)
# reopen the CSV and check the generated binary is identical # reopen the CSV and check the generated binary is identical
self.assertNotIn(b'WARNING', output) self.assertNotIn(b'WARNING', output)
with open(csvpath, 'r') as f: with open(csvpath, 'r') as f:
@ -413,8 +453,9 @@ class CommandLineTests(Py23TestCase):
self.assertEqual(_strip_trailing_ffs(from_csv.to_binary()), LONGER_BINARY_TABLE) self.assertEqual(_strip_trailing_ffs(from_csv.to_binary()), LONGER_BINARY_TABLE)
# run gen_esp32part.py to convert the CSV to binary again # run gen_esp32part.py to convert the CSV to binary again
output = subprocess.check_output([sys.executable, '../gen_esp32part.py', output = subprocess.check_output(
csvpath, binpath], stderr=subprocess.STDOUT) [sys.executable, '../gen_esp32part.py', csvpath, binpath], stderr=subprocess.STDOUT
)
self.assertNotIn(b'WARNING', output) self.assertNotIn(b'WARNING', output)
# assert that file reads back as identical # assert that file reads back as identical
with open(binpath, 'rb') as f: with open(binpath, 'rb') as f:
@ -431,13 +472,14 @@ class CommandLineTests(Py23TestCase):
class VerificationTests(Py23TestCase): class VerificationTests(Py23TestCase):
def _run_genesp32(self, csvcontents, args): def _run_genesp32(self, csvcontents, args):
csvpath = tempfile.mktemp() csvpath = tempfile.mktemp()
with open(csvpath, 'w') as f: with open(csvpath, 'w') as f:
f.write(csvcontents) f.write(csvcontents)
try: try:
output = subprocess.check_output([sys.executable, '../gen_esp32part.py', csvpath] + args, stderr=subprocess.STDOUT) output = subprocess.check_output(
[sys.executable, '../gen_esp32part.py', csvpath] + args, stderr=subprocess.STDOUT
)
return output.strip() return output.strip()
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
return e.output.strip() return e.output.strip()
@ -454,14 +496,15 @@ ota_1, app, ota_1, , 0x100800
return self._run_genesp32(sample_csv, args) return self._run_genesp32(sample_csv, args)
# Failure case 1, incorrect ota_1 partition size # Failure case 1, incorrect ota_1 partition size
self.assertEqual(rge(['-q']), self.assertEqual(rge(['-q']), b'Partition ota_1 invalid: Size 0x100800 is not aligned to 0x1000')
b'Partition ota_1 invalid: Size 0x100800 is not aligned to 0x1000')
# Failure case 2, incorrect ota_0 partition size # Failure case 2, incorrect ota_0 partition size
self.assertEqual(rge(['-q', '--secure', 'v1']), self.assertEqual(
b'Partition ota_0 invalid: Size 0x101000 is not aligned to 0x10000') rge(['-q', '--secure', 'v1']), b'Partition ota_0 invalid: Size 0x101000 is not aligned to 0x10000'
)
# Failure case 3, incorrect ota_1 partition size with Secure Boot V2 # Failure case 3, incorrect ota_1 partition size with Secure Boot V2
self.assertEqual(rge(['-q', '--secure', 'v2']), self.assertEqual(
b'Partition ota_1 invalid: Size 0x100800 is not aligned to 0x1000') rge(['-q', '--secure', 'v2']), b'Partition ota_1 invalid: Size 0x100800 is not aligned to 0x1000'
)
def test_bad_alignment(self): def test_bad_alignment(self):
csv = """ csv = """
@ -490,7 +533,9 @@ nvs, data, nvs, 0x0000, 0x6000,
phy_init, data, phy, , 0x1000, phy_init, data, phy, , 0x1000,
factory, app, factory, , 1M, factory, app, factory, , 1M,
""" """
with self.assertRaisesRegex(gen_esp32part.InputError, r'CSV Error at line 3: Partitions overlap. Partition sets offset 0x0'): with self.assertRaisesRegex(
gen_esp32part.InputError, r'CSV Error at line 3: Partitions overlap. Partition sets offset 0x0'
):
gen_esp32part.PartitionTable.from_csv(csv) gen_esp32part.PartitionTable.from_csv(csv)
def test_only_one_otadata(self): def test_only_one_otadata(self):
@ -561,15 +606,16 @@ factory, app, factory, 0x10000, 20M
class PartToolTests(Py23TestCase): class PartToolTests(Py23TestCase):
def _run_parttool(self, csvcontents, args): def _run_parttool(self, csvcontents, args):
csvpath = tempfile.mktemp() csvpath = tempfile.mktemp()
with open(csvpath, 'w') as f: with open(csvpath, 'w') as f:
f.write(csvcontents) f.write(csvcontents)
try: try:
output = subprocess.check_output([sys.executable, '../parttool.py', '-q', '--partition-table-file', output = subprocess.check_output(
csvpath, 'get_partition_info'] + args, [sys.executable, '../parttool.py', '-q', '--partition-table-file', csvpath, 'get_partition_info']
stderr=subprocess.STDOUT) + args,
stderr=subprocess.STDOUT,
)
self.assertNotIn(b'WARNING', output) self.assertNotIn(b'WARNING', output)
return output.strip() return output.strip()
finally: finally:
@ -590,42 +636,77 @@ nvs_key2, data, nvs_keys, 0x119000, 0x1000, encrypted
def rpt(args): def rpt(args):
return self._run_parttool(csv, args) return self._run_parttool(csv, args)
self.assertEqual(rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'offset']), b'0x9000')
self.assertEqual(rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'size']), b'0x4000')
self.assertEqual(rpt(['--partition-name', 'otadata', '--info', 'offset']), b'0xd000')
self.assertEqual(rpt(['--partition-boot-default', '--info', 'offset']), b'0x10000')
self.assertEqual( self.assertEqual(
rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'offset']), b'0x9000') rpt(
[
'--partition-type',
'data',
'--partition-subtype',
'nvs',
'--info',
'name',
'offset',
'size',
'encrypted',
]
),
b'nvs 0x9000 0x4000 False',
)
self.assertEqual( self.assertEqual(
rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'size']), b'0x4000') rpt(
self.assertEqual( [
rpt(['--partition-name', 'otadata', '--info', 'offset']), b'0xd000') '--partition-type',
self.assertEqual( 'data',
rpt(['--partition-boot-default', '--info', 'offset']), b'0x10000') '--partition-subtype',
self.assertEqual( 'nvs',
rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', 'offset', 'size', 'encrypted']), '--info',
b'nvs 0x9000 0x4000 False') 'name',
self.assertEqual( 'offset',
rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', 'offset', 'size', 'encrypted', '--part_list']), 'size',
b'nvs 0x9000 0x4000 False nvs1_user 0x110000 0x4000 False nvs2_user 0x114000 0x4000 False') 'encrypted',
'--part_list',
]
),
b'nvs 0x9000 0x4000 False nvs1_user 0x110000 0x4000 False nvs2_user 0x114000 0x4000 False',
)
self.assertEqual( self.assertEqual(
rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', '--part_list']), rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', '--part_list']),
b'nvs nvs1_user nvs2_user') b'nvs nvs1_user nvs2_user',
)
self.assertEqual( self.assertEqual(
rpt(['--partition-type', 'data', '--partition-subtype', 'nvs_keys', '--info', 'name', '--part_list']), rpt(['--partition-type', 'data', '--partition-subtype', 'nvs_keys', '--info', 'name', '--part_list']),
b'nvs_key1 nvs_key2') b'nvs_key1 nvs_key2',
)
self.assertEqual(rpt(['--partition-name', 'nvs', '--info', 'encrypted']), b'False')
self.assertEqual(rpt(['--partition-name', 'nvs1_user', '--info', 'encrypted']), b'False')
self.assertEqual(rpt(['--partition-name', 'nvs2_user', '--info', 'encrypted']), b'False')
self.assertEqual(rpt(['--partition-name', 'nvs_key1', '--info', 'encrypted']), b'True')
self.assertEqual(rpt(['--partition-name', 'nvs_key2', '--info', 'encrypted']), b'True')
self.assertEqual( self.assertEqual(
rpt(['--partition-name', 'nvs', '--info', 'encrypted']), b'False') rpt(
[
'--partition-type',
'data',
'--partition-subtype',
'nvs_keys',
'--info',
'name',
'encrypted',
'--part_list',
]
),
b'nvs_key1 True nvs_key2 True',
)
self.assertEqual( self.assertEqual(
rpt(['--partition-name', 'nvs1_user', '--info', 'encrypted']), b'False') rpt(
self.assertEqual( ['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', 'encrypted', '--part_list']
rpt(['--partition-name', 'nvs2_user', '--info', 'encrypted']), b'False') ),
self.assertEqual( b'nvs False nvs1_user False nvs2_user False',
rpt(['--partition-name', 'nvs_key1', '--info', 'encrypted']), b'True') )
self.assertEqual(
rpt(['--partition-name', 'nvs_key2', '--info', 'encrypted']), b'True')
self.assertEqual(
rpt(['--partition-type', 'data', '--partition-subtype', 'nvs_keys', '--info', 'name', 'encrypted', '--part_list']),
b'nvs_key1 True nvs_key2 True')
self.assertEqual(
rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', 'encrypted', '--part_list']),
b'nvs False nvs1_user False nvs2_user False')
def test_fallback(self): def test_fallback(self):
csv = """ csv = """
@ -640,13 +721,13 @@ ota_1, app, ota_1, , 1M
return self._run_parttool(csv, args) return self._run_parttool(csv, args)
self.assertEqual( self.assertEqual(
rpt(['--partition-type', 'app', '--partition-subtype', 'ota_1', '--info', 'offset']), b'0x130000') rpt(['--partition-type', 'app', '--partition-subtype', 'ota_1', '--info', 'offset']), b'0x130000'
self.assertEqual( )
rpt(['--partition-boot-default', '--info', 'offset']), b'0x30000') # ota_0 self.assertEqual(rpt(['--partition-boot-default', '--info', 'offset']), b'0x30000') # ota_0
csv_mod = csv.replace('ota_0', 'ota_2') csv_mod = csv.replace('ota_0', 'ota_2')
self.assertEqual( self.assertEqual(
self._run_parttool(csv_mod, ['--partition-boot-default', '--info', 'offset']), self._run_parttool(csv_mod, ['--partition-boot-default', '--info', 'offset']), b'0x130000'
b'0x130000') # now default is ota_1 ) # now default is ota_1
if __name__ == '__main__': if __name__ == '__main__':

BIN
components/partition_table/test_gen_esp32part_host/partitions-utf8-bom.bin Normal file
View File

Binary file not shown.

3
components/partition_table/test_gen_esp32part_host/partitions-utf8-bom.csv Normal file
View File

@ -0,0 +1,3 @@
nvs, data, nvs, 0x9000, 24K,
phy_инит_, data, phy, 0xf000, 0x1000,
factory, app, factory, 0x10000, 1M,
1 nvs data nvs 0x9000 24K
2 phy_инит_ data phy 0xf000 0x1000
3 factory app factory 0x10000 1M

3
components/partition_table/test_gen_esp32part_host/partitions-utf8_without-bom.csv Normal file
View File

@ -0,0 +1,3 @@
nvs, data, nvs, 0x9000, 24K,
phy_инит_, data, phy, 0xf000, 0x1000,
factory, app, factory, 0x10000, 1M,
1 nvs data nvs 0x9000 24K
2 phy_инит_ data phy 0xf000 0x1000
3 factory app factory 0x10000 1M

BIN
components/partition_table/test_gen_esp32part_host/partitions.bin Normal file
View File

Binary file not shown.