From 2ddcda29b31114ea332150d77565678b58130b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ga=C5=88o?= Date: Wed, 29 Jun 2022 15:37:06 +0200 Subject: [PATCH] fatfsparse.py: Add support for WL --- components/fatfs/fatfs_utils/entry.py | 5 +- components/fatfs/fatfs_utils/fs_object.py | 1 + components/fatfs/fatfs_utils/utils.py | 6 +- components/fatfs/fatfsparse.py | 21 +++- .../fatfs/test_fatfsgen/test_fatfsgen.py | 8 +- .../fatfs/test_fatfsgen/test_fatfsparse.py | 29 +++++ .../fatfs/test_fatfsgen/test_wl_fatfsgen.py | 3 - components/fatfs/wl_fatfsgen.py | 82 ++++++++---- .../storage/fatfsgen/main/Kconfig.projbuild | 7 ++ .../fatfsgen/main/fatfsgen_example_main.c | 16 ++- .../fatfsgen/pytest_fatfsgen_example.py | 117 ++++++++++++++++++ .../sdkconfig.ci.test_read_only_partition_gen | 1 + ...ci.test_read_only_partition_gen_default_dt | 1 + ...kconfig.ci.test_read_only_partition_gen_ln | 1 + ...test_read_only_partition_gen_ln_default_dt | 1 + ...sdkconfig.ci.test_read_write_partition_gen | 1 + ...i.test_read_write_partition_gen_default_dt | 1 + ...config.ci.test_read_write_partition_gen_ln | 1 + ...est_read_write_partition_gen_ln_default_dt | 1 + tools/ci/executable-list.txt | 1 + 20 files changed, 260 insertions(+), 44 deletions(-) mode change 100644 => 100755 components/fatfs/fatfsparse.py diff --git a/components/fatfs/fatfs_utils/entry.py b/components/fatfs/fatfs_utils/entry.py index 8dcc99f5f7..c33bd9e834 100644 --- a/components/fatfs/fatfs_utils/entry.py +++ b/components/fatfs/fatfs_utils/entry.py @@ -47,7 +47,7 @@ class Entry: 'DIR_Name' / PaddedString(MAX_NAME_SIZE, SHORT_NAMES_ENCODING), 'DIR_Name_ext' / PaddedString(MAX_EXT_SIZE, SHORT_NAMES_ENCODING), 'DIR_Attr' / Int8ul, - 'DIR_NTRes' / Const(EMPTY_BYTE), + 'DIR_NTRes' / Int8ul, # this tagged for lfn (0x00 for lfn prefix, 0x18 for short name in lfn) 'DIR_CrtTimeTenth' / Const(EMPTY_BYTE), # ignored by esp-idf fatfs library 'DIR_CrtTime' / Int16ul, # ignored by esp-idf fatfs library 'DIR_CrtDate' / Int16ul, # ignored by esp-idf fatfs library @@ -159,6 +159,7 @@ class Entry: lfn_order: int = SHORT_ENTRY, lfn_names: Optional[List[bytes]] = None, lfn_checksum_: int = 0, + fits_short: bool = False, lfn_is_last: bool = False) -> None: """ :param first_cluster_id: id of the first data cluster for given entry @@ -172,6 +173,7 @@ class Entry: :param lfn_names: if the entry is dedicated for long names the lfn_names contains LDIR_Name1, LDIR_Name2 and LDIR_Name3 in this order :param lfn_checksum_: use only for long file names, checksum calculated lfn_checksum function + :param fits_short: determines if the name fits in 8.3 filename :param lfn_is_last: determines if the long file name entry is holds last part of the name, thus its address is first in the physical order :returns: None @@ -213,6 +215,7 @@ class Entry: DIR_Name=pad_string(object_name, size=MAX_NAME_SIZE), DIR_Name_ext=pad_string(object_extension, size=MAX_EXT_SIZE), DIR_Attr=entity_type, + DIR_NTRes=0x00 if (not self.fatfs_state.long_names_enabled) or (not fits_short) else 0x18, DIR_FstClusLO=first_cluster_id, DIR_FileSize=size, DIR_CrtDate=date_entry_, # ignored by esp-idf fatfs library diff --git a/components/fatfs/fatfs_utils/fs_object.py b/components/fatfs/fatfs_utils/fs_object.py index 9107d14bfa..9d68eb27b5 100644 --- a/components/fatfs/fatfs_utils/fs_object.py +++ b/components/fatfs/fatfs_utils/fs_object.py @@ -260,6 +260,7 @@ class Directory: entity_extension=extension, date=fatfs_date_, time=fatfs_time_, + fits_short=True, entity_type=entity_type) return free_cluster, free_entry, target_dir return self.allocate_long_name_object(free_entry=free_entry, diff --git a/components/fatfs/fatfs_utils/utils.py b/components/fatfs/fatfs_utils/utils.py index 8cf57bf4a3..aad5731ca0 100644 --- a/components/fatfs/fatfs_utils/utils.py +++ b/components/fatfs/fatfs_utils/utils.py @@ -42,7 +42,7 @@ FATFS_SECONDS_GRANULARITY: int = 2 LONG_NAMES_ENCODING: str = 'utf-16' SHORT_NAMES_ENCODING: str = 'utf-8' -ALLOWED_SECTOR_SIZES: List[int] = [512, 1024, 2048, 4096] +ALLOWED_SECTOR_SIZES: List[int] = [4096] ALLOWED_SECTORS_PER_CLUSTER: List[int] = [1, 2, 4, 8, 16, 32, 64, 128] @@ -181,7 +181,7 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace: parser.add_argument('--fat_type', default=0, type=int, - choices=[12, 16, 0], + choices=[FAT12, FAT16, 0], help=""" Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic calculation using cluster size and partition size. @@ -269,3 +269,5 @@ class FATDefaults: VERSION: int = 2 TEMP_BUFFER_SIZE: int = 32 UPDATE_RATE: int = 16 + WR_SIZE: int = 16 + WL_SECTOR_SIZE: int = 4096 diff --git a/components/fatfs/fatfsparse.py b/components/fatfs/fatfsparse.py old mode 100644 new mode 100755 index 656f12daf9..ad8195798f --- a/components/fatfs/fatfsparse.py +++ b/components/fatfs/fatfsparse.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse @@ -9,6 +10,7 @@ from fatfs_utils.entry import Entry from fatfs_utils.fat import FAT from fatfs_utils.fatfs_state import BootSectorState from fatfs_utils.utils import FULL_BYTE, LONG_NAMES_ENCODING, PAD_CHAR, FATDefaults, lfn_checksum, read_filesystem +from wl_fatfsgen import remove_wl def build_file_name(name1: bytes, name2: bytes, name3: bytes) -> str: @@ -42,7 +44,7 @@ def traverse_folder_tree(directory_bytes_: bytes, name: str, state_: BootSectorState, fat_: FAT, - binary_array_: bytearray) -> None: + binary_array_: bytes) -> None: os.makedirs(name) assert len(directory_bytes_) % FATDefaults.ENTRY_SIZE == 0 @@ -89,9 +91,26 @@ if __name__ == '__main__': argument_parser.add_argument('--long-name-support', action='store_true', help='Set flag to enable long names support.') + + argument_parser.add_argument('--wear-leveling', + action='store_true', + help='Set flag to parse an image encoded using wear levelling.') + args = argument_parser.parse_args() fs = read_filesystem(args.input_image) + + # An algorithm for removing wear levelling: + # 1. find an remove dummy sector: + # a) dummy sector is at the position defined by the number of records in the state sector + # b) dummy may not be placed in state nor cfg sectors + # c) first (boot) sector position (boot_s_pos) is calculated using value of move count + # boot_s_pos = - mc + # 2. remove state sectors (trivial) + # 3. remove cfg sector (trivial) + # 4. valid fs is then old_fs[-mc:] + old_fs[:-mc] + if args.wear_leveling: + fs = remove_wl(fs) boot_sector_ = BootSector() boot_sector_.parse_boot_sector(fs) fat = FAT(boot_sector_.boot_sector_state, init_=False) diff --git a/components/fatfs/test_fatfsgen/test_fatfsgen.py b/components/fatfs/test_fatfsgen/test_fatfsgen.py index ab338dfc56..7ef1355c42 100755 --- a/components/fatfs/test_fatfsgen/test_fatfsgen.py +++ b/components/fatfs/test_fatfsgen/test_fatfsgen.py @@ -294,7 +294,7 @@ class FatFSGen(unittest.TestCase): fatfs.create_file('HELLO', extension='TXT') fatfs.write_filesystem(CFG['output_file']) file_system = read_filesystem(CFG['output_file']) - self.assertEqual(file_system[0x2000: 0x2019], b'HELLO TXT \x00\x00\x00\x00!\x00!\x00\x00\x00\x00\x00!') + self.assertEqual(file_system[0x2000: 0x2019], b'HELLO TXT \x18\x00\x00\x00!\x00!\x00\x00\x00\x00\x00!') def test_lfn_short_name(self) -> None: fatfs = fatfsgen.FATFS(long_names_enabled=True) @@ -302,7 +302,7 @@ class FatFSGen(unittest.TestCase): fatfs.write_content(path_from_root=['HELLO.TXT'], content=b'this is a test') fatfs.write_filesystem(CFG['output_file']) file_system = read_filesystem(CFG['output_file']) - self.assertEqual(file_system[0x2000: 0x2010], b'HELLO TXT \x00\x00\x00\x00') + self.assertEqual(file_system[0x2000: 0x2010], b'HELLO TXT \x18\x00\x00\x00') self.assertEqual(file_system[0x2010: 0x2020], b'!\x00!\x00\x00\x00\x00\x00!\x00\x02\x00\x0e\x00\x00\x00') self.assertEqual(file_system[0x6000: 0x6010], b'this is a test\x00\x00') @@ -367,7 +367,7 @@ class FatFSGen(unittest.TestCase): self.assertEqual(file_system[0x6012: 0x6020], b'!\x00\x00\x00\x00\x00!\x00\x02\x00\x00\x00\x00\x00') self.assertEqual(file_system[0x6020: 0x6030], b'.. \x10\x00\x00\x00\x00') self.assertEqual(file_system[0x6030: 0x6040], b'!\x00!\x00\x00\x00\x00\x00!\x00\x01\x00\x00\x00\x00\x00') - self.assertEqual(file_system[0x6040: 0x6050], b'HELLO TXT \x00\x00\x00\x00') + self.assertEqual(file_system[0x6040: 0x6050], b'HELLO TXT \x18\x00\x00\x00') self.assertEqual(file_system[0x6050: 0x6060], b'!\x00!\x00\x00\x00\x00\x00!\x00\x03\x00\x00\x00\x00\x00') def test_lfn_nested_long_empty(self) -> None: @@ -410,7 +410,7 @@ class FatFSGen(unittest.TestCase): self.assertEqual(file_system[0x6012: 0x6020], b'!\x00\x00\x00\x00\x00!\x00\x02\x00\x00\x00\x00\x00') self.assertEqual(file_system[0x6020: 0x6030], b'.. \x10\x00\x00\x00\x00') self.assertEqual(file_system[0x6030: 0x6040], b'!\x00!\x00\x00\x00\x00\x00!\x00\x01\x00\x00\x00\x00\x00') - self.assertEqual(file_system[0x6040: 0x6050], b'HELLO TXT \x00\x00\x00\x00') + self.assertEqual(file_system[0x6040: 0x6050], b'HELLO TXT \x18\x00\x00\x00') self.assertEqual(file_system[0x6050: 0x6060], b'!\x00!\x00\x00\x00\x00\x00!\x00\x03\x00\x0e\x00\x00\x00') self.assertEqual(file_system[0x7000: 0x7010], b'this is a test\x00\x00') diff --git a/components/fatfs/test_fatfsgen/test_fatfsparse.py b/components/fatfs/test_fatfsgen/test_fatfsparse.py index caa060579f..648926ca44 100755 --- a/components/fatfs/test_fatfsgen/test_fatfsparse.py +++ b/components/fatfs/test_fatfsgen/test_fatfsparse.py @@ -23,6 +23,7 @@ class FatFSGen(unittest.TestCase): shutil.rmtree('output_data', ignore_errors=True) shutil.rmtree('Espressif', ignore_errors=True) shutil.rmtree('testf', ignore_errors=True) + shutil.rmtree('testf_wl', ignore_errors=True) if os.path.exists('fatfs_image.img'): os.remove('fatfs_image.img') @@ -138,6 +139,15 @@ class FatFSGen(unittest.TestCase): ], stderr=STDOUT) run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT) assert compare_folders('testf', 'Espressif') + shutil.rmtree('Espressif', ignore_errors=True) + + run([ + 'python', + f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}', + 'testf' + ], stderr=STDOUT) + run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT) + assert compare_folders('testf', 'Espressif') def test_e2e_deeper(self) -> None: folder_ = { @@ -159,6 +169,7 @@ class FatFSGen(unittest.TestCase): folder_ ] } + generate_local_folder_structure(struct_, path_='.') run([ 'python', @@ -167,6 +178,15 @@ class FatFSGen(unittest.TestCase): ], stderr=STDOUT) run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT) assert compare_folders('testf', 'Espressif') + shutil.rmtree('Espressif', ignore_errors=True) + + run([ + 'python', + f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}', + 'testf' + ], stderr=STDOUT) + run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT) + assert compare_folders('testf', 'Espressif') def test_e2e_deeper_large(self) -> None: folder_ = { @@ -214,6 +234,15 @@ class FatFSGen(unittest.TestCase): ], stderr=STDOUT) run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT) assert compare_folders('testf', 'Espressif') + shutil.rmtree('Espressif', ignore_errors=True) + + run([ + 'python', + f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}', + 'testf' + ], stderr=STDOUT) + run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT) + assert compare_folders('testf', 'Espressif') def test_e2e_very_deep(self) -> None: folder_ = { diff --git a/components/fatfs/test_fatfsgen/test_wl_fatfsgen.py b/components/fatfs/test_fatfsgen/test_wl_fatfsgen.py index 9234d4df29..e39d7cd8c4 100755 --- a/components/fatfs/test_fatfsgen/test_wl_fatfsgen.py +++ b/components/fatfs/test_fatfsgen/test_wl_fatfsgen.py @@ -105,9 +105,6 @@ class WLFatFSGen(unittest.TestCase): fatfs.wl_create_directory('TESTFOLD') self.assertRaises(WLNotInitialized, fatfs.wl_write_filesystem, CFG['output_file']) - def test_wrong_sector_size(self) -> None: - self.assertRaises(NotImplementedError, wl_fatfsgen.WLFATFS, sector_size=1024) - def test_e2e_deep_folder_into_image_ext(self) -> None: fatfs = wl_fatfsgen.WLFATFS() fatfs.wl_generate(CFG['test_dir2']) diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py index c36c23c49c..2c9a41bbb9 100755 --- a/components/fatfs/wl_fatfsgen.py +++ b/components/fatfs/wl_fatfsgen.py @@ -11,6 +11,41 @@ from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, genera from fatfsgen import FATFS +def remove_wl(binary_image: bytes) -> bytes: + partition_size: int = len(binary_image) + total_sectors: int = partition_size // FATDefaults.WL_SECTOR_SIZE + wl_state_size: int = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * total_sectors + wl_state_sectors_cnt: int = (wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE + wl_state_total_size: int = wl_state_sectors_cnt * FATDefaults.WL_SECTOR_SIZE + wl_sectors_size: int = (wl_state_sectors_cnt + * FATDefaults.WL_SECTOR_SIZE + * WLFATFS.WL_STATE_COPY_COUNT + + FATDefaults.WL_SECTOR_SIZE) + + correct_wl_configuration = binary_image[-wl_sectors_size:] + + data_ = WLFATFS.WL_STATE_T_DATA.parse(correct_wl_configuration[:WLFATFS.WL_STATE_HEADER_SIZE]) + + total_records = 0 + # iterating over records field of the first copy of the state sector + for i in range(WLFATFS.WL_STATE_HEADER_SIZE, wl_state_total_size, WLFATFS.WL_STATE_RECORD_SIZE): + if correct_wl_configuration[i:i + WLFATFS.WL_STATE_RECORD_SIZE] != WLFATFS.WL_STATE_RECORD_SIZE * b'\xff': + total_records += 1 + else: + break + before_dummy = binary_image[:total_records * FATDefaults.WL_SECTOR_SIZE] + after_dummy = binary_image[total_records * FATDefaults.WL_SECTOR_SIZE + FATDefaults.WL_SECTOR_SIZE:] + new_image: bytes = before_dummy + after_dummy + + # remove wl sectors + new_image = new_image[:len(new_image) - (FATDefaults.WL_SECTOR_SIZE + 2 * wl_state_total_size)] + + # reorder to preserve original order + new_image = (new_image[-data_['move_count'] * FATDefaults.WL_SECTOR_SIZE:] + + new_image[:-data_['move_count'] * FATDefaults.WL_SECTOR_SIZE]) + return new_image + + class WLFATFS: # pylint: disable=too-many-instance-attributes WL_CFG_SECTORS_COUNT = 1 @@ -18,7 +53,7 @@ class WLFATFS: WL_CONFIG_HEADER_SIZE = 48 WL_STATE_RECORD_SIZE = 16 WL_STATE_HEADER_SIZE = 64 - WL_STATE_COPY_COUNT = 2 + WL_STATE_COPY_COUNT = 2 # always 2 copies for power failure safety WL_SECTOR_SIZE = 0x1000 WL_STATE_T_DATA = Struct( @@ -37,7 +72,7 @@ class WLFATFS: 'start_addr' / Int32ul, 'full_mem_size' / Int32ul, 'page_size' / Int32ul, - 'sector_size' / Int32ul, + 'sector_size' / Int32ul, # always 4096 for the types of NOR flash supported by ESP-IDF! 'updaterate' / Int32ul, 'wr_size' / Int32ul, 'version' / Int32ul, @@ -50,7 +85,6 @@ class WLFATFS: reserved_sectors_cnt: int = FATDefaults.RESERVED_SECTORS_COUNT, fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT, sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER, - sector_size: int = FATDefaults.SECTOR_SIZE, sectors_per_fat: int = FATDefaults.SECTORS_PER_FAT, explicit_fat_type: int = None, hidden_sectors: int = FATDefaults.HIDDEN_SECTORS, @@ -63,28 +97,22 @@ class WLFATFS: use_default_datetime: bool = True, version: int = FATDefaults.VERSION, temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE, - update_rate: int = FATDefaults.UPDATE_RATE, device_id: int = None, root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT, media_type: int = FATDefaults.MEDIA_TYPE) -> None: - if sector_size != WLFATFS.WL_SECTOR_SIZE: - raise NotImplementedError(f'The only supported sector size is currently {WLFATFS.WL_SECTOR_SIZE}') - self._initialized = False - self.sector_size = sector_size self._version = version self._temp_buff_size = temp_buff_size self._device_id = device_id - self._update_rate = update_rate self.partition_size = size - self.total_sectors = self.partition_size // self.sector_size + self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors # determine the number of required sectors (roundup to sector size) - self.wl_state_sectors = (self.wl_state_size + self.sector_size - 1) // self.sector_size + self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE - self.boot_sector_start = self.sector_size # shift by one "dummy" sector - self.fat_table_start = self.boot_sector_start + reserved_sectors_cnt * self.sector_size + self.boot_sector_start = FATDefaults.WL_SECTOR_SIZE # shift by one "dummy" sector + self.fat_table_start = self.boot_sector_start + reserved_sectors_cnt * FATDefaults.WL_SECTOR_SIZE wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT + self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT) @@ -92,11 +120,11 @@ class WLFATFS: self.plain_fatfs = FATFS( explicit_fat_type=explicit_fat_type, - size=self.plain_fat_sectors * self.sector_size, + size=self.plain_fat_sectors * FATDefaults.WL_SECTOR_SIZE, reserved_sectors_cnt=reserved_sectors_cnt, fat_tables_cnt=fat_tables_cnt, sectors_per_cluster=sectors_per_cluster, - sector_size=sector_size, + sector_size=FATDefaults.WL_SECTOR_SIZE, sectors_per_fat=sectors_per_fat, root_entry_count=root_entry_count, hidden_sectors=hidden_sectors, @@ -121,17 +149,17 @@ class WLFATFS: self._initialized = True def _add_dummy_sector(self) -> None: - self.fatfs_binary_image = self.sector_size * FULL_BYTE + self.fatfs_binary_image + self.fatfs_binary_image = FATDefaults.WL_SECTOR_SIZE * FULL_BYTE + self.fatfs_binary_image def _add_config_sector(self) -> None: wl_config_data = WLFATFS.WL_CONFIG_T_DATA.build( dict( start_addr=0, full_mem_size=self.partition_size, - page_size=self.sector_size, - sector_size=self.sector_size, - updaterate=self._update_rate, - wr_size=16, + page_size=FATDefaults.WL_SECTOR_SIZE, # equal to sector size (always 4096) + sector_size=FATDefaults.WL_SECTOR_SIZE, + updaterate=FATDefaults.UPDATE_RATE, + wr_size=FATDefaults.WR_SIZE, version=self._version, temp_buff_size=self._temp_buff_size ) @@ -143,7 +171,8 @@ class WLFATFS: # adding three 4 byte zeros to align the structure wl_config = wl_config_data + wl_config_crc + Int32ul.build(0) + Int32ul.build(0) + Int32ul.build(0) - self.fatfs_binary_image += (wl_config + (self.sector_size - WLFATFS.WL_CONFIG_HEADER_SIZE) * FULL_BYTE) + self.fatfs_binary_image += ( + wl_config + (FATDefaults.WL_SECTOR_SIZE - WLFATFS.WL_CONFIG_HEADER_SIZE) * FULL_BYTE) def _add_state_sectors(self) -> None: wl_state_data = WLFATFS.WL_STATE_T_DATA.build( @@ -152,8 +181,8 @@ class WLFATFS: max_pos=self.plain_fat_sectors + WLFATFS.WL_DUMMY_SECTORS_COUNT, move_count=0, access_count=0, - max_count=self._update_rate, - block_size=self.sector_size, + max_count=FATDefaults.UPDATE_RATE, + block_size=FATDefaults.WL_SECTOR_SIZE, # equal to page size, thus equal to wl sector size (4096) version=self._version, device_id=self._device_id or generate_4bytes_random(), ) @@ -161,9 +190,9 @@ class WLFATFS: crc = crc32(list(wl_state_data), UINT32_MAX) wl_state_crc = Int32ul.build(crc) wl_state = wl_state_data + wl_state_crc - wl_state_sector_padding: bytes = (self.sector_size - WLFATFS.WL_STATE_HEADER_SIZE) * FULL_BYTE + wl_state_sector_padding: bytes = (FATDefaults.WL_SECTOR_SIZE - WLFATFS.WL_STATE_HEADER_SIZE) * FULL_BYTE wl_state_sector: bytes = ( - wl_state + wl_state_sector_padding + (self.wl_state_sectors - 1) * self.sector_size * FULL_BYTE + wl_state + wl_state_sector_padding + (self.wl_state_sectors - 1) * FATDefaults.WL_SECTOR_SIZE * FULL_BYTE ) self.fatfs_binary_image += (WLFATFS.WL_STATE_COPY_COUNT * wl_state_sector) @@ -193,8 +222,7 @@ if __name__ == '__main__': desc = 'Create a FAT filesystem with support for wear levelling and populate it with directory content' args = get_args_for_partition_generator(desc) - wl_fatfs = WLFATFS(sector_size=args.sector_size, - sectors_per_cluster=args.sectors_per_cluster, + wl_fatfs = WLFATFS(sectors_per_cluster=args.sectors_per_cluster, size=args.partition_size, root_entry_count=args.root_entry_count, explicit_fat_type=args.fat_type, diff --git a/examples/storage/fatfsgen/main/Kconfig.projbuild b/examples/storage/fatfsgen/main/Kconfig.projbuild index 31c7c5f3d9..b715c917b8 100644 --- a/examples/storage/fatfsgen/main/Kconfig.projbuild +++ b/examples/storage/fatfsgen/main/Kconfig.projbuild @@ -7,6 +7,13 @@ menu "Example Configuration" If read-only mode is set, the generated fatfs image will be raw (without wear levelling support). Otherwise it will support wear levelling that enables read-write mounting. + config EXAMPLE_FATFS_WRITE_COUNT + int "Number of volumes" + default 1 + range 1 600 + help + Number of writes to the file (for testing purposes). + config EXAMPLE_FATFS_DEFAULT_DATETIME bool "Default modification date and time for generated FATFS image" default n diff --git a/examples/storage/fatfsgen/main/fatfsgen_example_main.c b/examples/storage/fatfsgen/main/fatfsgen_example_main.c index 21ead9e0a3..491622c4e4 100644 --- a/examples/storage/fatfsgen/main/fatfsgen_example_main.c +++ b/examples/storage/fatfsgen/main/fatfsgen_example_main.c @@ -66,13 +66,17 @@ void app_main(void) if (!EXAMPLE_FATFS_MODE_READ_ONLY){ // Open file for reading ESP_LOGI(TAG, "Opening file"); - FILE *f = fopen(device_filename, "wb"); - if (f == NULL) { - ESP_LOGE(TAG, "Failed to open file for writing"); - return; + FILE *f; + for(int i = 0; i < CONFIG_EXAMPLE_FATFS_WRITE_COUNT; i++){ + f = fopen(device_filename, "wb"); + if (f == NULL) { + ESP_LOGE(TAG, "Failed to open file for writing"); + return; + } + fprintf(f, "This is written by the device"); + fclose(f); } - fprintf(f, "This is written by the device"); - fclose(f); + ESP_LOGI(TAG, "File written"); // Open file for reading diff --git a/examples/storage/fatfsgen/pytest_fatfsgen_example.py b/examples/storage/fatfsgen/pytest_fatfsgen_example.py index a6778d249b..c19201c5a3 100644 --- a/examples/storage/fatfsgen/pytest_fatfsgen_example.py +++ b/examples/storage/fatfsgen/pytest_fatfsgen_example.py @@ -1,13 +1,55 @@ # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 +import os import re +import shutil +import sys from datetime import datetime +from subprocess import STDOUT, run from typing import List import pytest from pytest_embedded import Dut +idf_path = os.environ['IDF_PATH'] # get value of IDF_PATH from environment +parttool_dir = os.path.join(idf_path, 'components', 'partition_table') + + +sys.path.append(parttool_dir) +from parttool import PartitionName, ParttoolTarget # noqa E402 # pylint: disable=C0413 + + +def file_(x: str, content_: str = 'hey this is a test') -> dict: + return { + 'type': 'file', + 'name': x, + 'content': content_ + } + + +def generate_local_folder_structure(structure_: dict, path_: str) -> None: + if structure_['type'] == 'folder': + new_path_ = os.path.join(path_, structure_['name']) + os.makedirs(new_path_) + for item_ in structure_['content']: + generate_local_folder_structure(item_, new_path_) + else: + new_path_ = os.path.join(path_, structure_['name']) + with open(new_path_, 'w') as f_: + f_.write(structure_['content']) + + +def compare_folders(fp1: str, fp2: str) -> bool: + if os.path.isdir(fp1) != os.path.isdir(fp2): + return False + if os.path.isdir(fp1): + if set(os.listdir(fp1)) != set(os.listdir(fp2)): + return False + return all([compare_folders(os.path.join(fp1, path_), os.path.join(fp2, path_)) for path_ in os.listdir(fp1)]) + with open(fp1, 'rb') as f1_, open(fp2, 'rb') as f2_: + return f1_.read() == f2_.read() + # Example_GENERIC @pytest.mark.esp32 @@ -51,6 +93,7 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None: filename_sn = 'sub/test.txt' date_modified = datetime.today() date_default = datetime(1980, 1, 1) + fatfs_parser_path = os.path.join(idf_path, 'components', 'fatfs', 'fatfsparse.py') if config in ['test_read_write_partition_gen', 'test_read_write_partition_gen_default_dt']: filename = filename_sn @@ -68,6 +111,32 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None: 'example: Unmounting FAT filesystem', 'example: Done'], timeout) + target = ParttoolTarget(dut.port) + target.read_partition(PartitionName('storage'), 'temp.img') + run(['python', fatfs_parser_path, '--wear-leveling', 'temp.img'], stderr=STDOUT) + folder_ = { + 'type': 'folder', + 'name': 'SUB', + 'content': [ + file_('TEST.TXT', content_='this is test\n'), + ] + } + struct_: dict = { + 'type': 'folder', + 'name': 'testf', + 'content': [ + file_('HELLO.TXT', content_='This is generated on the host\n'), + file_('INNER.TXT', content_='This is written by the device'), + folder_ + ] + } + generate_local_folder_structure(struct_, path_='.') + try: + assert compare_folders('testf', 'Espressif') + finally: + shutil.rmtree('Espressif', ignore_errors=True) + shutil.rmtree('testf', ignore_errors=True) + elif config in ['test_read_only_partition_gen', 'test_read_only_partition_gen_default_dt']: filename = filename_sn filename_expected = f'/spiflash/{filename}' @@ -79,6 +148,30 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None: expect_all(['example: Read from file: \'this is test\'', 'example: Unmounting FAT filesystem', 'example: Done'], timeout) + target = ParttoolTarget(dut.port) + target.read_partition(PartitionName('storage'), 'temp.img') + run(['python', fatfs_parser_path, '--long-name-support', 'temp.img'], stderr=STDOUT) + folder_ = { + 'type': 'folder', + 'name': 'sublongnames', + 'content': [ + file_('testlongfilenames.txt', content_='this is test; long name it has\n'), + ] + } + struct_ = { + 'type': 'folder', + 'name': 'testf', + 'content': [ + file_('hellolongname.txt', content_='This is generated on the host; long name it has\n'), + folder_ + ] + } + generate_local_folder_structure(struct_, path_='.') + try: + assert compare_folders('testf', 'Espressif') + finally: + shutil.rmtree('Espressif', ignore_errors=True) + shutil.rmtree('testf', ignore_errors=True) elif config in ['test_read_write_partition_gen_ln', 'test_read_write_partition_gen_ln_default_dt']: filename = filename_ln @@ -107,3 +200,27 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None: expect_all(['example: Read from file: \'this is test; long name it has\'', 'example: Unmounting FAT filesystem', 'example: Done'], timeout) + target = ParttoolTarget(dut.port) + target.read_partition(PartitionName('storage'), 'temp.img') + run(['python', fatfs_parser_path, 'temp.img'], stderr=STDOUT) + folder_ = { + 'type': 'folder', + 'name': 'SUB', + 'content': [ + file_('TEST.TXT', content_='this is test\n'), + ] + } + struct_ = { + 'type': 'folder', + 'name': 'testf', + 'content': [ + file_('HELLO.TXT', content_='This is generated on the host\n'), + folder_ + ] + } + generate_local_folder_structure(struct_, path_='.') + try: + assert compare_folders('testf', 'Espressif') + finally: + shutil.rmtree('Espressif', ignore_errors=True) + shutil.rmtree('testf', ignore_errors=True) diff --git a/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen b/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen index 9642cc9005..da9aee7c26 100644 --- a/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen +++ b/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen @@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y CONFIG_FATFS_LFN_HEAP=n CONFIG_FATFS_LFN_STACK=n CONFIG_FATFS_LFN_NONE=y +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_default_dt b/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_default_dt index 007bef1e37..f356d88bdc 100644 --- a/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_default_dt +++ b/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_default_dt @@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=n CONFIG_FATFS_LFN_STACK=n CONFIG_FATFS_LFN_NONE=y CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln b/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln index 5122126dcb..279308a0bb 100644 --- a/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln +++ b/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln @@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y CONFIG_FATFS_LFN_HEAP=y CONFIG_FATFS_LFN_NONE=n CONFIG_FATFS_LFN_STACK=n +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln_default_dt b/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln_default_dt index 1fdc304e4f..32d2c7b8d2 100644 --- a/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln_default_dt +++ b/examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln_default_dt @@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=y CONFIG_FATFS_LFN_NONE=n CONFIG_FATFS_LFN_STACK=n CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen b/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen index c9c707f7bb..7ac9efc449 100644 --- a/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen +++ b/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen @@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n CONFIG_FATFS_LFN_HEAP=n CONFIG_FATFS_LFN_STACK=n CONFIG_FATFS_LFN_NONE=y +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_default_dt b/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_default_dt index f0135f8216..4b8a2d1e1e 100644 --- a/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_default_dt +++ b/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_default_dt @@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=n CONFIG_FATFS_LFN_STACK=n CONFIG_FATFS_LFN_NONE=y CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln b/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln index 33a0ccfad3..eee1bd998f 100644 --- a/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln +++ b/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln @@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n CONFIG_FATFS_LFN_HEAP=y CONFIG_FATFS_LFN_NONE=n CONFIG_FATFS_LFN_STACK=n +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln_default_dt b/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln_default_dt index ea48850fb4..e1a9b879f1 100644 --- a/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln_default_dt +++ b/examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln_default_dt @@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=y CONFIG_FATFS_LFN_NONE=n CONFIG_FATFS_LFN_STACK=n CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index de0bd8700d..5d823428e7 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -8,6 +8,7 @@ components/espcoredump/test/test_espcoredump.py components/espcoredump/test/test_espcoredump.sh components/espcoredump/test_apps/build_espcoredump.sh components/fatfs/fatfsgen.py +components/fatfs/fatfsparse.py components/fatfs/test_fatfsgen/test_fatfsgen.py components/fatfs/test_fatfsgen/test_fatfsparse.py components/fatfs/test_fatfsgen/test_wl_fatfsgen.py