Merge branch 'feature/add-wl-support-fatfsparse' into 'master'

fatfsparse.py: Add support for WL

Closes IDF-4994 and IDF-5522

See merge request espressif/esp-idf!18760

components/fatfs/fatfs_utils/entry.py

@@ -47,7 +47,7 @@ class Entry:
         'DIR_Name' / PaddedString(MAX_NAME_SIZE, SHORT_NAMES_ENCODING),
         'DIR_Name_ext' / PaddedString(MAX_EXT_SIZE, SHORT_NAMES_ENCODING),
         'DIR_Attr' / Int8ul,
-        'DIR_NTRes' / Const(EMPTY_BYTE),
+        'DIR_NTRes' / Int8ul,  # this tagged for lfn (0x00 for lfn prefix, 0x18 for short name in lfn)
         'DIR_CrtTimeTenth' / Const(EMPTY_BYTE),  # ignored by esp-idf fatfs library
         'DIR_CrtTime' / Int16ul,  # ignored by esp-idf fatfs library
         'DIR_CrtDate' / Int16ul,  # ignored by esp-idf fatfs library
@@ -159,6 +159,7 @@ class Entry:
                        lfn_order: int = SHORT_ENTRY,
                        lfn_names: Optional[List[bytes]] = None,
                        lfn_checksum_: int = 0,
+                       fits_short: bool = False,
                        lfn_is_last: bool = False) -> None:
         """
         :param first_cluster_id: id of the first data cluster for given entry
@@ -172,6 +173,7 @@ class Entry:
         :param lfn_names: if the entry is dedicated for long names the lfn_names contains
             LDIR_Name1, LDIR_Name2 and LDIR_Name3 in this order
         :param lfn_checksum_: use only for long file names, checksum calculated lfn_checksum function
+        :param fits_short: determines if the name fits in 8.3 filename
         :param lfn_is_last: determines if the long file name entry is holds last part of the name,
             thus its address is first in the physical order
         :returns: None
@@ -213,6 +215,7 @@ class Entry:
                 DIR_Name=pad_string(object_name, size=MAX_NAME_SIZE),
                 DIR_Name_ext=pad_string(object_extension, size=MAX_EXT_SIZE),
                 DIR_Attr=entity_type,
+                DIR_NTRes=0x00 if (not self.fatfs_state.long_names_enabled) or (not fits_short) else 0x18,
                 DIR_FstClusLO=first_cluster_id,
                 DIR_FileSize=size,
                 DIR_CrtDate=date_entry_,  # ignored by esp-idf fatfs library

components/fatfs/fatfs_utils/fs_object.py

@@ -260,6 +260,7 @@ class Directory:
                                       entity_extension=extension,
                                       date=fatfs_date_,
                                       time=fatfs_time_,
+                                      fits_short=True,
                                       entity_type=entity_type)
             return free_cluster, free_entry, target_dir
         return self.allocate_long_name_object(free_entry=free_entry,

components/fatfs/fatfs_utils/utils.py

@@ -42,7 +42,7 @@ FATFS_SECONDS_GRANULARITY: int = 2
 LONG_NAMES_ENCODING: str = 'utf-16'
 SHORT_NAMES_ENCODING: str = 'utf-8'
 
-ALLOWED_SECTOR_SIZES: List[int] = [512, 1024, 2048, 4096]
+ALLOWED_SECTOR_SIZES: List[int] = [4096]
 ALLOWED_SECTORS_PER_CLUSTER: List[int] = [1, 2, 4, 8, 16, 32, 64, 128]
 
 
@@ -181,7 +181,7 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
     parser.add_argument('--fat_type',
                         default=0,
                         type=int,
-                        choices=[12, 16, 0],
+                        choices=[FAT12, FAT16, 0],
                         help="""
                         Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
                         calculation using cluster size and partition size.
@@ -269,3 +269,5 @@ class FATDefaults:
     VERSION: int = 2
     TEMP_BUFFER_SIZE: int = 32
     UPDATE_RATE: int = 16
+    WR_SIZE: int = 16
+    WL_SECTOR_SIZE: int = 4096

components/fatfs/fatfsparse.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
@@ -9,6 +10,7 @@ from fatfs_utils.entry import Entry
 from fatfs_utils.fat import FAT
 from fatfs_utils.fatfs_state import BootSectorState
 from fatfs_utils.utils import FULL_BYTE, LONG_NAMES_ENCODING, PAD_CHAR, FATDefaults, lfn_checksum, read_filesystem
+from wl_fatfsgen import remove_wl
 
 
 def build_file_name(name1: bytes, name2: bytes, name3: bytes) -> str:
@@ -42,7 +44,7 @@ def traverse_folder_tree(directory_bytes_: bytes,
                          name: str,
                          state_: BootSectorState,
                          fat_: FAT,
-                         binary_array_: bytearray) -> None:
+                         binary_array_: bytes) -> None:
     os.makedirs(name)
 
     assert len(directory_bytes_) % FATDefaults.ENTRY_SIZE == 0
@@ -89,9 +91,26 @@ if __name__ == '__main__':
     argument_parser.add_argument('--long-name-support',
                                  action='store_true',
                                  help='Set flag to enable long names support.')
+
+    argument_parser.add_argument('--wear-leveling',
+                                 action='store_true',
+                                 help='Set flag to parse an image encoded using wear levelling.')
+
     args = argument_parser.parse_args()
 
     fs = read_filesystem(args.input_image)
+
+    # An algorithm for removing wear levelling:
+    # 1. find an remove dummy sector:
+    #    a) dummy sector is at the position defined by the number of records in the state sector
+    #    b) dummy may not be placed in state nor cfg sectors
+    #    c) first (boot) sector position (boot_s_pos) is calculated using value of move count
+    #    boot_s_pos = - mc
+    # 2. remove state sectors (trivial)
+    # 3. remove cfg sector (trivial)
+    # 4. valid fs is then old_fs[-mc:] + old_fs[:-mc]
+    if args.wear_leveling:
+        fs = remove_wl(fs)
     boot_sector_ = BootSector()
     boot_sector_.parse_boot_sector(fs)
     fat = FAT(boot_sector_.boot_sector_state, init_=False)

components/fatfs/test_fatfsgen/test_fatfsgen.py

@@ -294,7 +294,7 @@ class FatFSGen(unittest.TestCase):
         fatfs.create_file('HELLO', extension='TXT')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = read_filesystem(CFG['output_file'])
-        self.assertEqual(file_system[0x2000: 0x2019], b'HELLO   TXT \x00\x00\x00\x00!\x00!\x00\x00\x00\x00\x00!')
+        self.assertEqual(file_system[0x2000: 0x2019], b'HELLO   TXT \x18\x00\x00\x00!\x00!\x00\x00\x00\x00\x00!')
 
     def test_lfn_short_name(self) -> None:
         fatfs = fatfsgen.FATFS(long_names_enabled=True)
@@ -302,7 +302,7 @@ class FatFSGen(unittest.TestCase):
         fatfs.write_content(path_from_root=['HELLO.TXT'], content=b'this is a test')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = read_filesystem(CFG['output_file'])
-        self.assertEqual(file_system[0x2000: 0x2010], b'HELLO   TXT \x00\x00\x00\x00')
+        self.assertEqual(file_system[0x2000: 0x2010], b'HELLO   TXT \x18\x00\x00\x00')
         self.assertEqual(file_system[0x2010: 0x2020], b'!\x00!\x00\x00\x00\x00\x00!\x00\x02\x00\x0e\x00\x00\x00')
         self.assertEqual(file_system[0x6000: 0x6010], b'this is a test\x00\x00')
 
@@ -367,7 +367,7 @@ class FatFSGen(unittest.TestCase):
         self.assertEqual(file_system[0x6012: 0x6020], b'!\x00\x00\x00\x00\x00!\x00\x02\x00\x00\x00\x00\x00')
         self.assertEqual(file_system[0x6020: 0x6030], b'..         \x10\x00\x00\x00\x00')
         self.assertEqual(file_system[0x6030: 0x6040], b'!\x00!\x00\x00\x00\x00\x00!\x00\x01\x00\x00\x00\x00\x00')
-        self.assertEqual(file_system[0x6040: 0x6050], b'HELLO   TXT \x00\x00\x00\x00')
+        self.assertEqual(file_system[0x6040: 0x6050], b'HELLO   TXT \x18\x00\x00\x00')
         self.assertEqual(file_system[0x6050: 0x6060], b'!\x00!\x00\x00\x00\x00\x00!\x00\x03\x00\x00\x00\x00\x00')
 
     def test_lfn_nested_long_empty(self) -> None:
@@ -410,7 +410,7 @@ class FatFSGen(unittest.TestCase):
         self.assertEqual(file_system[0x6012: 0x6020], b'!\x00\x00\x00\x00\x00!\x00\x02\x00\x00\x00\x00\x00')
         self.assertEqual(file_system[0x6020: 0x6030], b'..         \x10\x00\x00\x00\x00')
         self.assertEqual(file_system[0x6030: 0x6040], b'!\x00!\x00\x00\x00\x00\x00!\x00\x01\x00\x00\x00\x00\x00')
-        self.assertEqual(file_system[0x6040: 0x6050], b'HELLO   TXT \x00\x00\x00\x00')
+        self.assertEqual(file_system[0x6040: 0x6050], b'HELLO   TXT \x18\x00\x00\x00')
         self.assertEqual(file_system[0x6050: 0x6060], b'!\x00!\x00\x00\x00\x00\x00!\x00\x03\x00\x0e\x00\x00\x00')
 
         self.assertEqual(file_system[0x7000: 0x7010], b'this is a test\x00\x00')

components/fatfs/test_fatfsgen/test_fatfsparse.py

@@ -23,6 +23,7 @@ class FatFSGen(unittest.TestCase):
         shutil.rmtree('output_data', ignore_errors=True)
         shutil.rmtree('Espressif', ignore_errors=True)
         shutil.rmtree('testf', ignore_errors=True)
+        shutil.rmtree('testf_wl', ignore_errors=True)
 
         if os.path.exists('fatfs_image.img'):
             os.remove('fatfs_image.img')
@@ -138,6 +139,15 @@ class FatFSGen(unittest.TestCase):
         ], stderr=STDOUT)
         run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT)
         assert compare_folders('testf', 'Espressif')
+        shutil.rmtree('Espressif', ignore_errors=True)
+
+        run([
+            'python',
+            f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}',
+            'testf'
+        ], stderr=STDOUT)
+        run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT)
+        assert compare_folders('testf', 'Espressif')
 
     def test_e2e_deeper(self) -> None:
         folder_ = {
@@ -159,6 +169,7 @@ class FatFSGen(unittest.TestCase):
                 folder_
             ]
         }
+
         generate_local_folder_structure(struct_, path_='.')
         run([
             'python',
@@ -167,6 +178,15 @@ class FatFSGen(unittest.TestCase):
         ], stderr=STDOUT)
         run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT)
         assert compare_folders('testf', 'Espressif')
+        shutil.rmtree('Espressif', ignore_errors=True)
+
+        run([
+            'python',
+            f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}',
+            'testf'
+        ], stderr=STDOUT)
+        run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT)
+        assert compare_folders('testf', 'Espressif')
 
     def test_e2e_deeper_large(self) -> None:
         folder_ = {
@@ -214,6 +234,15 @@ class FatFSGen(unittest.TestCase):
         ], stderr=STDOUT)
         run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT)
         assert compare_folders('testf', 'Espressif')
+        shutil.rmtree('Espressif', ignore_errors=True)
+
+        run([
+            'python',
+            f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}',
+            'testf'
+        ], stderr=STDOUT)
+        run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT)
+        assert compare_folders('testf', 'Espressif')
 
     def test_e2e_very_deep(self) -> None:
         folder_ = {

components/fatfs/test_fatfsgen/test_wl_fatfsgen.py

@@ -105,9 +105,6 @@ class WLFatFSGen(unittest.TestCase):
         fatfs.wl_create_directory('TESTFOLD')
         self.assertRaises(WLNotInitialized, fatfs.wl_write_filesystem, CFG['output_file'])
 
-    def test_wrong_sector_size(self) -> None:
-        self.assertRaises(NotImplementedError, wl_fatfsgen.WLFATFS, sector_size=1024)
-
     def test_e2e_deep_folder_into_image_ext(self) -> None:
         fatfs = wl_fatfsgen.WLFATFS()
         fatfs.wl_generate(CFG['test_dir2'])

components/fatfs/wl_fatfsgen.py

@@ -11,6 +11,41 @@ from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, genera
 from fatfsgen import FATFS
 
 
+def remove_wl(binary_image: bytes) -> bytes:
+    partition_size: int = len(binary_image)
+    total_sectors: int = partition_size // FATDefaults.WL_SECTOR_SIZE
+    wl_state_size: int = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * total_sectors
+    wl_state_sectors_cnt: int = (wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
+    wl_state_total_size: int = wl_state_sectors_cnt * FATDefaults.WL_SECTOR_SIZE
+    wl_sectors_size: int = (wl_state_sectors_cnt
+                            * FATDefaults.WL_SECTOR_SIZE
+                            * WLFATFS.WL_STATE_COPY_COUNT
+                            + FATDefaults.WL_SECTOR_SIZE)
+
+    correct_wl_configuration = binary_image[-wl_sectors_size:]
+
+    data_ = WLFATFS.WL_STATE_T_DATA.parse(correct_wl_configuration[:WLFATFS.WL_STATE_HEADER_SIZE])
+
+    total_records = 0
+    # iterating over records field of the first copy of the state sector
+    for i in range(WLFATFS.WL_STATE_HEADER_SIZE, wl_state_total_size, WLFATFS.WL_STATE_RECORD_SIZE):
+        if correct_wl_configuration[i:i + WLFATFS.WL_STATE_RECORD_SIZE] != WLFATFS.WL_STATE_RECORD_SIZE * b'\xff':
+            total_records += 1
+        else:
+            break
+    before_dummy = binary_image[:total_records * FATDefaults.WL_SECTOR_SIZE]
+    after_dummy = binary_image[total_records * FATDefaults.WL_SECTOR_SIZE + FATDefaults.WL_SECTOR_SIZE:]
+    new_image: bytes = before_dummy + after_dummy
+
+    # remove wl sectors
+    new_image = new_image[:len(new_image) - (FATDefaults.WL_SECTOR_SIZE + 2 * wl_state_total_size)]
+
+    # reorder to preserve original order
+    new_image = (new_image[-data_['move_count'] * FATDefaults.WL_SECTOR_SIZE:]
+                 + new_image[:-data_['move_count'] * FATDefaults.WL_SECTOR_SIZE])
+    return new_image
+
+
 class WLFATFS:
     # pylint: disable=too-many-instance-attributes
     WL_CFG_SECTORS_COUNT = 1
@@ -18,7 +53,7 @@ class WLFATFS:
     WL_CONFIG_HEADER_SIZE = 48
     WL_STATE_RECORD_SIZE = 16
     WL_STATE_HEADER_SIZE = 64
-    WL_STATE_COPY_COUNT = 2
+    WL_STATE_COPY_COUNT = 2  # always 2 copies for power failure safety
     WL_SECTOR_SIZE = 0x1000
 
     WL_STATE_T_DATA = Struct(
@@ -37,7 +72,7 @@ class WLFATFS:
         'start_addr' / Int32ul,
         'full_mem_size' / Int32ul,
         'page_size' / Int32ul,
-        'sector_size' / Int32ul,
+        'sector_size' / Int32ul,  # always 4096 for the types of NOR flash supported by ESP-IDF!
         'updaterate' / Int32ul,
         'wr_size' / Int32ul,
         'version' / Int32ul,
@@ -50,7 +85,6 @@ class WLFATFS:
                  reserved_sectors_cnt: int = FATDefaults.RESERVED_SECTORS_COUNT,
                  fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT,
                  sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER,
-                 sector_size: int = FATDefaults.SECTOR_SIZE,
                  sectors_per_fat: int = FATDefaults.SECTORS_PER_FAT,
                  explicit_fat_type: int = None,
                  hidden_sectors: int = FATDefaults.HIDDEN_SECTORS,
@@ -63,28 +97,22 @@ class WLFATFS:
                  use_default_datetime: bool = True,
                  version: int = FATDefaults.VERSION,
                  temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE,
-                 update_rate: int = FATDefaults.UPDATE_RATE,
                  device_id: int = None,
                  root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
                  media_type: int = FATDefaults.MEDIA_TYPE) -> None:
-        if sector_size != WLFATFS.WL_SECTOR_SIZE:
-            raise NotImplementedError(f'The only supported sector size is currently {WLFATFS.WL_SECTOR_SIZE}')
-
         self._initialized = False
-        self.sector_size = sector_size
         self._version = version
         self._temp_buff_size = temp_buff_size
         self._device_id = device_id
-        self._update_rate = update_rate
         self.partition_size = size
-        self.total_sectors = self.partition_size // self.sector_size
+        self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE
         self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
 
         # determine the number of required sectors (roundup to sector size)
-        self.wl_state_sectors = (self.wl_state_size + self.sector_size - 1) // self.sector_size
+        self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
 
-        self.boot_sector_start = self.sector_size  # shift by one "dummy" sector
+        self.boot_sector_start = FATDefaults.WL_SECTOR_SIZE  # shift by one "dummy" sector
-        self.fat_table_start = self.boot_sector_start + reserved_sectors_cnt * self.sector_size
+        self.fat_table_start = self.boot_sector_start + reserved_sectors_cnt * FATDefaults.WL_SECTOR_SIZE
 
         wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
                       self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
@@ -92,11 +120,11 @@ class WLFATFS:
 
         self.plain_fatfs = FATFS(
             explicit_fat_type=explicit_fat_type,
-            size=self.plain_fat_sectors * self.sector_size,
+            size=self.plain_fat_sectors * FATDefaults.WL_SECTOR_SIZE,
             reserved_sectors_cnt=reserved_sectors_cnt,
             fat_tables_cnt=fat_tables_cnt,
             sectors_per_cluster=sectors_per_cluster,
-            sector_size=sector_size,
+            sector_size=FATDefaults.WL_SECTOR_SIZE,
             sectors_per_fat=sectors_per_fat,
             root_entry_count=root_entry_count,
             hidden_sectors=hidden_sectors,
@@ -121,17 +149,17 @@ class WLFATFS:
         self._initialized = True
 
     def _add_dummy_sector(self) -> None:
-        self.fatfs_binary_image = self.sector_size * FULL_BYTE + self.fatfs_binary_image
+        self.fatfs_binary_image = FATDefaults.WL_SECTOR_SIZE * FULL_BYTE + self.fatfs_binary_image
 
     def _add_config_sector(self) -> None:
         wl_config_data = WLFATFS.WL_CONFIG_T_DATA.build(
             dict(
                 start_addr=0,
                 full_mem_size=self.partition_size,
-                page_size=self.sector_size,
+                page_size=FATDefaults.WL_SECTOR_SIZE,  # equal to sector size (always 4096)
-                sector_size=self.sector_size,
+                sector_size=FATDefaults.WL_SECTOR_SIZE,
-                updaterate=self._update_rate,
+                updaterate=FATDefaults.UPDATE_RATE,
-                wr_size=16,
+                wr_size=FATDefaults.WR_SIZE,
                 version=self._version,
                 temp_buff_size=self._temp_buff_size
             )
@@ -143,7 +171,8 @@ class WLFATFS:
         # adding three 4 byte zeros to align the structure
         wl_config = wl_config_data + wl_config_crc + Int32ul.build(0) + Int32ul.build(0) + Int32ul.build(0)
 
-        self.fatfs_binary_image += (wl_config + (self.sector_size - WLFATFS.WL_CONFIG_HEADER_SIZE) * FULL_BYTE)
+        self.fatfs_binary_image += (
+            wl_config + (FATDefaults.WL_SECTOR_SIZE - WLFATFS.WL_CONFIG_HEADER_SIZE) * FULL_BYTE)
 
     def _add_state_sectors(self) -> None:
         wl_state_data = WLFATFS.WL_STATE_T_DATA.build(
@@ -152,8 +181,8 @@ class WLFATFS:
                 max_pos=self.plain_fat_sectors + WLFATFS.WL_DUMMY_SECTORS_COUNT,
                 move_count=0,
                 access_count=0,
-                max_count=self._update_rate,
+                max_count=FATDefaults.UPDATE_RATE,
-                block_size=self.sector_size,
+                block_size=FATDefaults.WL_SECTOR_SIZE,  # equal to page size, thus equal to wl sector size (4096)
                 version=self._version,
                 device_id=self._device_id or generate_4bytes_random(),
             )
@@ -161,9 +190,9 @@ class WLFATFS:
         crc = crc32(list(wl_state_data), UINT32_MAX)
         wl_state_crc = Int32ul.build(crc)
         wl_state = wl_state_data + wl_state_crc
-        wl_state_sector_padding: bytes = (self.sector_size - WLFATFS.WL_STATE_HEADER_SIZE) * FULL_BYTE
+        wl_state_sector_padding: bytes = (FATDefaults.WL_SECTOR_SIZE - WLFATFS.WL_STATE_HEADER_SIZE) * FULL_BYTE
         wl_state_sector: bytes = (
-            wl_state + wl_state_sector_padding + (self.wl_state_sectors - 1) * self.sector_size * FULL_BYTE
+            wl_state + wl_state_sector_padding + (self.wl_state_sectors - 1) * FATDefaults.WL_SECTOR_SIZE * FULL_BYTE
         )
         self.fatfs_binary_image += (WLFATFS.WL_STATE_COPY_COUNT * wl_state_sector)
 
@@ -193,8 +222,7 @@ if __name__ == '__main__':
     desc = 'Create a FAT filesystem with support for wear levelling and populate it with directory content'
     args = get_args_for_partition_generator(desc)
 
-    wl_fatfs = WLFATFS(sector_size=args.sector_size,
+    wl_fatfs = WLFATFS(sectors_per_cluster=args.sectors_per_cluster,
-                       sectors_per_cluster=args.sectors_per_cluster,
                        size=args.partition_size,
                        root_entry_count=args.root_entry_count,
                        explicit_fat_type=args.fat_type,

examples/storage/fatfsgen/main/Kconfig.projbuild

@@ -7,6 +7,13 @@ menu "Example Configuration"
             If read-only mode is set, the generated fatfs image will be raw (without wear levelling support).
             Otherwise it will support wear levelling that enables read-write mounting.
 
+    config EXAMPLE_FATFS_WRITE_COUNT
+        int "Number of volumes"
+        default 1
+        range 1 600
+        help
+            Number of writes to the file (for testing purposes).
+
     config EXAMPLE_FATFS_DEFAULT_DATETIME
         bool "Default modification date and time for generated FATFS image"
         default n

examples/storage/fatfsgen/main/fatfsgen_example_main.c

@@ -66,13 +66,17 @@ void app_main(void)
     if (!EXAMPLE_FATFS_MODE_READ_ONLY){
         // Open file for reading
         ESP_LOGI(TAG, "Opening file");
-        FILE *f = fopen(device_filename, "wb");
+        FILE *f;
+        for(int i = 0; i < CONFIG_EXAMPLE_FATFS_WRITE_COUNT; i++){
+            f = fopen(device_filename, "wb");
             if (f == NULL) {
                 ESP_LOGE(TAG, "Failed to open file for writing");
                 return;
             }
             fprintf(f, "This is written by the device");
             fclose(f);
+        }
+
         ESP_LOGI(TAG, "File written");
 
         // Open file for reading

examples/storage/fatfsgen/pytest_fatfsgen_example.py

@@ -1,13 +1,55 @@
 # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Unlicense OR CC0-1.0
 
+import os
 import re
+import shutil
+import sys
 from datetime import datetime
+from subprocess import STDOUT, run
 from typing import List
 
 import pytest
 from pytest_embedded import Dut
 
+idf_path = os.environ['IDF_PATH']  # get value of IDF_PATH from environment
+parttool_dir = os.path.join(idf_path, 'components', 'partition_table')
+
+
+sys.path.append(parttool_dir)
+from parttool import PartitionName, ParttoolTarget  # noqa E402  # pylint: disable=C0413
+
+
+def file_(x: str, content_: str = 'hey this is a test') -> dict:
+    return {
+        'type': 'file',
+        'name': x,
+        'content': content_
+    }
+
+
+def generate_local_folder_structure(structure_: dict, path_: str) -> None:
+    if structure_['type'] == 'folder':
+        new_path_ = os.path.join(path_, structure_['name'])
+        os.makedirs(new_path_)
+        for item_ in structure_['content']:
+            generate_local_folder_structure(item_, new_path_)
+    else:
+        new_path_ = os.path.join(path_, structure_['name'])
+        with open(new_path_, 'w') as f_:
+            f_.write(structure_['content'])
+
+
+def compare_folders(fp1: str, fp2: str) -> bool:
+    if os.path.isdir(fp1) != os.path.isdir(fp2):
+        return False
+    if os.path.isdir(fp1):
+        if set(os.listdir(fp1)) != set(os.listdir(fp2)):
+            return False
+        return all([compare_folders(os.path.join(fp1, path_), os.path.join(fp2, path_)) for path_ in os.listdir(fp1)])
+    with open(fp1, 'rb') as f1_, open(fp2, 'rb') as f2_:
+        return f1_.read() == f2_.read()
+
 
 # Example_GENERIC
 @pytest.mark.esp32
@@ -51,6 +93,7 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None:
     filename_sn = 'sub/test.txt'
     date_modified = datetime.today()
     date_default = datetime(1980, 1, 1)
+    fatfs_parser_path = os.path.join(idf_path, 'components', 'fatfs', 'fatfsparse.py')
 
     if config in ['test_read_write_partition_gen', 'test_read_write_partition_gen_default_dt']:
         filename = filename_sn
@@ -68,6 +111,32 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None:
                     'example: Unmounting FAT filesystem',
                     'example: Done'], timeout)
 
+        target = ParttoolTarget(dut.port)
+        target.read_partition(PartitionName('storage'), 'temp.img')
+        run(['python', fatfs_parser_path, '--wear-leveling', 'temp.img'], stderr=STDOUT)
+        folder_ = {
+            'type': 'folder',
+            'name': 'SUB',
+            'content': [
+                file_('TEST.TXT', content_='this is test\n'),
+            ]
+        }
+        struct_: dict = {
+            'type': 'folder',
+            'name': 'testf',
+            'content': [
+                file_('HELLO.TXT', content_='This is generated on the host\n'),
+                file_('INNER.TXT', content_='This is written by the device'),
+                folder_
+            ]
+        }
+        generate_local_folder_structure(struct_, path_='.')
+        try:
+            assert compare_folders('testf', 'Espressif')
+        finally:
+            shutil.rmtree('Espressif', ignore_errors=True)
+            shutil.rmtree('testf', ignore_errors=True)
+
     elif config in ['test_read_only_partition_gen', 'test_read_only_partition_gen_default_dt']:
         filename = filename_sn
         filename_expected = f'/spiflash/{filename}'
@@ -79,6 +148,30 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None:
         expect_all(['example: Read from file: \'this is test\'',
                     'example: Unmounting FAT filesystem',
                     'example: Done'], timeout)
+        target = ParttoolTarget(dut.port)
+        target.read_partition(PartitionName('storage'), 'temp.img')
+        run(['python', fatfs_parser_path, '--long-name-support', 'temp.img'], stderr=STDOUT)
+        folder_ = {
+            'type': 'folder',
+            'name': 'sublongnames',
+            'content': [
+                file_('testlongfilenames.txt', content_='this is test; long name it has\n'),
+            ]
+        }
+        struct_ = {
+            'type': 'folder',
+            'name': 'testf',
+            'content': [
+                file_('hellolongname.txt', content_='This is generated on the host; long name it has\n'),
+                folder_
+            ]
+        }
+        generate_local_folder_structure(struct_, path_='.')
+        try:
+            assert compare_folders('testf', 'Espressif')
+        finally:
+            shutil.rmtree('Espressif', ignore_errors=True)
+            shutil.rmtree('testf', ignore_errors=True)
 
     elif config in ['test_read_write_partition_gen_ln', 'test_read_write_partition_gen_ln_default_dt']:
         filename = filename_ln
@@ -107,3 +200,27 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None:
         expect_all(['example: Read from file: \'this is test; long name it has\'',
                     'example: Unmounting FAT filesystem',
                     'example: Done'], timeout)
+        target = ParttoolTarget(dut.port)
+        target.read_partition(PartitionName('storage'), 'temp.img')
+        run(['python', fatfs_parser_path, 'temp.img'], stderr=STDOUT)
+        folder_ = {
+            'type': 'folder',
+            'name': 'SUB',
+            'content': [
+                file_('TEST.TXT', content_='this is test\n'),
+            ]
+        }
+        struct_ = {
+            'type': 'folder',
+            'name': 'testf',
+            'content': [
+                file_('HELLO.TXT', content_='This is generated on the host\n'),
+                folder_
+            ]
+        }
+        generate_local_folder_structure(struct_, path_='.')
+        try:
+            assert compare_folders('testf', 'Espressif')
+        finally:
+            shutil.rmtree('Espressif', ignore_errors=True)
+            shutil.rmtree('testf', ignore_errors=True)

examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen

@@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
 CONFIG_FATFS_LFN_HEAP=n
 CONFIG_FATFS_LFN_STACK=n
 CONFIG_FATFS_LFN_NONE=y
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_default_dt

@@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=n
 CONFIG_FATFS_LFN_STACK=n
 CONFIG_FATFS_LFN_NONE=y
 CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln

@@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
 CONFIG_FATFS_LFN_HEAP=y
 CONFIG_FATFS_LFN_NONE=n
 CONFIG_FATFS_LFN_STACK=n
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln_default_dt

@@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=y
 CONFIG_FATFS_LFN_NONE=n
 CONFIG_FATFS_LFN_STACK=n
 CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen

@@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
 CONFIG_FATFS_LFN_HEAP=n
 CONFIG_FATFS_LFN_STACK=n
 CONFIG_FATFS_LFN_NONE=y
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_default_dt

@@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=n
 CONFIG_FATFS_LFN_STACK=n
 CONFIG_FATFS_LFN_NONE=y
 CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln

@@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
 CONFIG_FATFS_LFN_HEAP=y
 CONFIG_FATFS_LFN_NONE=n
 CONFIG_FATFS_LFN_STACK=n
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln_default_dt

@@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=y
 CONFIG_FATFS_LFN_NONE=n
 CONFIG_FATFS_LFN_STACK=n
 CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

tools/ci/executable-list.txt

@@ -8,6 +8,7 @@ components/espcoredump/test/test_espcoredump.py
 components/espcoredump/test/test_espcoredump.sh
 components/espcoredump/test_apps/build_espcoredump.sh
 components/fatfs/fatfsgen.py
+components/fatfs/fatfsparse.py
 components/fatfs/test_fatfsgen/test_fatfsgen.py
 components/fatfs/test_fatfsgen/test_fatfsparse.py
 components/fatfs/test_fatfsgen/test_wl_fatfsgen.py