feat(storage/fatfs): add advanced example for fatfs

examples/storage/.build-test-rules.yml

@@ -34,6 +34,14 @@ examples/storage/ext_flash_fatfs:
       temporary: true
       reason: lack of runners
 
+examples/storage/fatfs_advanced:
+  depends_components:
+    - fatfs
+    - vfs
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      reason: only one target needed
+
 examples/storage/fatfs_basic:
   depends_components:
     - fatfs

examples/storage/README.md

@@ -8,6 +8,7 @@ This directory contains a range of examples ESP-IDF projects. These are intended
 The examples are grouped into sub-directories by category. Each category directory contains one or more example projects:
 
 * `fatfs_basic` minimal example of FatFS usage on SPI FLASH
+* `fatfs_advanced` example demonstrates how to use advanced features for working with FatFS such as automatic partition generation
 * `custom_flash_driver` example demonstrates how to implement your own flash chip driver by overriding the default driver.
 * `emmc` example demonstrates how to use an eMMC chip with an ESP device.
 * `ext_flash_fatfs` example demonstrates how to use FATFS partition with external SPI FLASH chip.

examples/storage/fatfs_advanced/CMakeLists.txt

@@ -0,0 +1,6 @@
+# The following lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(fatfsgen)

examples/storage/fatfs_advanced/README.md

@@ -0,0 +1,69 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- |
+
+# FATFS partition generation example
+
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
+This example demonstrates how to use the FATFS partition
+generation tool [fatfsgen.py](../../../components/fatfs/fatfsgen.py) to automatically create a FATFS
+filesystem image from the contents of a host folder during build, with an option of
+automatically flashing the created image on invocation of `idf.py -p PORT flash` as well as usage of long file names for FATFS.
+Beware that the minimal required size of the flash is 4 MB.
+You can specify using menuconfig weather example will use read-only or read-write mode. The default option is read-write mode.
+To change it just use menuconfig:
+
+```shell
+idf.py menuconfig
+```
+
+Then select `Example Configuration` a chose `Mode for generated FATFS image` either `Read-Write Mode` or `Read-Only Mode`.
+`Read-Only` option indicates generating raw fatfs image without wear levelling support.
+On the other hand, for `Read-Write` the generated fatfs image will support wear levelling thus can be mounted in read-write mode.
+
+
+The following gives an overview of the example:
+
+1. There is a directory `fatfs_image` from which the FATFS filesystem image will be created.
+
+2. The function `fatfs_create_rawflash_image` is used to specify that a FATFS image
+should be created during build for the `storage` partition.
+For CMake, it is called from [the main component's CMakeLists.txt](./main/CMakeLists.txt).
+`FLASH_IN_PROJECT` specifies that the created image
+should be flashed on invocation of `idf.py -p PORT flash` together with app, bootloader, partition table, etc.
+The image is created on the example's build directory with the output filename `storage.bin`.
+
+3. Upon invocation of `idf.py -p PORT flash monitor`, application loads and
+finds there is already a valid FATFS filesystem in the `storage` partition with files same as those in `fatfs_image` directory. The application is then
+able to read those files.
+
+## How to use example
+
+### Build and flash
+
+To run the example, type the following command:
+
+```CMake
+# CMake
+idf.py -p PORT flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
+## Example output
+
+Here is the example's console output:
+
+```
+...
+I (322) example: Mounting FAT filesystem
+I (332) example: Reading file
+I (332) example: Read from file: 'this is test'
+I (332) example: Unmounting FAT filesystem
+I (342) example: Done
+```
+
+The logic of the example is contained in a [single source file](./main/fatfsgen_example_main.c),
+and it should be relatively simple to match points in its execution with the log outputs above.

examples/storage/fatfs_advanced/fatfs_long_name_image/filegeneratedonhost.txt

@@ -0,0 +1 @@
+This is generated on the host; it has long name

examples/storage/fatfs_advanced/fatfs_long_name_image/subdirectoryfromhost/innerfile.txt

@@ -0,0 +1 @@
+this is test; it has long name

examples/storage/fatfs_advanced/main/CMakeLists.txt

@@ -0,0 +1,26 @@
+idf_component_register(SRCS "fatfs_advanced_example_main.c"
+                    INCLUDE_DIRS ".")
+
+# Create a FATFS image from the contents of the 'fatfs_image' directory
+# that fits the partition named 'storage'. FLASH_IN_PROJECT indicates that
+# the generated image should be flashed when the entire project is flashed to
+# the target with 'idf.py -p PORT flash'.
+# If read-only mode is set (CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY)
+# the generated image will be raw without wear levelling support.
+# Otherwise it will support wear levelling and thus enable read-write mounting of the image in the device.
+
+set(image ../fatfs_long_name_image)
+
+if(CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY)
+    if(CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME)
+        fatfs_create_rawflash_image(storage ${image} FLASH_IN_PROJECT)
+    else()
+        fatfs_create_rawflash_image(storage ${image} FLASH_IN_PROJECT PRESERVE_TIME)
+    endif()
+else()
+    if(CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME)
+        fatfs_create_spiflash_image(storage ${image} FLASH_IN_PROJECT)
+    else()
+        fatfs_create_spiflash_image(storage ${image} FLASH_IN_PROJECT PRESERVE_TIME)
+    endif()
+endif()

examples/storage/fatfs_advanced/main/Kconfig.projbuild

@@ -0,0 +1,24 @@
+menu "Example Configuration"
+
+    config EXAMPLE_FATFS_MODE_READ_ONLY
+        bool "Read only mode for generated FATFS image"
+        default n
+        help
+            If read-only mode is set, the generated fatfs image will be raw (without wear levelling support).
+            Otherwise it will support wear levelling that enables read-write mounting.
+
+    config EXAMPLE_FATFS_WRITE_COUNT
+        int "Number of volumes"
+        default 1
+        range 1 600
+        help
+            Number of writes to the file (for testing purposes).
+
+    config EXAMPLE_FATFS_DEFAULT_DATETIME
+        bool "Default modification date and time for generated FATFS image"
+        default n
+        help
+            If default datetime is set, all files created in the generated FATFS partition have default time
+            equal to FATFS origin time (1 January 1980)
+
+endmenu

examples/storage/fatfs_advanced/main/fatfs_advanced_example_main.c

@@ -0,0 +1,138 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/errno.h>
+#include "esp_vfs.h"
+#include "esp_vfs_fat.h"
+#include "sdkconfig.h"
+
+static const char *TAG = "example";
+
+// Mount path for the partition
+const char *base_path = "/spiflash";
+
+void write_file(const char *filename, const char *data)
+{
+    ESP_LOGI(TAG, "Opening file '%s' for writing", filename);
+
+    FILE *f = fopen(filename, "w");
+    if (f == NULL) {
+        ESP_LOGE(TAG, "Failed to open file for writing: %s", strerror(errno));
+        return;
+    }
+
+    ESP_LOGI(TAG, "Writing to file");
+
+    fputs(data, f);
+
+    ESP_LOGI(TAG, "File written");
+
+    ESP_LOGI(TAG, "Closing file");
+    fclose(f);
+}
+
+void read_file(const char *filename)
+{
+    ESP_LOGI(TAG, "Opening file '%s' for reading", filename);
+
+    FILE *f = fopen(filename, "r");
+    if (f == NULL) {
+        ESP_LOGE(TAG, "Failed to open file for reading: %s", strerror(errno));
+        return;
+    }
+
+    ESP_LOGI(TAG, "Reading from file");
+
+    char line[128];
+    fgets(line, sizeof(line), f);
+
+    // strip newline
+    char *pos = strchr(line, '\n');
+    if (pos) {
+        *pos = '\0'; // strip newline
+    }
+
+    ESP_LOGI(TAG, "Read from file: '%s'", line);
+
+    ESP_LOGI(TAG, "Closing file '%s'", filename);
+    fclose(f);
+}
+
+void stat_file(const char *filename)
+{
+
+    struct stat info;
+    struct tm timeinfo;
+    char buffer[32];
+
+    ESP_LOGI(TAG, "Stating file '%s' for modification time", filename);
+
+    if(stat(filename, &info) < 0){
+        ESP_LOGE(TAG, "Failed to read file stats: %s", strerror(errno));
+        return;
+    }
+    localtime_r(&info.st_mtime, &timeinfo);
+    strftime(buffer, sizeof(buffer), "%Y-%m-%d", &timeinfo);
+
+    ESP_LOGI(TAG, "The file '%s' was modified at date: %s", filename, buffer);
+}
+
+void app_main(void)
+{
+    ESP_LOGI(TAG, "Mounting FAT filesystem");
+
+    // To mount device we need name of device partition, define base_path
+    // and allow format partition in case if it is new one and was not formatted before
+    const esp_vfs_fat_mount_config_t mount_config = {
+            .max_files = 4,
+            .format_if_mount_failed = false,
+            .allocation_unit_size = CONFIG_WL_SECTOR_SIZE,
+            .use_one_fat = false,
+    };
+
+    esp_err_t err;
+
+#ifdef CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
+    err = esp_vfs_fat_spiflash_mount_ro(base_path, "storage", &mount_config);
+#else // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
+
+    // Handle of the wear levelling library instance
+    wl_handle_t wl_handle = WL_INVALID_HANDLE;
+    err = esp_vfs_fat_spiflash_mount_rw_wl(base_path, "storage", &mount_config, &wl_handle);
+#endif // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
+
+    if (err != ESP_OK) {
+        ESP_LOGE(TAG, "Failed to mount FATFS (%s)python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps", esp_err_to_name(err));
+        return;
+    }
+
+    // Read contents of a file
+    read_file("/spiflash/filegeneratedonhost.txt");
+
+
+#ifndef CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
+    // Create and write to a file
+    write_file("/spiflash/messagefromthedevice.txt", "This is written by the device");
+    read_file("/spiflash/messagefromthedevice.txt");
+#endif // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
+
+    // Check when the file was last modified
+    stat_file("/spiflash/subdirectoryfromhost/innerfile.txt");
+
+    // Unmount FATFS
+    ESP_LOGI(TAG, "Unmounting FAT filesystem");
+
+#ifdef CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
+        ESP_ERROR_CHECK(esp_vfs_fat_spiflash_unmount_ro(base_path, "storage"));
+#else // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
+        ESP_ERROR_CHECK(esp_vfs_fat_spiflash_unmount_rw_wl(base_path, wl_handle));
+#endif // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
+
+    ESP_LOGI(TAG, "Done");
+}

examples/storage/fatfs_advanced/partitions_example.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+nvs,      data, nvs,     0x9000,  0x6000,
+phy_init, data, phy,     0xf000,  0x1000,
+factory,  app,  factory, 0x10000, 1M,
+storage,  data, fat,     ,        1M,

examples/storage/fatfs_advanced/pytest_fatfs_advanced_example.py

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import re
+from datetime import datetime
+from typing import List
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32
+@pytest.mark.generic
+@pytest.mark.parametrize(
+    'config',
+    [
+        'test_read_only_partition_gen',
+        'test_read_only_partition_gen_default_dt',
+        'test_read_write_partition_gen',
+        'test_read_write_partition_gen_default_dt',
+    ],
+    indirect=True,
+)
+def test_examples_fatfs_advanced(config: str, dut: Dut) -> None:
+    # Expects list of strings sequentially
+    def expect_all(msg_list: List[str], to: int) -> None:
+        for msg in msg_list:
+            dut.expect(msg, timeout=to)
+
+    # Expects prefix string followed by date in the format 'yyyy-mm-dd'
+    def expect_date(prefix: str, to: int) -> datetime:
+        expect_str = prefix + '(\\d+)-(\\d+)-(\\d+)'
+        match_ = dut.expect(re.compile(str.encode(expect_str)), timeout=to)
+        year_ = int(match_[1].decode())
+        month_ = int(match_[2].decode())
+        day_ = int(match_[3].decode())
+        return datetime(year_, month_, day_)
+
+    # Calculates absolute difference in days between date_reference and date_actual.
+    # Raises exception if difference exceeds tolerance
+    def evaluate_dates(
+        date_reference: datetime, date_actual: datetime, days_tolerance: int
+    ) -> None:
+        td = date_actual - date_reference
+        if abs(td.days) > days_tolerance:
+            raise Exception(
+                f'Too big date difference. Actual: {date_actual}, reference: {date_reference}, tolerance: {days_tolerance} day(s)'
+            )
+
+    # Expect timeout
+    timeout = 20
+
+    # We tolerate 30 days difference between actual file creation and date when test was executed.
+    tolerance = 30
+
+    expected_date = (
+        datetime(1980, 1, 1) if config.endswith('default_dt') else datetime.today()
+    )
+
+    base_path = '/spiflash/'
+    folder_name = 'subdirectoryfromhost/'
+    read_filename = base_path + 'filegeneratedonhost.txt'
+    write_filename = base_path + 'messagefromthedevice.txt'
+    stat_filename = base_path + folder_name + 'innerfile.txt'
+
+    dut.expect('example: Mounting FAT filesystem', timeout=timeout)
+
+    # Check read
+    expect_all(
+        [
+            f'example: Opening file \'{read_filename}\' for reading',
+            f'example: Reading from file',
+            'example: Read from file: \'This is generated on the host; it has long name\'',
+            f'example: Closing file \'{read_filename}\'',
+        ],
+        timeout,
+    )
+
+    # Check write
+    if config.startswith('test_read_write'):
+        expect_all(
+            [
+                f'example: Opening file \'{write_filename}\' for writing',
+                'example: Writing to file',
+                'example: File written',
+                'example: Closing file',
+                f'example: Opening file \'{write_filename}\' for reading',
+                f'example: Reading from file',
+                'example: Read from file: \'This is written by the device\'',
+                f'example: Closing file \'{write_filename}\'',
+            ],
+            timeout,
+        )
+
+    # Check date using stat
+    dut.expect(
+        f'example: Stating file \'{stat_filename}\' for modification time', timeout=timeout
+    )
+
+    actual_date = expect_date(
+        f'The file \'{stat_filename}\' was modified at date: ', timeout
+    )
+
+    evaluate_dates(expected_date, actual_date, tolerance)
+
+    # Finish
+    expect_all(
+        [
+            'example: Unmounting FAT filesystem',
+            'example: Done',
+        ],
+        timeout,
+    )

examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen

@@ -0,0 +1,5 @@
+CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
+CONFIG_FATFS_LFN_HEAP=y
+CONFIG_FATFS_LFN_STACK=n
+CONFIG_FATFS_LFN_NONE=n
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen_default_dt

@@ -0,0 +1,6 @@
+CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
+CONFIG_FATFS_LFN_HEAP=y
+CONFIG_FATFS_LFN_STACK=n
+CONFIG_FATFS_LFN_NONE=n
+CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen

@@ -0,0 +1,5 @@
+CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
+CONFIG_FATFS_LFN_HEAP=y
+CONFIG_FATFS_LFN_STACK=n
+CONFIG_FATFS_LFN_NONE=n
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen_default_dt

@@ -0,0 +1,6 @@
+CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
+CONFIG_FATFS_LFN_HEAP=y
+CONFIG_FATFS_LFN_STACK=n
+CONFIG_FATFS_LFN_NONE=n
+CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
+CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

examples/storage/fatfs_advanced/sdkconfig.defaults

@@ -0,0 +1,7 @@
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_example.csv"
+CONFIG_PARTITION_TABLE_FILENAME="partitions_example.csv"
+CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
+CONFIG_FATFS_LFN_HEAP=y
+CONFIG_FATFS_LFN_NONE=n
+CONFIG_FATFS_LFN_STACK=n