From f633839eceffd424c20dacbec8f597ed46d1294e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Rohl=C3=ADnek?= Date: Mon, 3 Jun 2024 08:22:52 +0200 Subject: [PATCH] feat(storage/fatfs): add advanced example for fatfs --- examples/storage/.build-test-rules.yml | 8 + examples/storage/README.md | 1 + .../storage/fatfs_advanced/CMakeLists.txt | 6 + examples/storage/fatfs_advanced/README.md | 69 +++++++++ .../filegeneratedonhost.txt | 1 + .../subdirectoryfromhost/innerfile.txt | 1 + .../fatfs_advanced/main/CMakeLists.txt | 26 ++++ .../fatfs_advanced/main/Kconfig.projbuild | 24 +++ .../main/fatfs_advanced_example_main.c | 138 ++++++++++++++++++ .../fatfs_advanced/partitions_example.csv | 6 + .../pytest_fatfs_advanced_example.py | 112 ++++++++++++++ .../sdkconfig.ci.test_read_only_partition_gen | 5 + ...ci.test_read_only_partition_gen_default_dt | 6 + ...sdkconfig.ci.test_read_write_partition_gen | 5 + ...i.test_read_write_partition_gen_default_dt | 6 + .../storage/fatfs_advanced/sdkconfig.defaults | 7 + 16 files changed, 421 insertions(+) create mode 100644 examples/storage/fatfs_advanced/CMakeLists.txt create mode 100644 examples/storage/fatfs_advanced/README.md create mode 100644 examples/storage/fatfs_advanced/fatfs_long_name_image/filegeneratedonhost.txt create mode 100644 examples/storage/fatfs_advanced/fatfs_long_name_image/subdirectoryfromhost/innerfile.txt create mode 100644 examples/storage/fatfs_advanced/main/CMakeLists.txt create mode 100644 examples/storage/fatfs_advanced/main/Kconfig.projbuild create mode 100644 examples/storage/fatfs_advanced/main/fatfs_advanced_example_main.c create mode 100644 examples/storage/fatfs_advanced/partitions_example.csv create mode 100644 examples/storage/fatfs_advanced/pytest_fatfs_advanced_example.py create mode 100644 examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen create mode 100644 examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen_default_dt create mode 100644 examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen create mode 100644 examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen_default_dt create mode 100644 examples/storage/fatfs_advanced/sdkconfig.defaults diff --git a/examples/storage/.build-test-rules.yml b/examples/storage/.build-test-rules.yml index a15e95e333..5999ae95ea 100644 --- a/examples/storage/.build-test-rules.yml +++ b/examples/storage/.build-test-rules.yml @@ -34,6 +34,14 @@ examples/storage/ext_flash_fatfs: temporary: true reason: lack of runners +examples/storage/fatfs_advanced: + depends_components: + - fatfs + - vfs + disable_test: + - if: IDF_TARGET != "esp32" + reason: only one target needed + examples/storage/fatfs_basic: depends_components: - fatfs diff --git a/examples/storage/README.md b/examples/storage/README.md index ade0b883a1..ebfb75833d 100644 --- a/examples/storage/README.md +++ b/examples/storage/README.md @@ -8,6 +8,7 @@ This directory contains a range of examples ESP-IDF projects. These are intended The examples are grouped into sub-directories by category. Each category directory contains one or more example projects: * `fatfs_basic` minimal example of FatFS usage on SPI FLASH +* `fatfs_advanced` example demonstrates how to use advanced features for working with FatFS such as automatic partition generation * `custom_flash_driver` example demonstrates how to implement your own flash chip driver by overriding the default driver. * `emmc` example demonstrates how to use an eMMC chip with an ESP device. * `ext_flash_fatfs` example demonstrates how to use FATFS partition with external SPI FLASH chip. diff --git a/examples/storage/fatfs_advanced/CMakeLists.txt b/examples/storage/fatfs_advanced/CMakeLists.txt new file mode 100644 index 0000000000..1c0c2b8a1a --- /dev/null +++ b/examples/storage/fatfs_advanced/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.16) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(fatfsgen) diff --git a/examples/storage/fatfs_advanced/README.md b/examples/storage/fatfs_advanced/README.md new file mode 100644 index 0000000000..34ab038fc5 --- /dev/null +++ b/examples/storage/fatfs_advanced/README.md @@ -0,0 +1,69 @@ +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | + +# FATFS partition generation example + +(See the README.md file in the upper level 'examples' directory for more information about examples.) + +This example demonstrates how to use the FATFS partition +generation tool [fatfsgen.py](../../../components/fatfs/fatfsgen.py) to automatically create a FATFS +filesystem image from the contents of a host folder during build, with an option of +automatically flashing the created image on invocation of `idf.py -p PORT flash` as well as usage of long file names for FATFS. +Beware that the minimal required size of the flash is 4 MB. +You can specify using menuconfig weather example will use read-only or read-write mode. The default option is read-write mode. +To change it just use menuconfig: + +```shell +idf.py menuconfig +``` + +Then select `Example Configuration` a chose `Mode for generated FATFS image` either `Read-Write Mode` or `Read-Only Mode`. +`Read-Only` option indicates generating raw fatfs image without wear levelling support. +On the other hand, for `Read-Write` the generated fatfs image will support wear levelling thus can be mounted in read-write mode. + + +The following gives an overview of the example: + +1. There is a directory `fatfs_image` from which the FATFS filesystem image will be created. + +2. The function `fatfs_create_rawflash_image` is used to specify that a FATFS image +should be created during build for the `storage` partition. +For CMake, it is called from [the main component's CMakeLists.txt](./main/CMakeLists.txt). +`FLASH_IN_PROJECT` specifies that the created image +should be flashed on invocation of `idf.py -p PORT flash` together with app, bootloader, partition table, etc. +The image is created on the example's build directory with the output filename `storage.bin`. + +3. Upon invocation of `idf.py -p PORT flash monitor`, application loads and +finds there is already a valid FATFS filesystem in the `storage` partition with files same as those in `fatfs_image` directory. The application is then +able to read those files. + +## How to use example + +### Build and flash + +To run the example, type the following command: + +```CMake +# CMake +idf.py -p PORT flash monitor +``` + +(To exit the serial monitor, type ``Ctrl-]``.) + +See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects. + +## Example output + +Here is the example's console output: + +``` +... +I (322) example: Mounting FAT filesystem +I (332) example: Reading file +I (332) example: Read from file: 'this is test' +I (332) example: Unmounting FAT filesystem +I (342) example: Done +``` + +The logic of the example is contained in a [single source file](./main/fatfsgen_example_main.c), +and it should be relatively simple to match points in its execution with the log outputs above. diff --git a/examples/storage/fatfs_advanced/fatfs_long_name_image/filegeneratedonhost.txt b/examples/storage/fatfs_advanced/fatfs_long_name_image/filegeneratedonhost.txt new file mode 100644 index 0000000000..33f47c9b1d --- /dev/null +++ b/examples/storage/fatfs_advanced/fatfs_long_name_image/filegeneratedonhost.txt @@ -0,0 +1 @@ +This is generated on the host; it has long name diff --git a/examples/storage/fatfs_advanced/fatfs_long_name_image/subdirectoryfromhost/innerfile.txt b/examples/storage/fatfs_advanced/fatfs_long_name_image/subdirectoryfromhost/innerfile.txt new file mode 100644 index 0000000000..84505e3d3e --- /dev/null +++ b/examples/storage/fatfs_advanced/fatfs_long_name_image/subdirectoryfromhost/innerfile.txt @@ -0,0 +1 @@ +this is test; it has long name diff --git a/examples/storage/fatfs_advanced/main/CMakeLists.txt b/examples/storage/fatfs_advanced/main/CMakeLists.txt new file mode 100644 index 0000000000..28eed4c7c3 --- /dev/null +++ b/examples/storage/fatfs_advanced/main/CMakeLists.txt @@ -0,0 +1,26 @@ +idf_component_register(SRCS "fatfs_advanced_example_main.c" + INCLUDE_DIRS ".") + +# Create a FATFS image from the contents of the 'fatfs_image' directory +# that fits the partition named 'storage'. FLASH_IN_PROJECT indicates that +# the generated image should be flashed when the entire project is flashed to +# the target with 'idf.py -p PORT flash'. +# If read-only mode is set (CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY) +# the generated image will be raw without wear levelling support. +# Otherwise it will support wear levelling and thus enable read-write mounting of the image in the device. + +set(image ../fatfs_long_name_image) + +if(CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY) + if(CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME) + fatfs_create_rawflash_image(storage ${image} FLASH_IN_PROJECT) + else() + fatfs_create_rawflash_image(storage ${image} FLASH_IN_PROJECT PRESERVE_TIME) + endif() +else() + if(CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME) + fatfs_create_spiflash_image(storage ${image} FLASH_IN_PROJECT) + else() + fatfs_create_spiflash_image(storage ${image} FLASH_IN_PROJECT PRESERVE_TIME) + endif() +endif() diff --git a/examples/storage/fatfs_advanced/main/Kconfig.projbuild b/examples/storage/fatfs_advanced/main/Kconfig.projbuild new file mode 100644 index 0000000000..b715c917b8 --- /dev/null +++ b/examples/storage/fatfs_advanced/main/Kconfig.projbuild @@ -0,0 +1,24 @@ +menu "Example Configuration" + + config EXAMPLE_FATFS_MODE_READ_ONLY + bool "Read only mode for generated FATFS image" + default n + help + If read-only mode is set, the generated fatfs image will be raw (without wear levelling support). + Otherwise it will support wear levelling that enables read-write mounting. + + config EXAMPLE_FATFS_WRITE_COUNT + int "Number of volumes" + default 1 + range 1 600 + help + Number of writes to the file (for testing purposes). + + config EXAMPLE_FATFS_DEFAULT_DATETIME + bool "Default modification date and time for generated FATFS image" + default n + help + If default datetime is set, all files created in the generated FATFS partition have default time + equal to FATFS origin time (1 January 1980) + +endmenu diff --git a/examples/storage/fatfs_advanced/main/fatfs_advanced_example_main.c b/examples/storage/fatfs_advanced/main/fatfs_advanced_example_main.c new file mode 100644 index 0000000000..3d8032b163 --- /dev/null +++ b/examples/storage/fatfs_advanced/main/fatfs_advanced_example_main.c @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ + +#include +#include +#include +#include +#include "esp_vfs.h" +#include "esp_vfs_fat.h" +#include "sdkconfig.h" + +static const char *TAG = "example"; + +// Mount path for the partition +const char *base_path = "/spiflash"; + +void write_file(const char *filename, const char *data) +{ + ESP_LOGI(TAG, "Opening file '%s' for writing", filename); + + FILE *f = fopen(filename, "w"); + if (f == NULL) { + ESP_LOGE(TAG, "Failed to open file for writing: %s", strerror(errno)); + return; + } + + ESP_LOGI(TAG, "Writing to file"); + + fputs(data, f); + + ESP_LOGI(TAG, "File written"); + + ESP_LOGI(TAG, "Closing file"); + fclose(f); +} + +void read_file(const char *filename) +{ + ESP_LOGI(TAG, "Opening file '%s' for reading", filename); + + FILE *f = fopen(filename, "r"); + if (f == NULL) { + ESP_LOGE(TAG, "Failed to open file for reading: %s", strerror(errno)); + return; + } + + ESP_LOGI(TAG, "Reading from file"); + + char line[128]; + fgets(line, sizeof(line), f); + + // strip newline + char *pos = strchr(line, '\n'); + if (pos) { + *pos = '\0'; // strip newline + } + + ESP_LOGI(TAG, "Read from file: '%s'", line); + + ESP_LOGI(TAG, "Closing file '%s'", filename); + fclose(f); +} + +void stat_file(const char *filename) +{ + + struct stat info; + struct tm timeinfo; + char buffer[32]; + + ESP_LOGI(TAG, "Stating file '%s' for modification time", filename); + + if(stat(filename, &info) < 0){ + ESP_LOGE(TAG, "Failed to read file stats: %s", strerror(errno)); + return; + } + localtime_r(&info.st_mtime, &timeinfo); + strftime(buffer, sizeof(buffer), "%Y-%m-%d", &timeinfo); + + ESP_LOGI(TAG, "The file '%s' was modified at date: %s", filename, buffer); +} + +void app_main(void) +{ + ESP_LOGI(TAG, "Mounting FAT filesystem"); + + // To mount device we need name of device partition, define base_path + // and allow format partition in case if it is new one and was not formatted before + const esp_vfs_fat_mount_config_t mount_config = { + .max_files = 4, + .format_if_mount_failed = false, + .allocation_unit_size = CONFIG_WL_SECTOR_SIZE, + .use_one_fat = false, + }; + + esp_err_t err; + +#ifdef CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY + err = esp_vfs_fat_spiflash_mount_ro(base_path, "storage", &mount_config); +#else // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY + + // Handle of the wear levelling library instance + wl_handle_t wl_handle = WL_INVALID_HANDLE; + err = esp_vfs_fat_spiflash_mount_rw_wl(base_path, "storage", &mount_config, &wl_handle); +#endif // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY + + if (err != ESP_OK) { + ESP_LOGE(TAG, "Failed to mount FATFS (%s)python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps", esp_err_to_name(err)); + return; + } + + // Read contents of a file + read_file("/spiflash/filegeneratedonhost.txt"); + + +#ifndef CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY + // Create and write to a file + write_file("/spiflash/messagefromthedevice.txt", "This is written by the device"); + read_file("/spiflash/messagefromthedevice.txt"); +#endif // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY + + // Check when the file was last modified + stat_file("/spiflash/subdirectoryfromhost/innerfile.txt"); + + // Unmount FATFS + ESP_LOGI(TAG, "Unmounting FAT filesystem"); + +#ifdef CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY + ESP_ERROR_CHECK(esp_vfs_fat_spiflash_unmount_ro(base_path, "storage")); +#else // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY + ESP_ERROR_CHECK(esp_vfs_fat_spiflash_unmount_rw_wl(base_path, wl_handle)); +#endif // CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY + + ESP_LOGI(TAG, "Done"); +} diff --git a/examples/storage/fatfs_advanced/partitions_example.csv b/examples/storage/fatfs_advanced/partitions_example.csv new file mode 100644 index 0000000000..1c79321a10 --- /dev/null +++ b/examples/storage/fatfs_advanced/partitions_example.csv @@ -0,0 +1,6 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 1M, +storage, data, fat, , 1M, diff --git a/examples/storage/fatfs_advanced/pytest_fatfs_advanced_example.py b/examples/storage/fatfs_advanced/pytest_fatfs_advanced_example.py new file mode 100644 index 0000000000..ea7f491a8f --- /dev/null +++ b/examples/storage/fatfs_advanced/pytest_fatfs_advanced_example.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Unlicense OR CC0-1.0 +import re +from datetime import datetime +from typing import List + +import pytest +from pytest_embedded import Dut + + +@pytest.mark.esp32 +@pytest.mark.generic +@pytest.mark.parametrize( + 'config', + [ + 'test_read_only_partition_gen', + 'test_read_only_partition_gen_default_dt', + 'test_read_write_partition_gen', + 'test_read_write_partition_gen_default_dt', + ], + indirect=True, +) +def test_examples_fatfs_advanced(config: str, dut: Dut) -> None: + # Expects list of strings sequentially + def expect_all(msg_list: List[str], to: int) -> None: + for msg in msg_list: + dut.expect(msg, timeout=to) + + # Expects prefix string followed by date in the format 'yyyy-mm-dd' + def expect_date(prefix: str, to: int) -> datetime: + expect_str = prefix + '(\\d+)-(\\d+)-(\\d+)' + match_ = dut.expect(re.compile(str.encode(expect_str)), timeout=to) + year_ = int(match_[1].decode()) + month_ = int(match_[2].decode()) + day_ = int(match_[3].decode()) + return datetime(year_, month_, day_) + + # Calculates absolute difference in days between date_reference and date_actual. + # Raises exception if difference exceeds tolerance + def evaluate_dates( + date_reference: datetime, date_actual: datetime, days_tolerance: int + ) -> None: + td = date_actual - date_reference + if abs(td.days) > days_tolerance: + raise Exception( + f'Too big date difference. Actual: {date_actual}, reference: {date_reference}, tolerance: {days_tolerance} day(s)' + ) + + # Expect timeout + timeout = 20 + + # We tolerate 30 days difference between actual file creation and date when test was executed. + tolerance = 30 + + expected_date = ( + datetime(1980, 1, 1) if config.endswith('default_dt') else datetime.today() + ) + + base_path = '/spiflash/' + folder_name = 'subdirectoryfromhost/' + read_filename = base_path + 'filegeneratedonhost.txt' + write_filename = base_path + 'messagefromthedevice.txt' + stat_filename = base_path + folder_name + 'innerfile.txt' + + dut.expect('example: Mounting FAT filesystem', timeout=timeout) + + # Check read + expect_all( + [ + f'example: Opening file \'{read_filename}\' for reading', + f'example: Reading from file', + 'example: Read from file: \'This is generated on the host; it has long name\'', + f'example: Closing file \'{read_filename}\'', + ], + timeout, + ) + + # Check write + if config.startswith('test_read_write'): + expect_all( + [ + f'example: Opening file \'{write_filename}\' for writing', + 'example: Writing to file', + 'example: File written', + 'example: Closing file', + f'example: Opening file \'{write_filename}\' for reading', + f'example: Reading from file', + 'example: Read from file: \'This is written by the device\'', + f'example: Closing file \'{write_filename}\'', + ], + timeout, + ) + + # Check date using stat + dut.expect( + f'example: Stating file \'{stat_filename}\' for modification time', timeout=timeout + ) + + actual_date = expect_date( + f'The file \'{stat_filename}\' was modified at date: ', timeout + ) + + evaluate_dates(expected_date, actual_date, tolerance) + + # Finish + expect_all( + [ + 'example: Unmounting FAT filesystem', + 'example: Done', + ], + timeout, + ) diff --git a/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen b/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen new file mode 100644 index 0000000000..21a7938068 --- /dev/null +++ b/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen @@ -0,0 +1,5 @@ +CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y +CONFIG_FATFS_LFN_HEAP=y +CONFIG_FATFS_LFN_STACK=n +CONFIG_FATFS_LFN_NONE=n +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen_default_dt b/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen_default_dt new file mode 100644 index 0000000000..ed0be2575b --- /dev/null +++ b/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_only_partition_gen_default_dt @@ -0,0 +1,6 @@ +CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y +CONFIG_FATFS_LFN_HEAP=y +CONFIG_FATFS_LFN_STACK=n +CONFIG_FATFS_LFN_NONE=n +CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen b/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen new file mode 100644 index 0000000000..4ffa553233 --- /dev/null +++ b/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen @@ -0,0 +1,5 @@ +CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n +CONFIG_FATFS_LFN_HEAP=y +CONFIG_FATFS_LFN_STACK=n +CONFIG_FATFS_LFN_NONE=n +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen_default_dt b/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen_default_dt new file mode 100644 index 0000000000..6413768f3e --- /dev/null +++ b/examples/storage/fatfs_advanced/sdkconfig.ci.test_read_write_partition_gen_default_dt @@ -0,0 +1,6 @@ +CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n +CONFIG_FATFS_LFN_HEAP=y +CONFIG_FATFS_LFN_STACK=n +CONFIG_FATFS_LFN_NONE=n +CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y +CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300 diff --git a/examples/storage/fatfs_advanced/sdkconfig.defaults b/examples/storage/fatfs_advanced/sdkconfig.defaults new file mode 100644 index 0000000000..e5ac937533 --- /dev/null +++ b/examples/storage/fatfs_advanced/sdkconfig.defaults @@ -0,0 +1,7 @@ +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_example.csv" +CONFIG_PARTITION_TABLE_FILENAME="partitions_example.csv" +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_FATFS_LFN_HEAP=y +CONFIG_FATFS_LFN_NONE=n +CONFIG_FATFS_LFN_STACK=n