From 8815a3dfa01eb752b441d18ad4c6a09aad65f0f5 Mon Sep 17 00:00:00 2001 From: Felipe Neves Date: Mon, 3 Aug 2020 14:26:59 -0300 Subject: [PATCH 1/4] fatfs: expose FF_USE_FASTSEEK option --- components/fatfs/Kconfig | 12 ++++++++++++ components/fatfs/src/ffconf.h | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/components/fatfs/Kconfig b/components/fatfs/Kconfig index 31dd4f2783..3dda859da4 100644 --- a/components/fatfs/Kconfig +++ b/components/fatfs/Kconfig @@ -180,4 +180,16 @@ menu "FAT Filesystem support" Disable this option if optimizing for performance. Enable this option if optimizing for internal memory size. + + config FATFS_USE_FASTSEEK + bool "Enable fast seek algorithm when using f_lseek function" + default n + help + The fast seek feature enables fast backward/long seek operations without + FAT access by using an in-memory CLMT (cluster link map table). + It is applied to f_read and f_write function as well, however, + the file size cannot be expanded by f_write. + f_lseek function while the file is at fast seek mode. + + endmenu diff --git a/components/fatfs/src/ffconf.h b/components/fatfs/src/ffconf.h index 79792beef2..6578122c5f 100644 --- a/components/fatfs/src/ffconf.h +++ b/components/fatfs/src/ffconf.h @@ -44,7 +44,7 @@ /* This option switches f_mkfs() function. (0:Disable or 1:Enable) */ -#define FF_USE_FASTSEEK 0 +#define FF_USE_FASTSEEK CONFIG_FATFS_USE_FASTSEEK /* This option switches fast seek function. (0:Disable or 1:Enable) */ From 32e760adfb2b35512e9e06759ca3b617e663aa3f Mon Sep 17 00:00:00 2001 From: Felipe Neves Date: Mon, 31 Aug 2020 17:57:30 -0300 Subject: [PATCH 2/4] fat_vfs: added fast seek capable file operation --- components/fatfs/Kconfig | 15 +++++++-- components/fatfs/vfs/vfs_fat.c | 37 +++++++++++++++++++++ tools/ci/config/target-test.yml | 1 + tools/unit-test-app/configs/fatfs_fast_seek | 3 ++ 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 tools/unit-test-app/configs/fatfs_fast_seek diff --git a/components/fatfs/Kconfig b/components/fatfs/Kconfig index 3dda859da4..9cb067c7c6 100644 --- a/components/fatfs/Kconfig +++ b/components/fatfs/Kconfig @@ -182,14 +182,23 @@ menu "FAT Filesystem support" config FATFS_USE_FASTSEEK - bool "Enable fast seek algorithm when using f_lseek function" + bool "Enable fast seek algorithm when using lseek function through VFS FAT" default n help The fast seek feature enables fast backward/long seek operations without FAT access by using an in-memory CLMT (cluster link map table). It is applied to f_read and f_write function as well, however, - the file size cannot be expanded by f_write. - f_lseek function while the file is at fast seek mode. + the file size cannot be expanded by write function. + config FATFS_FAST_SEEK_BUFFER_SIZE + int "Fast seek CLMT buffer size" + default 64 + depends on FATFS_USE_FASTSEEK + help + If fast seek algorithm is enabled, this defines the size of + CLMT buffer used by this algorithm in 32-bit word units. + This value should be chosen based on prior knowledge of + maximum elements of each file entry would store. + endmenu diff --git a/components/fatfs/vfs/vfs_fat.c b/components/fatfs/vfs/vfs_fat.c index 102cb9602b..34de1bad94 100644 --- a/components/fatfs/vfs/vfs_fat.c +++ b/components/fatfs/vfs/vfs_fat.c @@ -307,6 +307,7 @@ static int vfs_fat_open(void* ctx, const char * path, int flags, int mode) errno = ENFILE; return -1; } + FRESULT res = f_open(&fat_ctx->files[fd], path, fat_mode_conv(flags)); if (res != FR_OK) { file_cleanup(fat_ctx, fd); @@ -315,6 +316,34 @@ static int vfs_fat_open(void* ctx, const char * path, int flags, int mode) errno = fresult_to_errno(res); return -1; } + +#ifdef CONFIG_FATFS_USE_FASTSEEK + FIL* file = &fat_ctx->files[fd]; + DWORD *clmt_mem = ff_memalloc(sizeof(DWORD) * CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE); + if (clmt_mem == NULL) { + f_close(file); + file_cleanup(fat_ctx, fd); + _lock_release(&fat_ctx->lock); + ESP_LOGE(TAG, "open: Failed to pre-allocate CLMT buffer for fast-seek"); + errno = ENOMEM; + return -1; + } + + file->cltbl = clmt_mem; + file->cltbl[0] = CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE; + res = f_lseek(file, CREATE_LINKMAP); + ESP_LOGD(TAG, "%s: fast-seek has: %s", + __func__, + (res == FR_OK) ? "activated" : "failed"); + if(res != FR_OK) { + ESP_LOGW(TAG, "%s: fast-seek not activated reason code: %d", + __func__, res); + //If linkmap creation fails, fallback to the non fast seek. + ff_memfree(file->cltbl); + file->cltbl = NULL; + } +#endif + // O_APPEND need to be stored because it is not compatible with FA_OPEN_APPEND: // - FA_OPEN_APPEND means to jump to the end of file only after open() // - O_APPEND means to jump to the end only before each write() @@ -465,6 +494,11 @@ static int vfs_fat_close(void* ctx, int fd) vfs_fat_ctx_t* fat_ctx = (vfs_fat_ctx_t*) ctx; _lock_acquire(&fat_ctx->lock); FIL* file = &fat_ctx->files[fd]; + +#ifdef CONFIG_FATFS_USE_FASTSEEK + ff_memfree(file->cltbl); +#endif + FRESULT res = f_close(file); file_cleanup(fat_ctx, fd); _lock_release(&fat_ctx->lock); @@ -494,6 +528,9 @@ static off_t vfs_fat_lseek(void* ctx, int fd, off_t offset, int mode) errno = EINVAL; return -1; } + + ESP_LOGD(TAG, "%s: offset=%ld, filesize:=%d", __func__, new_pos, f_size(file)); + FRESULT res = f_lseek(file, new_pos); if (res != FR_OK) { ESP_LOGD(TAG, "%s: fresult=%d", __func__, res); diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 613e178757..e0a9c30225 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -345,6 +345,7 @@ UT_002: UT_003: extends: .unit_test_template + parallel: 2 tags: - ESP32_IDF - UT_T1_SDMODE diff --git a/tools/unit-test-app/configs/fatfs_fast_seek b/tools/unit-test-app/configs/fatfs_fast_seek new file mode 100644 index 0000000000..b0c6966f2a --- /dev/null +++ b/tools/unit-test-app/configs/fatfs_fast_seek @@ -0,0 +1,3 @@ +TEST_COMPONENTS=fatfs +CONFIG_FATFS_USE_FASTSEEK=y +CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE=64 \ No newline at end of file From 93ffc009ef8f1ed75731094e2e113539a020aa44 Mon Sep 17 00:00:00 2001 From: Felipe Neves Date: Mon, 26 Oct 2020 16:06:12 -0300 Subject: [PATCH 3/4] vfs: restrict the fast seek for read-only files Since the files under fast-seek cannot be expanded with further writes, it does not make sense to enable fast-seek which may fail in write-mode files --- components/fatfs/test/test_fatfs_common.c | 18 +++++++- components/fatfs/vfs/vfs_fat.c | 52 +++++++++++++---------- tools/ci/config/target-test.yml | 4 +- 3 files changed, 49 insertions(+), 25 deletions(-) diff --git a/components/fatfs/test/test_fatfs_common.c b/components/fatfs/test/test_fatfs_common.c index c1dd3f1319..342d55a4fd 100644 --- a/components/fatfs/test/test_fatfs_common.c +++ b/components/fatfs/test/test_fatfs_common.c @@ -217,11 +217,27 @@ void test_fatfs_lseek(const char* filename) TEST_ASSERT_EQUAL(18, ftell(f)); TEST_ASSERT_EQUAL(0, fseek(f, 0, SEEK_SET)); char buf[20]; + TEST_ASSERT_EQUAL(18, fread(buf, 1, sizeof(buf), f)); const char ref_buf[] = "0123456789\n\0\0\0abc\n"; TEST_ASSERT_EQUAL_INT8_ARRAY(ref_buf, buf, sizeof(ref_buf) - 1); - TEST_ASSERT_EQUAL(0, fclose(f)); + +#ifdef CONFIG_FATFS_USE_FASTSEEK + f = fopen(filename, "rb+"); + TEST_ASSERT_NOT_NULL(f); + TEST_ASSERT_EQUAL(0, fseek(f, 0, SEEK_END)); + TEST_ASSERT_EQUAL(18, ftell(f)); + TEST_ASSERT_EQUAL(0, fseek(f, -4, SEEK_CUR)); + TEST_ASSERT_EQUAL(14, ftell(f)); + TEST_ASSERT_EQUAL(0, fseek(f, -14, SEEK_CUR)); + TEST_ASSERT_EQUAL(0, ftell(f)); + + TEST_ASSERT_EQUAL(18, fread(buf, 1, sizeof(buf), f)); + TEST_ASSERT_EQUAL_INT8_ARRAY(ref_buf, buf, sizeof(ref_buf) - 1); + TEST_ASSERT_EQUAL(0, fclose(f)); +#endif + } void test_fatfs_truncate_file(const char* filename) diff --git a/components/fatfs/vfs/vfs_fat.c b/components/fatfs/vfs/vfs_fat.c index 34de1bad94..54bc1194b9 100644 --- a/components/fatfs/vfs/vfs_fat.c +++ b/components/fatfs/vfs/vfs_fat.c @@ -319,29 +319,35 @@ static int vfs_fat_open(void* ctx, const char * path, int flags, int mode) #ifdef CONFIG_FATFS_USE_FASTSEEK FIL* file = &fat_ctx->files[fd]; - DWORD *clmt_mem = ff_memalloc(sizeof(DWORD) * CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE); - if (clmt_mem == NULL) { - f_close(file); - file_cleanup(fat_ctx, fd); - _lock_release(&fat_ctx->lock); - ESP_LOGE(TAG, "open: Failed to pre-allocate CLMT buffer for fast-seek"); - errno = ENOMEM; - return -1; - } + //fast-seek is only allowed in read mode, since file cannot be expanded + //to use it. + if(!(fat_mode_conv(flags) & (FA_WRITE))) { + DWORD *clmt_mem = ff_memalloc(sizeof(DWORD) * CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE); + if (clmt_mem == NULL) { + f_close(file); + file_cleanup(fat_ctx, fd); + _lock_release(&fat_ctx->lock); + ESP_LOGE(TAG, "open: Failed to pre-allocate CLMT buffer for fast-seek"); + errno = ENOMEM; + return -1; + } - file->cltbl = clmt_mem; - file->cltbl[0] = CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE; - res = f_lseek(file, CREATE_LINKMAP); - ESP_LOGD(TAG, "%s: fast-seek has: %s", - __func__, - (res == FR_OK) ? "activated" : "failed"); - if(res != FR_OK) { - ESP_LOGW(TAG, "%s: fast-seek not activated reason code: %d", - __func__, res); - //If linkmap creation fails, fallback to the non fast seek. - ff_memfree(file->cltbl); + file->cltbl = clmt_mem; + file->cltbl[0] = CONFIG_FATFS_FAST_SEEK_BUFFER_SIZE; + res = f_lseek(file, CREATE_LINKMAP); + ESP_LOGD(TAG, "%s: fast-seek has: %s", + __func__, + (res == FR_OK) ? "activated" : "failed"); + if(res != FR_OK) { + ESP_LOGW(TAG, "%s: fast-seek not activated reason code: %d", + __func__, res); + //If linkmap creation fails, fallback to the non fast seek. + ff_memfree(file->cltbl); + file->cltbl = NULL; + } + } else { file->cltbl = NULL; - } + } #endif // O_APPEND need to be stored because it is not compatible with FA_OPEN_APPEND: @@ -404,6 +410,7 @@ static ssize_t vfs_fat_pread(void *ctx, int fd, void *dst, size_t size, off_t of const off_t prev_pos = f_tell(file); FRESULT f_res = f_lseek(file, offset); + if (f_res != FR_OK) { ESP_LOGD(TAG, "%s: fresult=%d", __func__, f_res); errno = fresult_to_errno(f_res); @@ -443,6 +450,7 @@ static ssize_t vfs_fat_pwrite(void *ctx, int fd, const void *src, size_t size, o const off_t prev_pos = f_tell(file); FRESULT f_res = f_lseek(file, offset); + if (f_res != FR_OK) { ESP_LOGD(TAG, "%s: fresult=%d", __func__, f_res); errno = fresult_to_errno(f_res); @@ -497,6 +505,7 @@ static int vfs_fat_close(void* ctx, int fd) #ifdef CONFIG_FATFS_USE_FASTSEEK ff_memfree(file->cltbl); + file->cltbl = NULL; #endif FRESULT res = f_close(file); @@ -530,7 +539,6 @@ static off_t vfs_fat_lseek(void* ctx, int fd, off_t offset, int mode) } ESP_LOGD(TAG, "%s: offset=%ld, filesize:=%d", __func__, new_pos, f_size(file)); - FRESULT res = f_lseek(file, new_pos); if (res != FR_OK) { ESP_LOGD(TAG, "%s: fresult=%d", __func__, res); diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index e0a9c30225..6e330cc475 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -328,7 +328,7 @@ component_ut_test_001: UT_001: extends: .unit_test_template - parallel: 43 + parallel: 44 tags: - ESP32_IDF - UT_T1_1 @@ -480,7 +480,7 @@ UT_034: UT_035: extends: .unit_test_s2_template - parallel: 44 + parallel: 45 tags: - ESP32S2_IDF - UT_T1_1 From ac6ce5ddbdf3a950daf49912060af043656f89c6 Mon Sep 17 00:00:00 2001 From: Felipe Neves Date: Thu, 5 Nov 2020 17:01:41 -0300 Subject: [PATCH 4/4] docs: added some fast-seek documentation to the fatfs.rst --- components/fatfs/Kconfig | 5 +++-- docs/en/api-reference/storage/fatfs.rst | 12 +++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/components/fatfs/Kconfig b/components/fatfs/Kconfig index 9cb067c7c6..b14a214668 100644 --- a/components/fatfs/Kconfig +++ b/components/fatfs/Kconfig @@ -187,8 +187,9 @@ menu "FAT Filesystem support" help The fast seek feature enables fast backward/long seek operations without FAT access by using an in-memory CLMT (cluster link map table). - It is applied to f_read and f_write function as well, however, - the file size cannot be expanded by write function. + Please note, fast-seek is only allowed for read-mode files, if a + file is opened in write-mode, the seek mechanism will automatically fallback + to the default implementation. config FATFS_FAST_SEEK_BUFFER_SIZE diff --git a/docs/en/api-reference/storage/fatfs.rst b/docs/en/api-reference/storage/fatfs.rst index 6225c076d4..9333fd8993 100644 --- a/docs/en/api-reference/storage/fatfs.rst +++ b/docs/en/api-reference/storage/fatfs.rst @@ -29,15 +29,17 @@ Most applications use the following workflow when working with ``esp_vfs_fat_`` 4. Call the C standard library and POSIX API functions to perform such actions on files as open, read, write, erase, copy, etc. Use paths starting with the path prefix passed to :cpp:func:`esp_vfs_register` (for example, ``"/sdcard/hello.txt"``). -5. Optionally, call the FatFs library functions directly. In this case, use paths without a VFS prefix (for example, ``"/hello.txt"``). +5. Optionally, by enabling the option `CONFIG_FATFS_USE_FASTSEEK`, use the POSIX lseek function to perform it faster, the fast seek will not work for files in write mode, so to take advantage of fast seek, you should open (or close and then reopen) the file in read-only mode. -6. Close all open files. +6. Optionally, call the FatFs library functions directly. In this case, use paths without a VFS prefix (for example, ``"/hello.txt"``). -7. Call the FatFs function ``f_mount`` for the same drive number, with NULL ``FATFS*`` argument, to unmount the filesystem. +7. Close all open files. -8. Call the FatFs function :cpp:func:`ff_diskio_register` with NULL ``ff_diskio_impl_t*`` argument and the same drive number to unregister the disk I/O driver. +8. Call the FatFs function ``f_mount`` for the same drive number, with NULL ``FATFS*`` argument, to unmount the filesystem. -9. Call :cpp:func:`esp_vfs_fat_unregister_path` with the path where the file system is mounted to remove FatFs from VFS, and free the ``FATFS`` structure allocated in Step 1. +9. Call the FatFs function :cpp:func:`ff_diskio_register` with NULL ``ff_diskio_impl_t*`` argument and the same drive number to unregister the disk I/O driver. + +10. Call :cpp:func:`esp_vfs_fat_unregister_path` with the path where the file system is mounted to remove FatFs from VFS, and free the ``FATFS`` structure allocated in Step 1. The convenience functions ``esp_vfs_fat_sdmmc_mount``, ``esp_vfs_fat_sdspi_mount`` and ``esp_vfs_fat_sdcard_unmount`` wrap the steps described above and also handle SD card initialization. These two functions are described in the next section.