From d8ebca368cffb389b77c63d6cdc6aecc0bcb2be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20M=C3=BAdry?= Date: Wed, 5 Feb 2025 15:51:48 +0100 Subject: [PATCH 1/4] feat(nvs): Allow read-only NVS partitions smaller than 0x3000 E.g. for factory settings data Closes https://github.com/espressif/esp-idf/issues/15317 --- .../host_test/nvs_page_test/main/test_fixtures.hpp | 2 +- components/nvs_flash/src/nvs_pagemanager.cpp | 4 ++-- components/partition_table/gen_esp32part.py | 11 +++++++++-- docs/en/api-guides/partition-tables.rst | 1 + docs/en/api-reference/storage/nvs_flash.rst | 8 ++++++++ examples/storage/parttool/partitions_example.csv | 2 +- 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/components/nvs_flash/host_test/nvs_page_test/main/test_fixtures.hpp b/components/nvs_flash/host_test/nvs_page_test/main/test_fixtures.hpp index 6561dc6458..5a50ba65fe 100644 --- a/components/nvs_flash/host_test/nvs_page_test/main/test_fixtures.hpp +++ b/components/nvs_flash/host_test/nvs_page_test/main/test_fixtures.hpp @@ -110,7 +110,7 @@ public: size_t columns = size / column_size; size_t column; - for(column = 0; column < columns; column = column + 1) + for(column = 0; column < columns; ++column) { // read column if((err = esp_partition_read_raw(&esp_partition, dst_offset + (column * column_size), buff, column_size)) != ESP_OK) return err; diff --git a/components/nvs_flash/src/nvs_pagemanager.cpp b/components/nvs_flash/src/nvs_pagemanager.cpp index 9bbe32c98a..39017464ea 100644 --- a/components/nvs_flash/src/nvs_pagemanager.cpp +++ b/components/nvs_flash/src/nvs_pagemanager.cpp @@ -124,8 +124,8 @@ esp_err_t PageManager::load(Partition *partition, uint32_t baseSector, uint32_t } } - // partition should have at least one free page - if (mFreePageList.empty()) { + // partition should have at least one free page if it is not read-only + if (!partition->get_readonly() && mFreePageList.empty()) { return ESP_ERR_NVS_NO_FREE_PAGES; } diff --git a/components/partition_table/gen_esp32part.py b/components/partition_table/gen_esp32part.py index cf6993b9ae..05f8233297 100755 --- a/components/partition_table/gen_esp32part.py +++ b/components/partition_table/gen_esp32part.py @@ -7,7 +7,7 @@ # See https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/partition-tables.html # for explanation of partition table structure and uses. # -# SPDX-FileCopyrightText: 2016-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2016-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import binascii @@ -31,7 +31,7 @@ SECURE_NONE = None SECURE_V1 = 'v1' SECURE_V2 = 'v2' -__version__ = '1.4' +__version__ = '1.5' APP_TYPE = 0x00 DATA_TYPE = 0x01 @@ -45,6 +45,8 @@ TYPES = { 'data': DATA_TYPE, } +NVS_RW_MIN_PARTITION_SIZE = 0x3000 + def get_ptype_as_int(ptype): """ Convert a string which might be numeric or the name of a partition type to an integer """ @@ -533,6 +535,11 @@ class PartitionDefinition(object): raise ValidationError(self, "'%s' partition of type %s and subtype %s is always read-write and cannot be read-only" % (self.name, self.type, self.subtype)) + if self.type == TYPES['data'] and self.subtype == SUBTYPES[DATA_TYPE]['nvs']: + if self.size < NVS_RW_MIN_PARTITION_SIZE and self.readonly is False: + raise ValidationError(self, """'%s' partition of type %s and subtype %s of this size (0x%x) must be flagged as 'readonly' \ +(the size of read/write NVS has to be at least 0x%x)""" % (self.name, self.type, self.subtype, self.size, NVS_RW_MIN_PARTITION_SIZE)) + STRUCT_FORMAT = b'<2sBBLL16sL' @classmethod diff --git a/docs/en/api-guides/partition-tables.rst b/docs/en/api-guides/partition-tables.rst index b53eea158c..e2d1cd3220 100644 --- a/docs/en/api-guides/partition-tables.rst +++ b/docs/en/api-guides/partition-tables.rst @@ -176,6 +176,7 @@ See enum :cpp:type:`esp_partition_subtype_t` for the full list of subtypes defin - The NVS API can also be used for other application data. - It is strongly recommended that you include an NVS partition of at least 0x3000 bytes in your project. - If using NVS API to store a lot of data, increase the NVS partition size from the default 0x6000 bytes. + - When NVS is used to store factory settings, it is recommended to keep these settings in a separate read-only NVS partition. The minimal size of a read-only NVS partition is 0x1000 bytes. See :ref:`read-only-nvs` for more details. ESP-IDF provides :doc:`NVS Partition Generator Utility ` to generate NVS partitions with factory settings and to flash them along with the application. - ``nvs_keys`` (4) is for the NVS key partition. See :doc:`Non-Volatile Storage (NVS) API <../api-reference/storage/nvs_flash>` for more details. - It is used to store NVS encryption keys when `NVS Encryption` feature is enabled. diff --git a/docs/en/api-reference/storage/nvs_flash.rst b/docs/en/api-reference/storage/nvs_flash.rst index a5025ae972..d96d4c8f44 100644 --- a/docs/en/api-reference/storage/nvs_flash.rst +++ b/docs/en/api-reference/storage/nvs_flash.rst @@ -369,6 +369,14 @@ To reduce the number of reads from flash memory, each member of the Page class m Each node in the hash list contains a 24-bit hash and 8-bit item index. Hash is calculated based on item namespace, key name, and ChunkIndex. CRC32 is used for calculation; the result is truncated to 24 bits. To reduce the overhead for storing 32-bit entries in a linked list, the list is implemented as a double-linked list of arrays. Each array holds 29 entries, for the total size of 128 bytes, together with linked list pointers and a 32-bit count field. The minimum amount of extra RAM usage per page is therefore 128 bytes; maximum is 640 bytes. +.. _read-only-nvs: + +Read-only NVS +^^^^^^^^^^^^^^ + +The default minimal size for NVS to function properly is 12kiB (``0x3000``), meaning there have to be at least 3 pages with one of them being in Empty state. However if the NVS partition is flagged as ``readonly`` in the partition table CSV and is being opened in read-only mode, the partition can be as small as 4kiB (``0x1000``) with only one page in Active state and no Empty page. This is because the library does not need to write any data to the partition in this case. The partition can be used to store data that is not expected to change, such as calibration data or factory settings. Partitions of sizes 0x1000 and 0x2000 are always read-only and partitions of size 0x3000 and above are always read-write capable (still can be opened in read-only mode in the code). + + API Reference ------------- diff --git a/examples/storage/parttool/partitions_example.csv b/examples/storage/parttool/partitions_example.csv index 6c09529c47..e03a43c17e 100644 --- a/examples/storage/parttool/partitions_example.csv +++ b/examples/storage/parttool/partitions_example.csv @@ -3,5 +3,5 @@ nvs, data, nvs, 0x9000, 0x6000, phy_init, data, phy, 0xf000, 0x1000, factory, app, factory, 0x10000, 1M, -custom, data, nvs, , 0x1000, +custom, data, nvs, , 0x1000, readonly storage, data, spiffs, , 0x10000, From 674ad565f87342111c82da6e60beb974f7fbd3c7 Mon Sep 17 00:00:00 2001 From: renpeiying Date: Fri, 21 Mar 2025 10:41:08 +0800 Subject: [PATCH 2/4] docs: Update CN translation for 2 api files --- docs/zh_CN/api-guides/partition-tables.rst | 1 + docs/zh_CN/api-reference/storage/nvs_flash.rst | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/docs/zh_CN/api-guides/partition-tables.rst b/docs/zh_CN/api-guides/partition-tables.rst index 58090c5f94..7b6b620d75 100644 --- a/docs/zh_CN/api-guides/partition-tables.rst +++ b/docs/zh_CN/api-guides/partition-tables.rst @@ -176,6 +176,7 @@ SubType 字段长度为 8 bit，内容与具体分区 Type 有关。目前，ESP - NVS API 还可以用于其他应用程序数据。 - 强烈建议为 NVS 分区分配至少 0x3000 字节空间。 - 如果使用 NVS API 存储大量数据，请增加 NVS 分区的大小（默认是 0x6000 字节）。 + - 当 NVS 用于存储出厂设置时，建议将这些设置保存在单独的只读 NVS 分区中。只读 NVS 分区最小为 0x1000 字节。有关更多详情，请参阅 :ref:`read-only-nvs` 了解详情。ESP-IDF 提供了 :doc:`NVS 分区生成工具 `，能够生成包含出厂设置的 NVS 分区，并与应用程序一起烧录。 - ``nvs_keys`` (4) 是 NVS 秘钥分区。详细信息，请参考 :doc:`非易失性存储 (NVS) API <../api-reference/storage/nvs_flash>` 文档。 - 用于存储加密密钥（如果启用了 `NVS 加密` 功能）。 diff --git a/docs/zh_CN/api-reference/storage/nvs_flash.rst b/docs/zh_CN/api-reference/storage/nvs_flash.rst index 59d131cf51..2282bf611d 100644 --- a/docs/zh_CN/api-reference/storage/nvs_flash.rst +++ b/docs/zh_CN/api-reference/storage/nvs_flash.rst @@ -369,6 +369,14 @@ CRC32 哈希列表中每个节点均包含一个 24 位哈希值和 8 位条目索引。哈希值根据条目命名空间、键名和块索引由 CRC32 计算所得，计算结果保留 24 位。为减少将 32 位条目存储在链表中的开销，链表采用了数组的双向链表。每个数组占用 128 个字节，包含 29 个条目、两个链表指针和一个 32 位计数字段。因此，每页额外需要的 RAM 最少为 128 字节，最多为 640 字节。 +.. _read-only-nvs: + +只读 NVS +^^^^^^^^ + +NVS 正常运行所需的最小大小默认为 12kiB (``0x3000``)，这意味着至少需要 3 个页面，其中一个页面必须处于 Empty 状态。但是，如果 NVS 分区在分区表 CSV 中标记为 ``readonly`` 并以只读 (read-only) 模式打开，则该分区大小最少只需 4kiB（``0x1000``），此时仅需一个 Active 状态的页面，无需 Empty 页面。因为在这种情况下，库无需向分区写入任何数据。此类型分区适用于存储不会更改的数据，如校准数据或出厂设置。大小为 0x1000 和 0x2000 的分区始终为只读分区。大小为 0x3000 及以上的分区始终支持读写 (read-write)，但仍可以在代码中以只读模式打开。 + + API 参考 ------------- From 851e869bb2ca18138da537411898e03bcfd85b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20M=C3=BAdry?= Date: Mon, 24 Mar 2025 14:14:17 +0100 Subject: [PATCH 3/4] fix: parttool print subprocess output on fail --- examples/storage/parttool/pytest_parttool_example.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/storage/parttool/pytest_parttool_example.py b/examples/storage/parttool/pytest_parttool_example.py index f73ff17364..a6a6041fd7 100644 --- a/examples/storage/parttool/pytest_parttool_example.py +++ b/examples/storage/parttool/pytest_parttool_example.py @@ -41,7 +41,11 @@ def test_examples_parttool(dut: Dut) -> None: ] for cmd in cmds: - subprocess.check_call(BASE_CMD + cmd.split()) + try: + subprocess.check_call(BASE_CMD + cmd.split()) + except subprocess.CalledProcessError as e: + print(e.output) + raise clean_files = ['custom.bin', 'custom1.bin'] for clean_file in clean_files: From 4a6b99bc4a4bacc928e5b6ce70f9fc9a0548d03e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20M=C3=BAdry?= Date: Mon, 24 Mar 2025 15:55:18 +0100 Subject: [PATCH 4/4] feat(nvs): Optimize read-only NVS loading --- .../nvs_page_test/main/test_fixtures.hpp | 2 +- components/nvs_flash/src/nvs_pagemanager.cpp | 114 +++++++++--------- .../parttool/pytest_parttool_example.py | 8 +- 3 files changed, 61 insertions(+), 63 deletions(-) diff --git a/components/nvs_flash/host_test/nvs_page_test/main/test_fixtures.hpp b/components/nvs_flash/host_test/nvs_page_test/main/test_fixtures.hpp index 5a50ba65fe..6561dc6458 100644 --- a/components/nvs_flash/host_test/nvs_page_test/main/test_fixtures.hpp +++ b/components/nvs_flash/host_test/nvs_page_test/main/test_fixtures.hpp @@ -110,7 +110,7 @@ public: size_t columns = size / column_size; size_t column; - for(column = 0; column < columns; ++column) + for(column = 0; column < columns; column = column + 1) { // read column if((err = esp_partition_read_raw(&esp_partition, dst_offset + (column * column_size), buff, column_size)) != ESP_OK) return err; diff --git a/components/nvs_flash/src/nvs_pagemanager.cpp b/components/nvs_flash/src/nvs_pagemanager.cpp index 39017464ea..c472215934 100644 --- a/components/nvs_flash/src/nvs_pagemanager.cpp +++ b/components/nvs_flash/src/nvs_pagemanager.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -56,77 +56,79 @@ esp_err_t PageManager::load(Partition *partition, uint32_t baseSector, uint32_t // if power went out after a new item for the given key was written, // but before the old one was erased, we end up with a duplicate item - Page& lastPage = back(); - size_t lastItemIndex = SIZE_MAX; - Item item; - size_t itemIndex = 0; - while (lastPage.findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { - itemIndex += item.span; - lastItemIndex = itemIndex; - } - - if (lastItemIndex != SIZE_MAX) { - auto last = PageManager::TPageListIterator(&lastPage); - TPageListIterator it; - - for (it = begin(); it != last; ++it) { - - if ((it->state() != Page::PageState::FREEING) && - (it->eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex) == ESP_OK)) { - break; - } + if (!partition->get_readonly()) { + Page& lastPage = back(); + size_t lastItemIndex = SIZE_MAX; + Item item; + size_t itemIndex = 0; + while (lastPage.findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { + itemIndex += item.span; + lastItemIndex = itemIndex; } - if ((it == last) && (item.datatype == ItemType::BLOB_IDX)) { - /* Rare case in which the blob was stored using old format, but power went just after writing - * blob index during modification. Loop again and delete the old version blob*/ + + if (lastItemIndex != SIZE_MAX) { + auto last = PageManager::TPageListIterator(&lastPage); + TPageListIterator it; + for (it = begin(); it != last; ++it) { if ((it->state() != Page::PageState::FREEING) && - (it->eraseItem(item.nsIndex, ItemType::BLOB, item.key, item.chunkIndex) == ESP_OK)) { + (it->eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex) == ESP_OK)) { break; } } - } - } + if ((it == last) && (item.datatype == ItemType::BLOB_IDX)) { + /* Rare case in which the blob was stored using old format, but power went just after writing + * blob index during modification. Loop again and delete the old version blob*/ + for (it = begin(); it != last; ++it) { - // check if power went out while page was being freed - for (auto it = begin(); it!= end(); ++it) { - if (it->state() == Page::PageState::FREEING) { - Page* newPage = &mPageList.back(); - if (newPage->state() == Page::PageState::ACTIVE) { - auto err = newPage->erase(); + if ((it->state() != Page::PageState::FREEING) && + (it->eraseItem(item.nsIndex, ItemType::BLOB, item.key, item.chunkIndex) == ESP_OK)) { + break; + } + } + } + } + + // check if power went out while page was being freed + for (auto it = begin(); it!= end(); ++it) { + if (it->state() == Page::PageState::FREEING) { + Page* newPage = &mPageList.back(); + if (newPage->state() == Page::PageState::ACTIVE) { + auto err = newPage->erase(); + if (err != ESP_OK) { + return err; + } + mPageList.erase(newPage); + mFreePageList.push_back(newPage); + } + auto err = activatePage(); if (err != ESP_OK) { return err; } - mPageList.erase(newPage); - mFreePageList.push_back(newPage); - } - auto err = activatePage(); - if (err != ESP_OK) { - return err; - } - newPage = &mPageList.back(); + newPage = &mPageList.back(); - err = it->copyItems(*newPage); - if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { - return err; - } + err = it->copyItems(*newPage); + if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { + return err; + } - err = it->erase(); - if (err != ESP_OK) { - return err; - } + err = it->erase(); + if (err != ESP_OK) { + return err; + } - Page* p = static_cast(it); - mPageList.erase(it); - mFreePageList.push_back(p); - break; + Page* p = static_cast(it); + mPageList.erase(it); + mFreePageList.push_back(p); + break; + } } - } - // partition should have at least one free page if it is not read-only - if (!partition->get_readonly() && mFreePageList.empty()) { - return ESP_ERR_NVS_NO_FREE_PAGES; + // partition should have at least one free page if it is not read-only + if (mFreePageList.empty()) { + return ESP_ERR_NVS_NO_FREE_PAGES; + } } return ESP_OK; diff --git a/examples/storage/parttool/pytest_parttool_example.py b/examples/storage/parttool/pytest_parttool_example.py index a6a6041fd7..567329a409 100644 --- a/examples/storage/parttool/pytest_parttool_example.py +++ b/examples/storage/parttool/pytest_parttool_example.py @@ -36,16 +36,12 @@ def test_examples_parttool(dut: Dut) -> None: cmds = [ 'read_partition --partition-type=data --partition-subtype=nvs --output custom1.bin', 'erase_partition --partition-name=custom', - 'write_partition --partition-name=custom --input custom.bin', + 'write_partition --partition-name=custom --input custom.bin --ignore-readonly', 'get_partition_info --partition-boot-default --info size', ] for cmd in cmds: - try: - subprocess.check_call(BASE_CMD + cmd.split()) - except subprocess.CalledProcessError as e: - print(e.output) - raise + subprocess.check_call(BASE_CMD + cmd.split()) clean_files = ['custom.bin', 'custom1.bin'] for clean_file in clean_files: