Merge branch 'feat/read_only_nvs' into 'master'

feat(nvs): Allow read-only NVS partitions smaller than 0x3000

Closes IDFGH-14558

See merge request espressif/esp-idf!36767

components/nvs_flash/src/nvs_pagemanager.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -56,77 +56,79 @@ esp_err_t PageManager::load(Partition *partition, uint32_t baseSector, uint32_t
 
     // if power went out after a new item for the given key was written,
     // but before the old one was erased, we end up with a duplicate item
-    Page& lastPage = back();
-    size_t lastItemIndex = SIZE_MAX;
-    Item item;
-    size_t itemIndex = 0;
-    while (lastPage.findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) {
-        itemIndex += item.span;
-        lastItemIndex = itemIndex;
-    }
-
-    if (lastItemIndex != SIZE_MAX) {
-        auto last = PageManager::TPageListIterator(&lastPage);
-        TPageListIterator it;
-
-        for (it = begin(); it != last; ++it) {
-
-            if ((it->state() != Page::PageState::FREEING) &&
-                    (it->eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex) == ESP_OK)) {
-                break;
-            }
+    if (!partition->get_readonly()) {
+        Page& lastPage = back();
+        size_t lastItemIndex = SIZE_MAX;
+        Item item;
+        size_t itemIndex = 0;
+        while (lastPage.findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) {
+            itemIndex += item.span;
+            lastItemIndex = itemIndex;
         }
-        if ((it == last) && (item.datatype == ItemType::BLOB_IDX)) {
-            /* Rare case in which the blob was stored using old format, but power went just after writing
-             * blob index during modification. Loop again and delete the old version blob*/
+
+        if (lastItemIndex != SIZE_MAX) {
+            auto last = PageManager::TPageListIterator(&lastPage);
+            TPageListIterator it;
+
             for (it = begin(); it != last; ++it) {
 
                 if ((it->state() != Page::PageState::FREEING) &&
-                        (it->eraseItem(item.nsIndex, ItemType::BLOB, item.key, item.chunkIndex) == ESP_OK)) {
+                        (it->eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex) == ESP_OK)) {
                     break;
                 }
             }
-        }
-    }
+            if ((it == last) && (item.datatype == ItemType::BLOB_IDX)) {
+                /* Rare case in which the blob was stored using old format, but power went just after writing
+                * blob index during modification. Loop again and delete the old version blob*/
+                for (it = begin(); it != last; ++it) {
 
-    // check if power went out while page was being freed
-    for (auto it = begin(); it!= end(); ++it) {
-        if (it->state() == Page::PageState::FREEING) {
-            Page* newPage = &mPageList.back();
-            if (newPage->state() == Page::PageState::ACTIVE) {
-                auto err = newPage->erase();
+                    if ((it->state() != Page::PageState::FREEING) &&
+                            (it->eraseItem(item.nsIndex, ItemType::BLOB, item.key, item.chunkIndex) == ESP_OK)) {
+                        break;
+                    }
+                }
+            }
+        }
+
+        // check if power went out while page was being freed
+        for (auto it = begin(); it!= end(); ++it) {
+            if (it->state() == Page::PageState::FREEING) {
+                Page* newPage = &mPageList.back();
+                if (newPage->state() == Page::PageState::ACTIVE) {
+                    auto err = newPage->erase();
+                    if (err != ESP_OK) {
+                        return err;
+                    }
+                    mPageList.erase(newPage);
+                    mFreePageList.push_back(newPage);
+                }
+                auto err = activatePage();
                 if (err != ESP_OK) {
                     return err;
                 }
-                mPageList.erase(newPage);
-                mFreePageList.push_back(newPage);
-            }
-            auto err = activatePage();
-            if (err != ESP_OK) {
-                return err;
-            }
-            newPage = &mPageList.back();
+                newPage = &mPageList.back();
 
-            err = it->copyItems(*newPage);
-            if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
-                return err;
-            }
+                err = it->copyItems(*newPage);
+                if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
+                    return err;
+                }
 
-            err = it->erase();
-            if (err != ESP_OK) {
-                return err;
-            }
+                err = it->erase();
+                if (err != ESP_OK) {
+                    return err;
+                }
 
-            Page* p = static_cast<Page*>(it);
-            mPageList.erase(it);
-            mFreePageList.push_back(p);
-            break;
+                Page* p = static_cast<Page*>(it);
+                mPageList.erase(it);
+                mFreePageList.push_back(p);
+                break;
+            }
         }
-    }
 
-    // partition should have at least one free page
-    if (mFreePageList.empty()) {
-        return ESP_ERR_NVS_NO_FREE_PAGES;
+        // partition should have at least one free page if it is not read-only
+        if (mFreePageList.empty()) {
+            return ESP_ERR_NVS_NO_FREE_PAGES;
+        }
     }
 
     return ESP_OK;

components/partition_table/gen_esp32part.py

@@ -7,7 +7,7 @@
 # See https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/partition-tables.html
 # for explanation of partition table structure and uses.
 #
-# SPDX-FileCopyrightText: 2016-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2016-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import binascii
@@ -31,7 +31,7 @@ SECURE_NONE = None
 SECURE_V1 = 'v1'
 SECURE_V2 = 'v2'
 
-__version__ = '1.4'
+__version__ = '1.5'
 
 APP_TYPE = 0x00
 DATA_TYPE = 0x01
@@ -45,6 +45,8 @@ TYPES = {
     'data': DATA_TYPE,
 }
 
+NVS_RW_MIN_PARTITION_SIZE = 0x3000
+
 
 def get_ptype_as_int(ptype):
     """ Convert a string which might be numeric or the name of a partition type to an integer """
@@ -533,6 +535,11 @@ class PartitionDefinition(object):
             raise ValidationError(self, "'%s' partition of type %s and subtype %s is always read-write and cannot be read-only" %
                                   (self.name, self.type, self.subtype))
 
+        if self.type == TYPES['data'] and self.subtype == SUBTYPES[DATA_TYPE]['nvs']:
+            if self.size < NVS_RW_MIN_PARTITION_SIZE and self.readonly is False:
+                raise ValidationError(self, """'%s' partition of type %s and subtype %s of this size (0x%x) must be flagged as 'readonly' \
+(the size of read/write NVS has to be at least 0x%x)""" % (self.name, self.type, self.subtype, self.size, NVS_RW_MIN_PARTITION_SIZE))
+
     STRUCT_FORMAT = b'<2sBBLL16sL'
 
     @classmethod

docs/en/api-guides/partition-tables.rst

@@ -176,6 +176,7 @@ See enum :cpp:type:`esp_partition_subtype_t` for the full list of subtypes defin
         - The NVS API can also be used for other application data.
         - It is strongly recommended that you include an NVS partition of at least 0x3000 bytes in your project.
         - If using NVS API to store a lot of data, increase the NVS partition size from the default 0x6000 bytes.
+        - When NVS is used to store factory settings, it is recommended to keep these settings in a separate read-only NVS partition. The minimal size of a read-only NVS partition is 0x1000 bytes. See :ref:`read-only-nvs` for more details. ESP-IDF provides :doc:`NVS Partition Generator Utility </api-reference/storage/nvs_partition_gen>` to generate NVS partitions with factory settings and to flash them along with the application.
     - ``nvs_keys`` (4) is for the NVS key partition. See :doc:`Non-Volatile Storage (NVS) API <../api-reference/storage/nvs_flash>` for more details.
 
         - It is used to store NVS encryption keys when `NVS Encryption` feature is enabled.

docs/en/api-reference/storage/nvs_flash.rst

@@ -369,6 +369,14 @@ To reduce the number of reads from flash memory, each member of the Page class m
 
 Each node in the hash list contains a 24-bit hash and 8-bit item index. Hash is calculated based on item namespace, key name, and ChunkIndex. CRC32 is used for calculation; the result is truncated to 24 bits. To reduce the overhead for storing 32-bit entries in a linked list, the list is implemented as a double-linked list of arrays. Each array holds 29 entries, for the total size of 128 bytes, together with linked list pointers and a 32-bit count field. The minimum amount of extra RAM usage per page is therefore 128 bytes; maximum is 640 bytes.
 
+.. _read-only-nvs:
+
+Read-only NVS
+^^^^^^^^^^^^^^
+
+The default minimal size for NVS to function properly is 12kiB (``0x3000``), meaning there have to be at least 3 pages with one of them being in Empty state. However if the NVS partition is flagged as ``readonly`` in the partition table CSV and is being opened in read-only mode, the partition can be as small as 4kiB (``0x1000``) with only one page in Active state and no Empty page. This is because the library does not need to write any data to the partition in this case. The partition can be used to store data that is not expected to change, such as calibration data or factory settings. Partitions of sizes 0x1000 and 0x2000 are always read-only and partitions of size 0x3000 and above are always read-write capable (still can be opened in read-only mode in the code).
+
+
 API Reference
 -------------
 

docs/zh_CN/api-guides/partition-tables.rst

@@ -176,6 +176,7 @@ SubType 字段长度为 8 bit，内容与具体分区 Type 有关。目前，ESP
         -  NVS API 还可以用于其他应用程序数据。
         -  强烈建议为 NVS 分区分配至少 0x3000 字节空间。
         -  如果使用 NVS API 存储大量数据，请增加 NVS 分区的大小（默认是 0x6000 字节）。
+        - 当 NVS 用于存储出厂设置时，建议将这些设置保存在单独的只读 NVS 分区中。只读 NVS 分区最小为 0x1000 字节。有关更多详情，请参阅 :ref:`read-only-nvs` 了解详情。ESP-IDF 提供了 :doc:`NVS 分区生成工具 </api-reference/storage/nvs_partition_gen>`，能够生成包含出厂设置的 NVS 分区，并与应用程序一起烧录。
     - ``nvs_keys`` (4) 是 NVS 秘钥分区。详细信息，请参考 :doc:`非易失性存储 (NVS) API <../api-reference/storage/nvs_flash>` 文档。
 
         -  用于存储加密密钥（如果启用了 `NVS 加密` 功能）。

docs/zh_CN/api-reference/storage/nvs_flash.rst

@@ -369,6 +369,14 @@ CRC32
 
 哈希列表中每个节点均包含一个 24 位哈希值和 8 位条目索引。哈希值根据条目命名空间、键名和块索引由 CRC32 计算所得，计算结果保留 24 位。为减少将 32 位条目存储在链表中的开销，链表采用了数组的双向链表。每个数组占用 128 个字节，包含 29 个条目、两个链表指针和一个 32 位计数字段。因此，每页额外需要的 RAM 最少为 128 字节，最多为 640 字节。
 
+.. _read-only-nvs:
+
+只读 NVS
+^^^^^^^^
+
+NVS 正常运行所需的最小大小默认为 12kiB (``0x3000``)，这意味着至少需要 3 个页面，其中一个页面必须处于 Empty 状态。但是，如果 NVS 分区在分区表 CSV 中标记为 ``readonly`` 并以只读 (read-only) 模式打开，则该分区大小最少只需 4kiB（``0x1000``），此时仅需一个 Active 状态的页面，无需 Empty 页面。因为在这种情况下，库无需向分区写入任何数据。此类型分区适用于存储不会更改的数据，如校准数据或出厂设置。大小为 0x1000 和 0x2000 的分区始终为只读分区。大小为 0x3000 及以上的分区始终支持读写 (read-write)，但仍可以在代码中以只读模式打开。
+
+
 API 参考
 -------------
 

examples/storage/parttool/partitions_example.csv

@@ -3,5 +3,5 @@
 nvs,      data, nvs,     0x9000,  0x6000,
 phy_init, data, phy,     0xf000,  0x1000,
 factory,  app,  factory, 0x10000, 1M,
-custom,   data, nvs,     ,        0x1000,
+custom,   data, nvs,     ,        0x1000,  readonly
 storage,  data, spiffs,  ,        0x10000,

examples/storage/parttool/pytest_parttool_example.py

@@ -36,7 +36,7 @@ def test_examples_parttool(dut: Dut) -> None:
     cmds = [
         'read_partition --partition-type=data --partition-subtype=nvs --output custom1.bin',
         'erase_partition --partition-name=custom',
-        'write_partition --partition-name=custom --input custom.bin',
+        'write_partition --partition-name=custom --input custom.bin --ignore-readonly',
         'get_partition_info --partition-boot-default --info size',
     ]