Merge branch 'feature/storage_nvs_perf_blob_v5.1' into 'release/v5.1'

Improvement of NVS Blob performance (v5.1)

See merge request espressif/esp-idf!41595

components/nvs_flash/.build-test-rules.yml

@@ -0,0 +1,3 @@
+components/nvs_flash/host_test:
+  enable:
+    - if: IDF_TARGET == "linux"

components/nvs_flash/Kconfig

@@ -25,4 +25,13 @@ menu "NVS"
         default n
         help
             This option switches error checking type between assertions (y) or return codes (n).
+
+    config NVS_LEGACY_DUP_KEYS_COMPATIBILITY
+        bool "Enable legacy nvs_set function behavior when same key is reused with different data types"
+        default n
+        help
+            Enabling this will switch the nvs_set family of functions to the legacy mode. When called with same
+            key and different data type, existing value stored in NVS remains active and as a side effect, the
+            new value is also stored into NVS, although not accessible using respective nvs_get function. Use only
+            if your application relies on this NVS API behaviour.
 endmenu

components/nvs_flash/host_test/nvs_host_test/CMakeLists.txt

@@ -2,9 +2,7 @@ cmake_minimum_required(VERSION 3.16)
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 set(COMPONENTS main)
-# Freertos is included via common components. However, CATCH isn't compatible with the FreeRTOS component yet, hence
+# This test app doesn't require FreeRTOS, using mock instead
-# using the FreeRTOS mock component.
-# target.
 list(APPEND EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/mocks/freertos/")
 
 project(nvs_host_test)

components/nvs_flash/host_test/nvs_host_test/main/CMakeLists.txt

@@ -1,5 +1,4 @@
-idf_component_register(SRCS "main.cpp"
+idf_component_register(SRCS "test_nvs.cpp"
-                            "test_nvs.cpp"
                             "test_partition_manager.cpp"
                             "test_nvs_cxx_api.cpp"
                             "test_nvs_handle.cpp"
@@ -9,6 +8,13 @@ idf_component_register(SRCS "main.cpp"
                             "../../../src"
                             "../../../private_include"
                             "../../../../mbedtls/mbedtls/include"
-                            "../../../../../tools/catch"
                        WHOLE_ARCHIVE
                        REQUIRES nvs_flash)
+
+if(CMAKE_C_COMPILER_ID MATCHES "Clang")
+    target_compile_options(${COMPONENT_LIB} PRIVATE -std=gnu++20)
+endif()
+
+# Currently 'main' for IDF_TARGET=linux is defined in freertos component.
+# Since we are using a freertos mock here, need to let Catch2 provide 'main'.
+target_link_libraries(${COMPONENT_LIB} PRIVATE Catch2WithMain)

components/nvs_flash/host_test/nvs_host_test/main/idf_component.yml

@@ -0,0 +1,2 @@
+dependencies:
+  espressif/catch2: "^3.4.0"

components/nvs_flash/host_test/nvs_host_test/main/main.cpp

@@ -1,7 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-#define CATCH_CONFIG_MAIN
-#include "catch.hpp"

components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp

@@ -1,9 +1,9 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-#include "catch.hpp"
+#include <catch2/catch_test_macros.hpp>
 #include "nvs.hpp"
 #include "sdkconfig.h"
 #include "nvs_partition_manager.hpp"
@@ -381,8 +381,7 @@ TEST_CASE("storage can find items on second page if first is not fully written a
 {
     PartitionEmulationFixture f(0, 3);
     nvs::Storage storage(f.part());
-    CHECK(storage.init(0, 3) == ESP_OK);
+    TEST_ESP_OK(storage.init(0, 3));
-    int bar = 0;
     uint8_t bigdata[(nvs::Page::CHUNK_MAX_SIZE - nvs::Page::ENTRY_SIZE) / 2] = {0};
     // write one big chunk of data
     ESP_ERROR_CHECK(storage.writeItem(0, nvs::ItemType::BLOB, "1", bigdata, sizeof(bigdata)));
@@ -504,7 +503,7 @@ TEST_CASE("erase operations are distributed among sectors", "[nvs]")
     }
 
     /* Check that erase counts are distributed between the remaining sectors */
-    const size_t max_erase_cnt = write_ops / nvs::Page::ENTRY_COUNT / (sectors - static_sectors) + 1;
+    TEMPORARILY_DISABLED(const size_t max_erase_cnt = write_ops / nvs::Page::ENTRY_COUNT / (sectors - static_sectors) + 1;)
     for (size_t i = 0; i < sectors; ++i) {
         TEMPORARILY_DISABLED(
             auto erase_cnt = f.emu.getSectorEraseCount(i);
@@ -592,7 +591,7 @@ TEST_CASE("nvs api tests", "[nvs]")
     nvs_handle_t handle_2;
     TEST_ESP_OK(nvs_open("namespace2", NVS_READWRITE, &handle_2));
     TEST_ESP_OK(nvs_set_i32(handle_2, "foo", 0x3456789a));
-    const char *str = "value 0123456789abcdef0123456789abcdef";
+    char str[] = "value 0123456789abcdef0123456789abcdef";
     TEST_ESP_OK(nvs_set_str(handle_2, "key", str));
 
     int32_t v1;
@@ -603,7 +602,7 @@ TEST_CASE("nvs api tests", "[nvs]")
     TEST_ESP_OK(nvs_get_i32(handle_2, "foo", &v2));
     CHECK(0x3456789a == v2);
 
-    char buf[strlen(str) + 1];
+    char buf[sizeof(str)];
     size_t buf_len = sizeof(buf);
 
     size_t buf_len_needed;
@@ -632,8 +631,6 @@ TEST_CASE("deinit partition doesn't affect other partition's open handles", "[nv
     const char *OTHER_PARTITION_NAME = "other_part";
     PartitionEmulationFixture f(0, 10);
     PartitionEmulationFixture f_other(0, 10, OTHER_PARTITION_NAME);
-    const char *str = "value 0123456789abcdef0123456789abcdef";
-    const uint8_t blob[8] = {0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7};
 
     nvs_handle_t handle_1;
     const uint32_t NVS_FLASH_SECTOR = 6;
@@ -693,7 +690,6 @@ TEST_CASE("nvs_entry_info fails with ESP_ERR_INVALID_ARG if a parameter is NULL"
 TEST_CASE("nvs_entry_info doesn't change iterator on parameter error", "[nvs]")
 {
     nvs_iterator_t it = reinterpret_cast<nvs_iterator_t>(0xbeef);
-    nvs_entry_info_t info;
     REQUIRE(nvs_entry_info(it, nullptr) == ESP_ERR_INVALID_ARG);
     CHECK(it == reinterpret_cast<nvs_iterator_t>(0xbeef));
 
@@ -972,8 +968,8 @@ TEST_CASE("wifi test", "[nvs]")
     TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.opmode", opmode));
 
     uint8_t country = 0;
-    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.country", &opmode), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.country", &country), ESP_ERR_NVS_NOT_FOUND);
-    TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.country", opmode));
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.country", country));
 
     char ssid[36];
     size_t size = sizeof(ssid);
@@ -1175,12 +1171,13 @@ public:
                     blobBufLen = largeBlobLen ;
 
                 }
-                uint8_t buf[blobBufLen];
+                uint8_t* buf = new uint8_t[blobBufLen];
                 memset(buf, 0, blobBufLen);
 
                 size_t len = blobBufLen;
                 auto err = nvs_get_blob(handle, keys[index], buf, &len);
                 if (err == ESP_ERR_FLASH_OP_FAIL) {
+                    delete [] buf;
                     return err;
                 }
                 if (!written[index]) {
@@ -1189,6 +1186,7 @@ public:
                     REQUIRE(err == ESP_OK);
                     REQUIRE(memcmp(buf, reinterpret_cast<const uint8_t *>(values[index]), blobBufLen) == 0);
                 }
+                delete [] buf;
                 break;
             }
 
@@ -1239,7 +1237,6 @@ public:
 
             case nvs::ItemType::SZ: {
                 char buf[strBufLen];
-                size_t len = strBufLen;
 
                 size_t strLen = gen() % (strBufLen - 1);
                 std::generate_n(buf, strLen, [&]() -> char {
@@ -1270,7 +1267,7 @@ public:
                 } else {
                     blobBufLen = largeBlobLen ;
                 }
-                uint8_t buf[blobBufLen];
+                uint8_t* buf = new uint8_t[blobBufLen];
                 memset(buf, 0, blobBufLen);
                 size_t blobLen = gen() % blobBufLen;
                 std::generate_n(buf, blobLen, [&]() -> uint8_t {
@@ -1279,16 +1276,19 @@ public:
 
                 auto err = nvs_set_blob(handle, keys[index], buf, blobLen);
                 if (err == ESP_ERR_FLASH_OP_FAIL) {
+                    delete [] buf;
                     return err;
                 }
                 if (err == ESP_ERR_NVS_REMOVE_FAILED) {
                     written[index] = true;
                     memcpy(reinterpret_cast<uint8_t *>(values[index]), buf, blobBufLen);
+                    delete [] buf;
                     return ESP_ERR_FLASH_OP_FAIL;
                 }
                 REQUIRE(err == ESP_OK);
                 written[index] = true;
                 memcpy(reinterpret_cast<char *>(values[index]), buf, blobBufLen);
+                delete [] buf;
                 break;
             }
 
@@ -1751,8 +1751,6 @@ TEST_CASE("Check that orphaned blobs are erased during init", "[nvs]")
 {
     const size_t blob_size = nvs::Page::CHUNK_MAX_SIZE * 3 ;
     uint8_t blob[blob_size] = {0x11};
-    uint8_t blob2[blob_size] = {0x22};
-    uint8_t blob3[blob_size] = {0x33};
     PartitionEmulationFixture f(0, 5);
     nvs::Storage storage(f.part());
 
@@ -2048,3 +2046,61 @@ TEST_CASE("Recovery from power-off during modification of blob present in old-fo
 
     TEST_ESP_OK(nvs_flash_deinit_partition(f.part()->get_partition_name()));
 }
+
+TEST_CASE("Page handles invalid CRC of variable length items", "[nvs][cur]")
+{
+    PartitionEmulationFixture f(0, 4);
+    {
+        nvs::Page p;
+        TEST_ESP_OK(p.load(f.part(), 0));
+        char buf[128] = {0};
+        TEST_ESP_OK(p.writeItem(1, nvs::ItemType::BLOB, "1", buf, sizeof(buf)));
+    }
+    // corrupt header of the item (64 is the offset of the first item in page)
+    uint32_t overwrite_buf = 0;
+    TEST_ESP_OK(esp_partition_write(&f.esp_partition, 64, &overwrite_buf, 4));
+    // load page again
+    {
+        nvs::Page p1;
+        TEST_ESP_OK(p1.load(f.part(), 0));
+    }
+}
+
+TEST_CASE("namespace name is deep copy", "[nvs]")
+{
+    char ns_name[16];
+    strcpy(ns_name, "const_name");
+
+    nvs_handle_t handle_1;
+    nvs_handle_t handle_2;
+    const uint32_t NVS_FLASH_SECTOR = 6;
+    const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3;
+
+    PartitionEmulationFixture f(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN);
+
+    TEMPORARILY_DISABLED(f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN);)
+
+    TEST_ESP_OK( nvs::NVSPartitionManager::get_instance()->init_custom(f.part(),
+                 NVS_FLASH_SECTOR,
+                 NVS_FLASH_SECTOR_COUNT_MIN));
+
+    TEST_ESP_OK(nvs_open("const_name", NVS_READWRITE, &handle_1));
+    strcpy(ns_name, "just_kidding");
+
+    CHECK(nvs_open("just_kidding", NVS_READONLY, &handle_2) == ESP_ERR_NVS_NOT_FOUND);
+
+    nvs_close(handle_1);
+    nvs_close(handle_2);
+
+    nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME);
+}
+
+/* Add new tests above */
+/* This test has to be the final one */
+
+TEST_CASE("dump all performance data", "[nvs]")
+{
+    std::cout << "====================" << std::endl << "Dumping benchmarks" << std::endl;
+    std::cout << s_perf.str() << std::endl;
+    std::cout << "====================" << std::endl;
+}

components/nvs_flash/host_test/nvs_host_test/main/test_nvs_cxx_api.cpp

@@ -3,7 +3,7 @@
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-#include "catch.hpp"
+#include <catch2/catch_test_macros.hpp>
 #include <algorithm>
 #include <cstring>
 #include "nvs_handle_simple.hpp"

components/nvs_flash/host_test/nvs_host_test/main/test_nvs_handle.cpp

@@ -3,7 +3,7 @@
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-#include "catch.hpp"
+#include <catch2/catch_test_macros.hpp>
 #include <algorithm>
 #include <cstring>
 #include "nvs_handle_simple.hpp"

components/nvs_flash/host_test/nvs_host_test/main/test_nvs_initialization.cpp

@@ -3,7 +3,7 @@
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-#include "catch.hpp"
+#include <catch2/catch_test_macros.hpp>
 #include "nvs.hpp"
 #include "nvs_partition_manager.hpp"
 #include "nvs_partition.hpp"
@@ -12,8 +12,6 @@
 
 TEST_CASE("nvs_flash_init_partition_ptr fails due to nullptr arg", "[nvs_custom_part]")
 {
-    const uint32_t NVS_FLASH_SECTOR = 6;
-    const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3;
     uint8_t *p_part_desc_addr_start;
     CHECK(esp_partition_file_mmap((const uint8_t **)&p_part_desc_addr_start) == ESP_OK);
 

components/nvs_flash/host_test/nvs_host_test/main/test_nvs_storage.cpp

@@ -3,7 +3,7 @@
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-#include "catch.hpp"
+#include <catch2/catch_test_macros.hpp>
 #include <cstring>
 #include "nvs_storage.hpp"
 #include "nvs_partition_manager.hpp"

components/nvs_flash/host_test/nvs_host_test/main/test_partition_manager.cpp

@@ -3,7 +3,7 @@
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-#include "catch.hpp"
+#include <catch2/catch_test_macros.hpp>
 #include <algorithm>
 #include <cstring>
 #include "nvs_handle_simple.hpp"

components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py

@@ -6,5 +6,8 @@ from pytest_embedded import Dut
 
 @pytest.mark.linux
 @pytest.mark.host_test
+@pytest.mark.parametrize('config', [
+    'default_set_key', 'legacy_set_key'
+], indirect=True)
 def test_nvs_host_linux(dut: Dut) -> None:
     dut.expect_exact('All tests passed', timeout=60)

components/nvs_flash/host_test/nvs_host_test/sdkconfig.ci.default_set_key

@@ -0,0 +1 @@
+# CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY=n

components/nvs_flash/host_test/nvs_host_test/sdkconfig.ci.legacy_set_key

@@ -0,0 +1 @@
+CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY=y

components/nvs_flash/include/nvs.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -464,6 +464,25 @@ esp_err_t nvs_get_str (nvs_handle_t handle, const char* key, char* out_value, si
 esp_err_t nvs_get_blob(nvs_handle_t handle, const char* key, void* out_value, size_t* length);
 /**@}*/
 
+/**
+ * @brief      Lookup key-value pair with given key name.
+ *
+ * Note that function may indicate both existence of the key as well as the data type of NVS entry if it is found.
+ *
+ * @param[in]  handle    Storage handle obtained with nvs_open.
+ * @param[in]  key       Key name. Maximum length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty.
+ * @param[out] out_type  Pointer to the output variable populated with data type of NVS entry in case key was found.
+ *                       May be NULL, respective data type is then not provided.
+ * @return
+ *              - ESP_OK if NVS entry for key provided was found
+ *              - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist
+ *              - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL
+ *              - ESP_FAIL if there is an internal error; most likely due to corrupted
+ *                NVS partition (only if NVS assertion checks are disabled)
+ *              - other error codes from the underlying storage driver
+ */
+esp_err_t nvs_find_key(nvs_handle_t handle, const char* key, nvs_type_t* out_type);
+
 /**
  * @brief      Erase key-value pair with given key name.
  *

components/nvs_flash/include/nvs_handle.hpp

@@ -164,6 +164,21 @@ public:
      */
     virtual esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) = 0;
 
+    /**
+     * @brief Checks whether key exists and optionally returns also data type of associated entry.
+     *
+     * @param[in]     key        Key name. Maximum length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty.
+     * @param[out]    nvstype    Nvs data type to of entry, if it exists.
+     *
+     * @return      - ESP_OK if NVS entry for key provided was found. Data type will be returned via \c nvstype.
+     *              - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist.
+     *              - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL.
+     *              - ESP_FAIL if there is an internal error; most likely due to corrupted
+     *                NVS partition (only if NVS assertion checks are disabled).
+     *              - other error codes from the underlying storage driver.
+     */
+    virtual esp_err_t find_key(const char* key, nvs_type_t &nvstype) = 0;
+
     /**
      * @brief Erases an entry.
      */

components/nvs_flash/src/nvs_api.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -310,6 +310,25 @@ extern "C" void nvs_close(nvs_handle_t handle)
     delete static_cast<NVSHandleEntry*>(it);
 }
 
+extern "C" esp_err_t nvs_find_key(nvs_handle_t c_handle, const char* key, nvs_type_t* out_type)
+{
+    Lock lock;
+    ESP_LOGD(TAG, "%s %s", __func__, key);
+    NVSHandleSimple *handle;
+    auto err = nvs_find_ns_handle(c_handle, &handle);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    nvs_type_t nvstype;
+    err = handle->find_key(key, nvstype);
+
+    if(err == ESP_OK && out_type != nullptr)
+        *out_type = nvstype;
+
+    return err;
+}
+
 extern "C" esp_err_t nvs_erase_key(nvs_handle_t c_handle, const char* key)
 {
     Lock lock;

components/nvs_flash/src/nvs_handle_locked.cpp

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+/*
-//
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
-// Licensed under the Apache License, Version 2.0 (the "License");
+ *
-// you may not use this file except in compliance with the License.
+ * SPDX-License-Identifier: Apache-2.0
-// You may obtain a copy of the License at
+ */
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 #include "nvs_handle_locked.hpp"
 
 namespace nvs {
@@ -47,6 +39,12 @@ esp_err_t NVSHandleLocked::get_item_size(ItemType datatype, const char *key, siz
     return handle->get_item_size(datatype, key, size);
 }
 
+esp_err_t NVSHandleLocked::find_key(const char* key, nvs_type_t &nvstype)
+{
+    Lock lock;
+    return handle->find_key(key, nvstype);
+}
+
 esp_err_t NVSHandleLocked::erase_item(const char* key) {
     Lock lock;
     return handle->erase_item(key);

components/nvs_flash/src/nvs_handle_locked.hpp

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+/*
-//
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
-// Licensed under the Apache License, Version 2.0 (the "License");
+ *
-// you may not use this file except in compliance with the License.
+ * SPDX-License-Identifier: Apache-2.0
-// You may obtain a copy of the License at
+ */
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 #ifndef NVS_HANDLE_LOCKED_HPP_
 #define NVS_HANDLE_LOCKED_HPP_
 
@@ -49,6 +41,8 @@ public:
 
     esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) override;
 
+    esp_err_t find_key(const char* key, nvs_type_t &nvstype) override;
+
     esp_err_t erase_item(const char* key) override;
 
     esp_err_t erase_all() override;

components/nvs_flash/src/nvs_handle_simple.cpp

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+/*
-//
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
-// Licensed under the Apache License, Version 2.0 (the "License");
+ *
-// you may not use this file except in compliance with the License.
+ * SPDX-License-Identifier: Apache-2.0
-// You may obtain a copy of the License at
+ */
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 #include <cstdlib>
 #include "nvs_handle.hpp"
 #include "nvs_partition_manager.hpp"
@@ -73,6 +65,23 @@ esp_err_t NVSHandleSimple::get_item_size(ItemType datatype, const char *key, siz
     return mStoragePtr->getItemDataSize(mNsIndex, datatype, key, size);
 }
 
+esp_err_t NVSHandleSimple::find_key(const char* key, nvs_type_t &nvstype)
+{
+    if (!valid) return ESP_ERR_NVS_INVALID_HANDLE;
+
+    nvs::ItemType datatype;
+    esp_err_t err = mStoragePtr->findKey(mNsIndex, key, &datatype);
+    if(err != ESP_OK)
+        return err;
+
+    if(datatype == ItemType::BLOB_IDX || datatype == ItemType::BLOB)
+        datatype = ItemType::BLOB_DATA;
+
+    nvstype = (nvs_type_t) datatype;
+
+    return err;
+}
+
 esp_err_t NVSHandleSimple::erase_item(const char* key)
 {
     if (!valid) return ESP_ERR_NVS_INVALID_HANDLE;

components/nvs_flash/src/nvs_handle_simple.hpp

@@ -51,6 +51,8 @@ public:
 
     esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) override;
 
+    esp_err_t find_key(const char *key, nvs_type_t &nvstype) override;
+
     esp_err_t erase_item(const char *key) override;
 
     esp_err_t erase_all() override;

components/nvs_flash/src/nvs_page.cpp

@@ -246,6 +246,43 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c
     return ESP_OK;
 }
 
+// Reads the data entries of the variable length item.
+// The metadata entry is already read in the item object.
+// index is the index of the metadata entry on the page.
+// data is pointer to the buffer where the data will be copied to. It has to be at least
+// item.varLength.dataSize bytes long.
+// The function returns ESP_OK if the data was read successfully, or an error code if there was an error.
+esp_err_t Page::readVariableLengthItemData(const Item& item, const size_t index, void* data)
+{
+    if (mState == PageState::INVALID) {
+        return ESP_ERR_NVS_INVALID_STATE;
+    }
+
+    esp_err_t rc;
+    uint8_t* dst = reinterpret_cast<uint8_t*>(data);
+    size_t left = item.varLength.dataSize;
+    for (size_t i = index + 1; i < index + item.span; ++i) {
+        Item ditem;
+        rc = readEntry(i, ditem);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        size_t willCopy = ENTRY_SIZE;
+        willCopy = (left < willCopy) ? left : willCopy;
+        memcpy(dst, ditem.rawData, willCopy);
+        left -= willCopy;
+        dst += willCopy;
+    }
+    if (Item::calculateCrc32(reinterpret_cast<uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+        rc = eraseEntryAndSpan(index);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        return ESP_ERR_NVS_NOT_FOUND;
+    }
+    return ESP_OK;
+}
+
 esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart)
 {
     size_t index = 0;
@@ -273,28 +310,7 @@ esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, vo
         return ESP_ERR_NVS_INVALID_LENGTH;
     }
 
-    uint8_t* dst = reinterpret_cast<uint8_t*>(data);
+    return readVariableLengthItemData(item, index, data);
-    size_t left = item.varLength.dataSize;
-    for (size_t i = index + 1; i < index + item.span; ++i) {
-        Item ditem;
-        rc = readEntry(i, ditem);
-        if (rc != ESP_OK) {
-            return rc;
-        }
-        size_t willCopy = ENTRY_SIZE;
-        willCopy = (left < willCopy) ? left : willCopy;
-        memcpy(dst, ditem.rawData, willCopy);
-        left -= willCopy;
-        dst += willCopy;
-    }
-    if (Item::calculateCrc32(reinterpret_cast<uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
-        rc = eraseEntryAndSpan(index);
-        if (rc != ESP_OK) {
-            return rc;
-        }
-        return ESP_ERR_NVS_NOT_FOUND;
-    }
-    return ESP_OK;
 }
 
 esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart)
@@ -326,9 +342,23 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con
         return ESP_ERR_NVS_INVALID_LENGTH;
     }
 
+    // We have metadata of the variable length data chunk. It contains the length of the data and the crc32.
+    // As a first step we can calculate the crc32 of the data buffer to be compared with the crc32 of the item in the flash.
+    // If they are not equal, immediately return ESP_ERR_NVS_CONTENT_DIFFERS.
+    // If they are equal, to avoid crc32 collision false positive, we will read the data from the flash entry by entry and compare
+    // it with the respective chunk of input data buffer. The crc32 of the data read from the flash will be calculated on the fly.
+    // At the end, we will compare the crc32 of the data read from the flash with the crc32 of the metadata item in the flash to make sure
+    // that the data in the flash is not corrupted.
+    if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+        return ESP_ERR_NVS_CONTENT_DIFFERS;
+    }
+
     const uint8_t* dst = reinterpret_cast<const uint8_t*>(data);
     size_t left = item.varLength.dataSize;
-    for (size_t i = index + 1; i < index + item.span; ++i) {
+    uint32_t accumulatedCRC32;
+    size_t initial_index = index + 1;
+
+    for (size_t i = initial_index; i < index + item.span; ++i) {
         Item ditem;
         rc = readEntry(i, ditem);
         if (rc != ESP_OK) {
@@ -339,11 +369,18 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con
         if (memcmp(dst, ditem.rawData, willCopy)) {
             return ESP_ERR_NVS_CONTENT_DIFFERS;
         }
+
+        // Calculate the crc32 of the actual ditem.rawData buffer. Do not pass accumulatedCRC32 in the first call.
+        // In the first call, calculateCrc32 will use its default. In the subsequent calls, accumulatedCRC32 is the crc32 of the previous buffer.
+        accumulatedCRC32 = Item::calculateCrc32(ditem.rawData, willCopy, (i == initial_index) ? nullptr : &accumulatedCRC32);
+
         left -= willCopy;
         dst += willCopy;
     }
-    if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+    // Check if the CRC32 calculated on the fly matches the variable length data CRC32 indicated in the metadata entry.
-        return ESP_ERR_NVS_NOT_FOUND;
+    // If they are not equal, it means the data in the flash is corrupt, we will return ESP_ERR_NVS_CONTENT_DIFFERS.
+    if (accumulatedCRC32 != item.varLength.dataCrc32) {
+        return ESP_ERR_NVS_CONTENT_DIFFERS;
     }
 
     return ESP_OK;

components/nvs_flash/src/nvs_page.hpp

@@ -88,6 +88,8 @@ public:
 
     esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY);
 
+    esp_err_t readVariableLengthItemData(const Item& item, const size_t index, void* data);
+
     esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);
 
     esp_err_t cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);

components/nvs_flash/src/nvs_storage.cpp

@@ -241,13 +241,16 @@ bool Storage::isValid() const
     return mState == StorageState::ACTIVE;
 }
 
-esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart)
+esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart, size_t* itemIndex)
 {
     for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
-        size_t itemIndex = 0;
+        size_t tmpItemIndex = 0;
-        auto err = it->findItem(nsIndex, datatype, key, itemIndex, item, chunkIdx, chunkStart);
+        auto err = it->findItem(nsIndex, datatype, key, tmpItemIndex, item, chunkIdx, chunkStart);
         if(err == ESP_OK) {
             page = it;
+            if(itemIndex) {
+                *itemIndex = tmpItemIndex;
+            }
             return ESP_OK;
         }
     }
@@ -359,87 +362,166 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
     return err;
 }
 
+// datatype BLOB is written as BLOB_INDEX and BLOB_DATA and is searched for previous value as BLOB_INDEX and/or BLOB
+// datatype BLOB_INDEX and BLOB_DATA are not supported as input parameters, the layer above should always use BLOB
 esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize)
 {
     if(mState != StorageState::ACTIVE) {
         return ESP_ERR_NVS_NOT_INITIALIZED;
     }
 
+    // pointer to the page where the existing item was found
     Page* findPage = nullptr;
+    // index of the item in the page where the existing item was found
+    size_t itemIndex = 0;
+    // page sequence number helping to detect whether the page with old value was relocated during the new write
+    uint32_t findPageSeqNumber = UINT32_MAX;
+
+    // indicates the datatype representation match between the old value and the new one
+    bool matchedTypePageFound = false;
     Item item;
 
-    esp_err_t err;
+    esp_err_t err = ESP_OK;
+
+    // Try to find existing item with the same key and namespace index
+    // We are performing the findItem with datatype specified (it is not ANY) to ensure the hash list lookup is done.
     if(datatype == ItemType::BLOB) {
-        err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
+        // Specific lookup if performed for input datatype BLOB. The searched datatype for exact match is BLOB_INDEX.
+        // The BLOB_INDEX is used to store the metadata of the (originally typed) BLOB data in current V2 implementation.
+        err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
+        if(err == ESP_OK && findPage != nullptr) {
+            matchedTypePageFound = true;
+        }
+#ifdef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY
+        // In legacy mode, we also try to find the item as BLOB if not found as BLOB_INDEX.
+        // In this mode, it is possible to have multiple active values under the same (logical) key.
+        // For BLOBs (which may have different physical representations in V1 it is BLOB, in V2 it is BLOB_INDEX) it in turn means
+        // that we have to check both datatypes to find the old value.
+        // The general case for compatibility flag disabled is below and handles all datatypes including BLOB.
+        // To save some cycles, we do not compile both findItem calls in this case.
+        if(err == ESP_ERR_NVS_NOT_FOUND) {
+            // If not found as BLOB_INDEX, try to find it as BLOB (legacy support).
+            err = findItem(nsIndex, ItemType::BLOB, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
+            if(err == ESP_OK && findPage != nullptr) {
+                matchedTypePageFound = false;   // datatype does not match, we cannot extract chunkStart from the item
+
+                // keep the sequence number of the page where the item was found for later check of relocation
+                err = findPage->getSeqNumber(findPageSeqNumber);
+                if(err != ESP_OK) {
+                    return err;
+                }
+            }
+        }
+#endif
     } else {
-        err = findItem(nsIndex, datatype, key, findPage, item);
+        // Handle all other data types than BLOB
+        err = findItem(nsIndex, datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
+        if(err == ESP_OK && findPage != nullptr) {
+            matchedTypePageFound = true;
+        }
     }
 
+#ifndef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY
+    // If the item was not found under assumed datatype, try to find it as ANY.
+    if(findPage == nullptr) {
+
+        // We should not find BLOB_DATA chunks as CHUNK_ANY is never used by the BLOB_DATA.
+        err = findItem(nsIndex, nvs::ItemType::ANY, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
+        if(err == ESP_OK && findPage != nullptr) {
+            // keep the sequence number of the page where the item was found for later check of relocation
+            err = findPage->getSeqNumber(findPageSeqNumber);
+            if(err != ESP_OK) {
+                return err;
+            }
+            // item was found with the same key and namespace index but data type is different
+            matchedTypePageFound = false;
+        }
+    }
+#endif
+
+#ifndef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY
+    // If the item was not found, try to find it with any other datatype. Omit the BLOB_DATA and handle BLOB with respect
+    // to the V1 and V2 representation.
+    const ItemType dataTypes[] = {
+        ItemType::U8,
+        ItemType::U16,
+        ItemType::U32,
+        ItemType::U64,
+        ItemType::I8,
+        ItemType::I16,
+        ItemType::I32,
+        ItemType::I64,
+        ItemType::SZ,
+        ItemType::BLOB_IDX,
+        ItemType::BLOB
+    };
+
+    if(findPage == nullptr) {
+        // Iterate over potential data types to allow findItem() search using the hash list instead of bruteforce search.
+        for(const auto& currType : dataTypes) {
+            // Skip search for BLOB_IDX if the requested datatype is BLOB. BLOB_IDX was already searched above.
+            if(datatype == ItemType::BLOB && currType == ItemType::BLOB_IDX) continue;
+
+            // Skip search if requested datatype is not BLOB and the current datatype is equal to the requested one. This was already searched above.
+            if(datatype != ItemType::BLOB && currType == datatype) continue;
+
+            err = findItem(nsIndex, currType, key, findPage, item);
+            if(err == ESP_OK && findPage != nullptr) {
+                // keep the sequence number of the page where the item was found for later check of relocation
+                err = findPage->getSeqNumber(findPageSeqNumber);
+                if(err != ESP_OK) {
+                    return err;
+                }
+                // item was found with the same key and namespace index but data type is different
+                matchedTypePageFound = false;
+                break;
+            }
+        }
+    }
+#endif
+
+    // Here the findPage is either nullptr or points to the page where the item was found.
+    // The matchedTypePageFound is true if the old value item was found and its datatype representation matches the new one.
+    // This flag is used to determine if the item should be checked for same value.
     if(err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
         return err;
     }
 
+    // Handle value update
     if(datatype == ItemType::BLOB) {
         VerOffset prevStart,  nextStart;
         prevStart = nextStart = VerOffset::VER_0_OFFSET;
-        if (findPage) {
+        if(matchedTypePageFound) {
-            // Do a sanity check that the item in question is actually being modified.
+            // Do a check that the item in question is actually being modified.
             // If it isn't, it is cheaper to purposefully not write out new data.
             // since it may invoke an erasure of flash.
             if(cmpMultiPageBlob(nsIndex, key, data, dataSize) == ESP_OK) {
                 return ESP_OK;
             }
 
-            if (findPage->state() == Page::PageState::UNINITIALIZED ||
+            // Get the version of the previous index with same <ns,key>
-                    findPage->state() == Page::PageState::INVALID) {
-                err = findItem(nsIndex, datatype, key, findPage, item);
-                if (err != ESP_OK) {
-                    return err;
-                }
-            }
-            /* Get the version of the previous index with same <ns,key> */
             prevStart = item.blobIndex.chunkStart;
             NVS_ASSERT_OR_RETURN(prevStart == VerOffset::VER_0_OFFSET || prevStart == VerOffset::VER_1_OFFSET, ESP_FAIL);
 
-
+            // Toggle the version by changing the offset
-            /* Toggle the version by changing the offset */
             nextStart
                 = (prevStart == VerOffset::VER_1_OFFSET) ? VerOffset::VER_0_OFFSET : VerOffset::VER_1_OFFSET;
         }
-        /* Write the blob with new version*/
+        // Write the blob with new version
         err = writeMultiPageBlob(nsIndex, key, data, dataSize, nextStart);
 
         if(err == ESP_ERR_NVS_PAGE_FULL) {
             return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
         }
+
         if (err != ESP_OK) {
             return err;
         }
-
-        if (findPage) {
-            /* Erase the blob with earlier version*/
-            err = eraseMultiPageBlob(nsIndex, key, prevStart);
-
-            if (err == ESP_ERR_FLASH_OP_FAIL) {
-                return ESP_ERR_NVS_REMOVE_FAILED;
-            }
-            if (err != ESP_OK) {
-                return err;
-            }
-
-            findPage = nullptr;
     } else {
-            /* Support for earlier versions where BLOBS were stored without index */
+        // Do a check that the item in question is actually being modified.
-            err = findItem(nsIndex, datatype, key, findPage, item);
-            if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
-                return err;
-            }
-        }
-    } else {
-        // Do a sanity check that the item in question is actually being modified.
         // If it isn't, it is cheaper to purposefully not write out new data.
         // since it may invoke an erasure of flash.
-        if (findPage != nullptr &&
+        if(matchedTypePageFound &&
                 findPage->cmpItem(nsIndex, datatype, key, data, dataSize) == ESP_OK) {
             return ESP_OK;
         }
@@ -462,34 +544,75 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key
             if(err == ESP_ERR_NVS_PAGE_FULL) {
                 return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
             }
-            if (err != ESP_OK) {
-                return err;
         }
-        } else if (err != ESP_OK) {
+
+        if(err != ESP_OK) {
             return err;
         }
     }
 
-    if (findPage) {
+    // Delete previous value
-        if (findPage->state() == Page::PageState::UNINITIALIZED ||
+    // Note: The old entry won't be deleted if the new value is the same as the old value - code won't reach here in that case.
-                findPage->state() == Page::PageState::INVALID) {
+
-            err = findItem(nsIndex, datatype, key, findPage, item);
+    // If findPage is null then previous value was not present in NVS and nothig is to be deleted.
-            if (err != ESP_OK) {
+    if(findPage == nullptr) {
         return err;
     }
-        }
+
-        err = findPage->eraseItem(nsIndex, datatype, key);
+    if(item.datatype == ItemType::BLOB_IDX) {
+        // If the item found was BLOB_INDEX, the eraseMultiPageBlob is used to erase the whole multi-page blob.
+        // It is not necessary to check the potential page relocation as the function will find the blob again anyway.
+        VerOffset prevStart = item.blobIndex.chunkStart;
+        err = eraseMultiPageBlob(nsIndex, key, prevStart);
+
         if(err == ESP_ERR_FLASH_OP_FAIL) {
             return ESP_ERR_NVS_REMOVE_FAILED;
         }
+    } else {
+        // For all other data types, we have to check the potential page relocation.
+
+        // The findPage might have been relocated as a part of space reclaiming.
+        // First indication is the page state. It might become the "spare" page thus changing the state from FULL or ACTIVE.
+        bool wasRelocated = false;
+
+        if( findPage->state() != Page::PageState::ACTIVE &&
+            findPage->state() != Page::PageState::FULL) {
+                wasRelocated = true;
+        }
+
+        // Other indication of the multi step relocation is the page sequence number. If the sequence number is different than page
+        // sequence number at the moment initial item was found, the page was relocated.
+        if(!wasRelocated) {
+            uint32_t newPageSeqNumber;
+            err = findPage->getSeqNumber(newPageSeqNumber);
+            if(err != ESP_OK) {
+                return err;
+            }
+            if(newPageSeqNumber != findPageSeqNumber) {
+                wasRelocated = true;
+            }
+        }
+
+        if(wasRelocated) {
+            // The page was relocated. We have to find the old value again from the beginning.
+            // As the item was already found before relocation, we can use the exact datatype from item.
+            err = findItem(nsIndex, item.datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
             if(err != ESP_OK) {
                 return err;
             }
         }
+
+        // Page containing the old value is now refreshed. We can erase the old value.
+        err = findPage->eraseEntryAndSpan(itemIndex);
+        if(err == ESP_ERR_FLASH_OP_FAIL) {
+            return ESP_ERR_NVS_REMOVE_FAILED;
+        }
+    }
+
 #ifdef DEBUG_STORAGE
     debugCheck();
 #endif
-    return ESP_OK;
+    return err;
 }
 
 esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uint8_t& nsIndex)
@@ -549,8 +672,10 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat
 {
     Item item;
     Page* findPage = nullptr;
+    size_t itemIndex = 0;
 
-    /* First read the blob index */
+
+    // First read the blob index
     auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
     if(err != ESP_OK) {
         return err;
@@ -562,24 +687,28 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat
 
     NVS_ASSERT_OR_RETURN(dataSize == item.blobIndex.dataSize, ESP_FAIL);
 
-    /* Now read corresponding chunks */
+    // Now read related blob data chunks
+    // Remember the itemIndex as it is used to fast locate the entry in the page
     for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) {
-        err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum);
+        err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum, nvs::VerOffset::VER_ANY, &itemIndex);
         if(err != ESP_OK) {
             if(err == ESP_ERR_NVS_NOT_FOUND) {
                 break;
             }
             return err;
         }
+
+        // Check if the blob data chunk length indicated for actual item still fits into the total length of the buffer
         if(item.varLength.dataSize > dataSize - offset) {
-            /* The size of the entry in the index is inconsistent with the sum of the sizes of chunks */
             err = ESP_ERR_NVS_INVALID_LENGTH;
             break;
         }
-        err = findPage->readItem(nsIndex, ItemType::BLOB_DATA, key, static_cast<uint8_t*>(data) + offset, item.varLength.dataSize, static_cast<uint8_t> (chunkStart) + chunkNum);
+
+        err = findPage->readVariableLengthItemData(item, itemIndex, static_cast<uint8_t*>(data) + offset);
         if(err != ESP_OK) {
             return err;
         }
+
         NVS_ASSERT_OR_RETURN(static_cast<uint8_t> (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL);
 
         offset += item.varLength.dataSize;
@@ -599,8 +728,9 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void
 {
     Item item;
     Page* findPage = nullptr;
+    size_t itemIndex = 0;
 
-    /* First read the blob index */
+    // First read the blob index
     auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
     if(err != ESP_OK) {
         return err;
@@ -615,15 +745,28 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void
         return ESP_ERR_NVS_CONTENT_DIFFERS;
     }
 
-    /* Now read corresponding chunks */
+    // Now read corresponding chunks
     for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) {
-        err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum);
+        err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum, nvs::VerOffset::VER_ANY, &itemIndex);
         if(err != ESP_OK) {
             if(err == ESP_ERR_NVS_NOT_FOUND) {
                 break;
             }
             return err;
         }
+
+        if(item.varLength.dataSize > dataSize - offset) {
+            // The size of the entry in the index is bigger than the size of the remaining data to be compared
+            return ESP_ERR_NVS_CONTENT_DIFFERS;
+        }
+
+        // calculate crc32 of the incoming data window related to the BLOB_DATA chunk and compare it with the crc32 from the BLOB_DATA metadata entry
+        // Different crc32 indicates data mismatch.
+        // If crc32 matches, we have to compare the data in the chunk with the buffer data to exclude crc32 collision.
+        if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+            return ESP_ERR_NVS_CONTENT_DIFFERS;
+        }
+
         err = findPage->cmpItem(nsIndex, ItemType::BLOB_DATA, key, static_cast<const uint8_t*>(data) + offset, item.varLength.dataSize, static_cast<uint8_t> (chunkStart) + chunkNum);
         if(err != ESP_OK) {
             return err;
@@ -667,13 +810,18 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse
     }
     Item item;
     Page* findPage = nullptr;
+    size_t itemIndex = 0;
+    uint8_t chunkCount = 0;
 
-    auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart);
+    auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart, &itemIndex);
     if(err != ESP_OK) {
         return err;
     }
+
+    chunkCount = item.blobIndex.chunkCount;
+
     // Erase the index first and make children blobs orphan
-    err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart);
+    err = findPage->eraseEntryAndSpan(itemIndex);
     if(err != ESP_OK) {
         return err;
     }
@@ -681,50 +829,78 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse
     // If caller requires delete of VER_ANY
     // We may face dirty NVS partition and version duplicates can be there
     // Make second attempt to delete index and ignore eventual not found
-    if(chunkStart == VerOffset::VER_ANY)
+    if(chunkStart == VerOffset::VER_ANY) {
-    {
+        // Specific case called during initialisation of the storage
-        err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart);
+        // We need to delete all chunks with the same key and namespace index
+
+        // If there exists another BLOB_IDX with the same key and namespace index, delete it
+        // Ignore potential error if the item is not found
+        err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart, &itemIndex);
         if(err == ESP_OK) {
-            err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart);
+            err = findPage->eraseEntryAndSpan(itemIndex);
             if(err != ESP_OK) {
                 return err;
             }
         } else if(err != ESP_ERR_NVS_NOT_FOUND) {
             return err;
         }
-    }
-
-    // setup limits for chunkIndex-es to be deleted
-    uint8_t minChunkIndex = (uint8_t) VerOffset::VER_0_OFFSET;
-    uint8_t maxChunkIndex = (uint8_t) VerOffset::VER_ANY;
-
-    if(chunkStart == VerOffset::VER_0_OFFSET) {
-        maxChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET;
-    } else if (chunkStart == VerOffset::VER_1_OFFSET) {
-        minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET;
-    }
 
+        // To delete all chunks, we will visit every page and delete all chunks regardless of chunkIndex
+        // This approach cannot use the hash list as the chunkIndex is not known.
         for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
-        size_t itemIndex = 0;
+            // reset itemIndex to zero for each page to search from the beginning
+            itemIndex = 0;
             do {
+                // (Re)Try to find the item at the position starting at the itemIndex
                 err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item);
+
+                // If the item is not found, we can break the actual loop and continue with the next page
                 if(err == ESP_ERR_NVS_NOT_FOUND) {
                     break;
                 } else if(err == ESP_OK) {
-                // check if item.chunkIndex is within the version range indicated by chunkStart, if so, delete it
-                if((item.chunkIndex >= minChunkIndex) && (item.chunkIndex < maxChunkIndex)) {
                     err = it->eraseEntryAndSpan(itemIndex);
-                }
 
-                // continue findItem until end of page
+                    // advance itemIndex to the next potential entry on the page
+                    // findItem checks the consistency of the entry metadata so we can safely assume the span is non-zero
                     itemIndex += item.span;
                 }
                 if(err != ESP_OK) {
                     return err;
                 }
+            // Continue the loop until all items on the current page are found and erased
             } while(err == ESP_OK && itemIndex < Page::ENTRY_COUNT);
         }
 
+    } else {
+        // Most common condition
+        // The caller has specified the chunk version, delete all chunks within the chunk index range indicated by the BLOB_IDX entry
+        // The loop will iterate the chunk index, page will be found and chunk index will be erased
+        // This approach uses the hash list to find the item on the page, so it is efficient.
+        uint8_t minChunkIndex = (uint8_t) VerOffset::VER_ANY;
+        uint8_t maxChunkIndex = (uint8_t) VerOffset::VER_ANY;
+
+        if(chunkStart == VerOffset::VER_0_OFFSET) {
+            minChunkIndex = (uint8_t) VerOffset::VER_0_OFFSET;
+            maxChunkIndex = minChunkIndex + chunkCount;
+        } else if(chunkStart == VerOffset::VER_1_OFFSET) {
+            minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET;
+            maxChunkIndex = minChunkIndex + chunkCount;
+        }
+
+        for(uint8_t chunkIndex = minChunkIndex; chunkIndex < maxChunkIndex; chunkIndex++) {
+            err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, chunkIndex, nvs::VerOffset::VER_ANY, &itemIndex);
+            if(err != ESP_OK) {
+                return err;
+            }
+
+            // Erase the entry
+            err = findPage->eraseEntryAndSpan(itemIndex);
+            if(err != ESP_OK) {
+                return err;
+            }
+        }
+    }
+
     return ESP_OK;
 }
 
@@ -734,22 +910,21 @@ esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key
         return ESP_ERR_NVS_NOT_INITIALIZED;
     }
 
-    if (datatype == ItemType::BLOB) {
-        return eraseMultiPageBlob(nsIndex, key);
-    }
-
     Item item;
     Page* findPage = nullptr;
-    auto err = findItem(nsIndex, datatype, key, findPage, item);
+    esp_err_t err = ESP_OK;
+    size_t itemIndex = 0;
+
+    err = findItem(nsIndex, datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
     if(err != ESP_OK) {
         return err;
     }
-
+    // If the item found is BLOB_IDX, the eraseMultiPageBlob is used to erase the whole multi-page blob.
-    if (item.datatype == ItemType::BLOB_DATA || item.datatype == ItemType::BLOB_IDX) {
+    if (item.datatype == ItemType::BLOB_IDX) {
-        return eraseMultiPageBlob(nsIndex, key);
+        return eraseMultiPageBlob(nsIndex, key, item.blobIndex.chunkStart);
     }
 
-    return findPage->eraseItem(nsIndex, datatype, key);
+    return findPage->eraseEntryAndSpan(itemIndex);
 }
 
 esp_err_t Storage::eraseNamespace(uint8_t nsIndex)
@@ -773,6 +948,26 @@ esp_err_t Storage::eraseNamespace(uint8_t nsIndex)
 
 }
 
+esp_err_t Storage::findKey(const uint8_t nsIndex, const char* key, ItemType* datatype)
+{
+    if (mState != StorageState::ACTIVE) {
+        return ESP_ERR_NVS_NOT_INITIALIZED;
+    }
+
+    Item item;
+    Page* findPage = nullptr;
+    auto err = findItem(nsIndex, ItemType::ANY, key, findPage, item);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    if(datatype != nullptr) {
+        *datatype = item.datatype;
+    }
+
+    return err;
+}
+
 esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize)
 {
     if(mState != StorageState::ACTIVE) {
@@ -781,19 +976,24 @@ esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const cha
 
     Item item;
     Page* findPage = nullptr;
-    auto err = findItem(nsIndex, datatype, key, findPage, item);
+    esp_err_t err = ESP_OK;
-    if (err != ESP_OK) {
+
-        if (datatype != ItemType::BLOB) {
+    // If requested datatype is BLOB, first try to find the item with datatype BLOB_IDX - new format
-            return err;
+    // If not found, try to find the item with datatype BLOB - old format.
-        }
+    if(datatype == ItemType::BLOB) {
         err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
-        if (err != ESP_OK) {
+        if(err == ESP_OK) {
+            dataSize = item.blobIndex.dataSize;
+            return err;
+        } else if(err != ESP_ERR_NVS_NOT_FOUND) {
             return err;
         }
-        dataSize = item.blobIndex.dataSize;
-        return ESP_OK;
     }
 
+    err = findItem(nsIndex, datatype, key, findPage, item);
+    if(err != ESP_OK) {
+        return err;
+    }
     dataSize = item.varLength.dataSize;
     return ESP_OK;
 }

components/nvs_flash/src/nvs_storage.hpp

@@ -74,6 +74,8 @@ public:
 
     esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize);
 
+    esp_err_t findKey(const uint8_t nsIndex, const char* key, ItemType* datatype);
+
     esp_err_t getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize);
 
     esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key);
@@ -149,7 +151,7 @@ protected:
 
     void fillEntryInfo(Item &item, nvs_entry_info_t &info);
 
-    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);
+    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY, size_t* itemIndex = NULL);
 
 protected:
     Partition *mPartition;

components/nvs_flash/src/nvs_types.cpp

@@ -33,9 +33,12 @@ uint32_t Item::calculateCrc32WithoutValue() const
     return result;
 }
 
-uint32_t Item::calculateCrc32(const uint8_t* data, size_t size)
+uint32_t Item::calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32)
 {
     uint32_t result = 0xffffffff;
+    if(initial_crc32) {
+        result = *initial_crc32;
+    }
     result = esp_rom_crc32_le(result, data, size);
     return result;
 }

components/nvs_flash/src/nvs_types.hpp

@@ -93,7 +93,7 @@ public:
 
     uint32_t calculateCrc32() const;
     uint32_t calculateCrc32WithoutValue() const;
-    static uint32_t calculateCrc32(const uint8_t* data, size_t size);
+    static uint32_t calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32 = nullptr);
 
     void getKey(char* dst, size_t dstSize)
     {

docs/en/api-reference/storage/nvs_flash.rst

@@ -35,12 +35,9 @@ NVS operates on key-value pairs. Keys are ASCII strings; the maximum key length
 
 Additional types, such as ``float`` and ``double`` might be added later.
 
-Keys are required to be unique. Assigning a new value to an existing key works as follows:
+Keys are required to be unique. Assigning a new value to an existing key replaces the old value and data type with the value and data type specified by a write operation.
 
--  If the new value is of the same type as the old one, value is updated.
+A data type check is performed when reading a value. An error is returned if the data type expected by read operation does not match the data type of entry found for the key provided.
--  If the new value has a different data type, an error is returned.
-
-Data type check is also performed when reading a value. An error is returned if the data type of the read operation does not match the data type of the value.
 
 
 Namespaces

docs/zh_CN/api-reference/storage/nvs_flash.rst

@@ -35,12 +35,9 @@ NVS 的操作对象为键值对，其中键是 ASCII 字符串，当前支持的
 
 后续可能会增加对 ``float`` 和 ``double`` 等其他类型数据的支持。
 
-键必须唯一。为现有的键写入新的值可能产生如下结果：
+键必须唯一。为现有的键写入新值时，会将旧的值及数据类型更新为写入操作指定的值和数据类型。
 
--  如果新旧值数据类型相同，则更新值；
+读取值时会执行数据类型检查。如果读取操作预期的数据类型与对应键的数据类型不匹配，则返回错误。
--  如果新旧值数据类型不同，则返回错误。
-
-读取值时也会执行数据类型检查。如果读取操作的数据类型与该值的数据类型不匹配，则返回错误。
 
 
 命名空间

tools/ci/check_copyright_ignore.txt

@@ -700,9 +700,6 @@ components/nvs_flash/include/nvs_flash.h
 components/nvs_flash/include/nvs_handle.hpp
 components/nvs_flash/src/nvs_cxx_api.cpp
 components/nvs_flash/src/nvs_encrypted_partition.hpp
-components/nvs_flash/src/nvs_handle_locked.cpp
-components/nvs_flash/src/nvs_handle_locked.hpp
-components/nvs_flash/src/nvs_handle_simple.cpp
 components/nvs_flash/src/nvs_item_hash_list.cpp
 components/nvs_flash/src/nvs_pagemanager.hpp
 components/nvs_flash/src/nvs_partition.cpp