espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-10-03 10:30:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/storage_nvs_perf_blob_v5.1' into 'release/v5.1'

Browse Source 
Improvement of NVS Blob performance (v5.1)

See merge request espressif/esp-idf!41595
This commit is contained in:
Martin Vychodil
2025-09-15 17:48:53 +08:00
parent b4afd60f0d 13cc380dbf
commit ac8e7a79c3
31 changed files with 714 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
components/nvs_flash/.build-test-rules.yml Normal file
View File

@@ -0,0 +1,3 @@
components/nvs_flash/host_test:
enable:
- if: IDF_TARGET == "linux"

9
components/nvs_flash/Kconfig
View File

@@ -25,4 +25,13 @@ menu "NVS"
default n
help
This option switches error checking type between assertions (y) or return codes (n).
config NVS_LEGACY_DUP_KEYS_COMPATIBILITY
bool "Enable legacy nvs_set function behavior when same key is reused with different data types"
default n
help
Enabling this will switch the nvs_set family of functions to the legacy mode. When called with same
key and different data type, existing value stored in NVS remains active and as a side effect, the
new value is also stored into NVS, although not accessible using respective nvs_get function. Use only
if your application relies on this NVS API behaviour.
endmenu

4
components/nvs_flash/host_test/nvs_host_test/CMakeLists.txt
View File

@@ -2,9 +2,7 @@ cmake_minimum_required(VERSION 3.16)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
set(COMPONENTS main)
# Freertos is included via common components. However, CATCH isn't compatible with the FreeRTOS component yet, hence
# using the FreeRTOS mock component.
# target.
# This test app doesn't require FreeRTOS, using mock instead
list(APPEND EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/mocks/freertos/")
project(nvs_host_test)

12
components/nvs_flash/host_test/nvs_host_test/main/CMakeLists.txt
View File

@@ -1,5 +1,4 @@
idf_component_register(SRCS "main.cpp"
"test_nvs.cpp"
idf_component_register(SRCS "test_nvs.cpp"
"test_partition_manager.cpp"
"test_nvs_cxx_api.cpp"
"test_nvs_handle.cpp"
@@ -9,6 +8,13 @@ idf_component_register(SRCS "main.cpp"
"../../../src"
"../../../private_include"
"../../../../mbedtls/mbedtls/include"
"../../../../../tools/catch"
WHOLE_ARCHIVE
REQUIRES nvs_flash)
if(CMAKE_C_COMPILER_ID MATCHES "Clang")
target_compile_options(${COMPONENT_LIB} PRIVATE -std=gnu++20)
endif()
# Currently 'main' for IDF_TARGET=linux is defined in freertos component.
# Since we are using a freertos mock here, need to let Catch2 provide 'main'.
target_link_libraries(${COMPONENT_LIB} PRIVATE Catch2WithMain)

2
components/nvs_flash/host_test/nvs_host_test/main/idf_component.yml Normal file
View File

@@ -0,0 +1,2 @@
dependencies:
espressif/catch2: "^3.4.0"

7
components/nvs_flash/host_test/nvs_host_test/main/main.cpp
View File

@@ -1,7 +0,0 @@
/*
* SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#define CATCH_CONFIG_MAIN
#include "catch.hpp"

90
components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp
View File

@@ -1,9 +1,9 @@
/*
* SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "catch.hpp"
#include <catch2/catch_test_macros.hpp>
#include "nvs.hpp"
#include "sdkconfig.h"
#include "nvs_partition_manager.hpp"
@@ -381,8 +381,7 @@ TEST_CASE("storage can find items on second page if first is not fully written a
{
PartitionEmulationFixture f(0, 3);
nvs::Storage storage(f.part());
CHECK(storage.init(0, 3) == ESP_OK);
int bar = 0;
TEST_ESP_OK(storage.init(0, 3));
uint8_t bigdata[(nvs::Page::CHUNK_MAX_SIZE - nvs::Page::ENTRY_SIZE) / 2] = {0};
// write one big chunk of data
ESP_ERROR_CHECK(storage.writeItem(0, nvs::ItemType::BLOB, "1", bigdata, sizeof(bigdata)));
@@ -504,7 +503,7 @@ TEST_CASE("erase operations are distributed among sectors", "[nvs]")
}
/* Check that erase counts are distributed between the remaining sectors */
const size_t max_erase_cnt = write_ops / nvs::Page::ENTRY_COUNT / (sectors - static_sectors) + 1;
TEMPORARILY_DISABLED(const size_t max_erase_cnt = write_ops / nvs::Page::ENTRY_COUNT / (sectors - static_sectors) + 1;)
for (size_t i = 0; i < sectors; ++i) {
TEMPORARILY_DISABLED(
auto erase_cnt = f.emu.getSectorEraseCount(i);
@@ -592,7 +591,7 @@ TEST_CASE("nvs api tests", "[nvs]")
nvs_handle_t handle_2;
TEST_ESP_OK(nvs_open("namespace2", NVS_READWRITE, &handle_2));
TEST_ESP_OK(nvs_set_i32(handle_2, "foo", 0x3456789a));
const char *str = "value 0123456789abcdef0123456789abcdef";
char str[] = "value 0123456789abcdef0123456789abcdef";
TEST_ESP_OK(nvs_set_str(handle_2, "key", str));
int32_t v1;
@@ -603,7 +602,7 @@ TEST_CASE("nvs api tests", "[nvs]")
TEST_ESP_OK(nvs_get_i32(handle_2, "foo", &v2));
CHECK(0x3456789a == v2);
char buf[strlen(str) + 1];
char buf[sizeof(str)];
size_t buf_len = sizeof(buf);
size_t buf_len_needed;
@@ -632,8 +631,6 @@ TEST_CASE("deinit partition doesn't affect other partition's open handles", "[nv
const char *OTHER_PARTITION_NAME = "other_part";
PartitionEmulationFixture f(0, 10);
PartitionEmulationFixture f_other(0, 10, OTHER_PARTITION_NAME);
const char *str = "value 0123456789abcdef0123456789abcdef";
const uint8_t blob[8] = {0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7};
nvs_handle_t handle_1;
const uint32_t NVS_FLASH_SECTOR = 6;
@@ -693,7 +690,6 @@ TEST_CASE("nvs_entry_info fails with ESP_ERR_INVALID_ARG if a parameter is NULL"
TEST_CASE("nvs_entry_info doesn't change iterator on parameter error", "[nvs]")
{
nvs_iterator_t it = reinterpret_cast<nvs_iterator_t>(0xbeef);
nvs_entry_info_t info;
REQUIRE(nvs_entry_info(it, nullptr) == ESP_ERR_INVALID_ARG);
CHECK(it == reinterpret_cast<nvs_iterator_t>(0xbeef));
@@ -972,8 +968,8 @@ TEST_CASE("wifi test", "[nvs]")
TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.opmode", opmode));
uint8_t country = 0;
TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.country", &opmode), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.country", opmode));
TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.country", &country), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.country", country));
char ssid[36];
size_t size = sizeof(ssid);
@@ -1175,12 +1171,13 @@ public:
blobBufLen = largeBlobLen ;
}
uint8_t buf[blobBufLen];
uint8_t* buf = new uint8_t[blobBufLen];
memset(buf, 0, blobBufLen);
size_t len = blobBufLen;
auto err = nvs_get_blob(handle, keys[index], buf, &len);
if (err == ESP_ERR_FLASH_OP_FAIL) {
delete [] buf;
return err;
}
if (!written[index]) {
@@ -1189,6 +1186,7 @@ public:
REQUIRE(err == ESP_OK);
REQUIRE(memcmp(buf, reinterpret_cast<const uint8_t *>(values[index]), blobBufLen) == 0);
}
delete [] buf;
break;
}
@@ -1239,7 +1237,6 @@ public:
case nvs::ItemType::SZ: {
char buf[strBufLen];
size_t len = strBufLen;
size_t strLen = gen() % (strBufLen - 1);
std::generate_n(buf, strLen, [&]() -> char {
@@ -1270,7 +1267,7 @@ public:
} else {
blobBufLen = largeBlobLen ;
}
uint8_t buf[blobBufLen];
uint8_t* buf = new uint8_t[blobBufLen];
memset(buf, 0, blobBufLen);
size_t blobLen = gen() % blobBufLen;
std::generate_n(buf, blobLen, [&]() -> uint8_t {
@@ -1279,16 +1276,19 @@ public:
auto err = nvs_set_blob(handle, keys[index], buf, blobLen);
if (err == ESP_ERR_FLASH_OP_FAIL) {
delete [] buf;
return err;
}
if (err == ESP_ERR_NVS_REMOVE_FAILED) {
written[index] = true;
memcpy(reinterpret_cast<uint8_t *>(values[index]), buf, blobBufLen);
delete [] buf;
return ESP_ERR_FLASH_OP_FAIL;
}
REQUIRE(err == ESP_OK);
written[index] = true;
memcpy(reinterpret_cast<char *>(values[index]), buf, blobBufLen);
delete [] buf;
break;
}
@@ -1751,8 +1751,6 @@ TEST_CASE("Check that orphaned blobs are erased during init", "[nvs]")
{
const size_t blob_size = nvs::Page::CHUNK_MAX_SIZE * 3 ;
uint8_t blob[blob_size] = {0x11};
uint8_t blob2[blob_size] = {0x22};
uint8_t blob3[blob_size] = {0x33};
PartitionEmulationFixture f(0, 5);
nvs::Storage storage(f.part());
@@ -2048,3 +2046,61 @@ TEST_CASE("Recovery from power-off during modification of blob present in old-fo
TEST_ESP_OK(nvs_flash_deinit_partition(f.part()->get_partition_name()));
}
TEST_CASE("Page handles invalid CRC of variable length items", "[nvs][cur]")
{
PartitionEmulationFixture f(0, 4);
{
nvs::Page p;
TEST_ESP_OK(p.load(f.part(), 0));
char buf[128] = {0};
TEST_ESP_OK(p.writeItem(1, nvs::ItemType::BLOB, "1", buf, sizeof(buf)));
}
// corrupt header of the item (64 is the offset of the first item in page)
uint32_t overwrite_buf = 0;
TEST_ESP_OK(esp_partition_write(&f.esp_partition, 64, &overwrite_buf, 4));
// load page again
{
nvs::Page p1;
TEST_ESP_OK(p1.load(f.part(), 0));
}
}
TEST_CASE("namespace name is deep copy", "[nvs]")
{
char ns_name[16];
strcpy(ns_name, "const_name");
nvs_handle_t handle_1;
nvs_handle_t handle_2;
const uint32_t NVS_FLASH_SECTOR = 6;
const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3;
PartitionEmulationFixture f(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN);
TEMPORARILY_DISABLED(f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN);)
TEST_ESP_OK( nvs::NVSPartitionManager::get_instance()->init_custom(f.part(),
NVS_FLASH_SECTOR,
NVS_FLASH_SECTOR_COUNT_MIN));
TEST_ESP_OK(nvs_open("const_name", NVS_READWRITE, &handle_1));
strcpy(ns_name, "just_kidding");
CHECK(nvs_open("just_kidding", NVS_READONLY, &handle_2) == ESP_ERR_NVS_NOT_FOUND);
nvs_close(handle_1);
nvs_close(handle_2);
nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME);
}
/* Add new tests above */
/* This test has to be the final one */
TEST_CASE("dump all performance data", "[nvs]")
{
std::cout << "====================" << std::endl << "Dumping benchmarks" << std::endl;
std::cout << s_perf.str() << std::endl;
std::cout << "====================" << std::endl;
}

2
components/nvs_flash/host_test/nvs_host_test/main/test_nvs_cxx_api.cpp
View File

@@ -3,7 +3,7 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "catch.hpp"
#include <catch2/catch_test_macros.hpp>
#include <algorithm>
#include <cstring>
#include "nvs_handle_simple.hpp"

2
components/nvs_flash/host_test/nvs_host_test/main/test_nvs_handle.cpp
View File

@@ -3,7 +3,7 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "catch.hpp"
#include <catch2/catch_test_macros.hpp>
#include <algorithm>
#include <cstring>
#include "nvs_handle_simple.hpp"

4
components/nvs_flash/host_test/nvs_host_test/main/test_nvs_initialization.cpp
View File

@@ -3,7 +3,7 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "catch.hpp"
#include <catch2/catch_test_macros.hpp>
#include "nvs.hpp"
#include "nvs_partition_manager.hpp"
#include "nvs_partition.hpp"
@@ -12,8 +12,6 @@
TEST_CASE("nvs_flash_init_partition_ptr fails due to nullptr arg", "[nvs_custom_part]")
{
const uint32_t NVS_FLASH_SECTOR = 6;
const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3;
uint8_t *p_part_desc_addr_start;
CHECK(esp_partition_file_mmap((const uint8_t **)&p_part_desc_addr_start) == ESP_OK);

2
components/nvs_flash/host_test/nvs_host_test/main/test_nvs_storage.cpp
View File

@@ -3,7 +3,7 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "catch.hpp"
#include <catch2/catch_test_macros.hpp>
#include <cstring>
#include "nvs_storage.hpp"
#include "nvs_partition_manager.hpp"

2
components/nvs_flash/host_test/nvs_host_test/main/test_partition_manager.cpp
View File

@@ -3,7 +3,7 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "catch.hpp"
#include <catch2/catch_test_macros.hpp>
#include <algorithm>
#include <cstring>
#include "nvs_handle_simple.hpp"

3
components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py
View File

@@ -6,5 +6,8 @@ from pytest_embedded import Dut
@pytest.mark.linux
@pytest.mark.host_test
@pytest.mark.parametrize('config', [
'default_set_key', 'legacy_set_key'
], indirect=True)
def test_nvs_host_linux(dut: Dut) -> None:
dut.expect_exact('All tests passed', timeout=60)

1
components/nvs_flash/host_test/nvs_host_test/sdkconfig.ci.default_set_key Normal file
View File

@@ -0,0 +1 @@
# CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY=n

1
components/nvs_flash/host_test/nvs_host_test/sdkconfig.ci.legacy_set_key Normal file
View File

@@ -0,0 +1 @@
CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY=y

21
components/nvs_flash/include/nvs.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -464,6 +464,25 @@ esp_err_t nvs_get_str (nvs_handle_t handle, const char* key, char* out_value, si
esp_err_t nvs_get_blob(nvs_handle_t handle, const char* key, void* out_value, size_t* length);
/**@}*/
/**
* @brief Lookup key-value pair with given key name.
*
* Note that function may indicate both existence of the key as well as the data type of NVS entry if it is found.
*
* @param[in] handle Storage handle obtained with nvs_open.
* @param[in] key Key name. Maximum length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty.
* @param[out] out_type Pointer to the output variable populated with data type of NVS entry in case key was found.
* May be NULL, respective data type is then not provided.
* @return
* - ESP_OK if NVS entry for key provided was found
* - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist
* - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL
* - ESP_FAIL if there is an internal error; most likely due to corrupted
* NVS partition (only if NVS assertion checks are disabled)
* - other error codes from the underlying storage driver
*/
esp_err_t nvs_find_key(nvs_handle_t handle, const char* key, nvs_type_t* out_type);
/**
* @brief Erase key-value pair with given key name.
*

15
components/nvs_flash/include/nvs_handle.hpp
View File

@@ -164,6 +164,21 @@ public:
*/
virtual esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) = 0;
/**
* @brief Checks whether key exists and optionally returns also data type of associated entry.
*
* @param[in] key Key name. Maximum length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty.
* @param[out] nvstype Nvs data type to of entry, if it exists.
*
* @return - ESP_OK if NVS entry for key provided was found. Data type will be returned via \c nvstype.
* - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist.
* - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL.
* - ESP_FAIL if there is an internal error; most likely due to corrupted
* NVS partition (only if NVS assertion checks are disabled).
* - other error codes from the underlying storage driver.
*/
virtual esp_err_t find_key(const char* key, nvs_type_t &nvstype) = 0;
/**
* @brief Erases an entry.
*/

21
components/nvs_flash/src/nvs_api.cpp
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -310,6 +310,25 @@ extern "C" void nvs_close(nvs_handle_t handle)
delete static_cast<NVSHandleEntry*>(it);
}
extern "C" esp_err_t nvs_find_key(nvs_handle_t c_handle, const char* key, nvs_type_t* out_type)
{
Lock lock;
ESP_LOGD(TAG, "%s %s", __func__, key);
NVSHandleSimple *handle;
auto err = nvs_find_ns_handle(c_handle, &handle);
if (err != ESP_OK) {
return err;
}
nvs_type_t nvstype;
err = handle->find_key(key, nvstype);
if(err == ESP_OK && out_type != nullptr)
*out_type = nvstype;
return err;
}
extern "C" esp_err_t nvs_erase_key(nvs_handle_t c_handle, const char* key)
{
Lock lock;

24
components/nvs_flash/src/nvs_handle_locked.cpp
View File

@@ -1,16 +1,8 @@
// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "nvs_handle_locked.hpp"
namespace nvs {
@@ -47,6 +39,12 @@ esp_err_t NVSHandleLocked::get_item_size(ItemType datatype, const char *key, siz
return handle->get_item_size(datatype, key, size);
}
esp_err_t NVSHandleLocked::find_key(const char* key, nvs_type_t &nvstype)
{
Lock lock;
return handle->find_key(key, nvstype);
}
esp_err_t NVSHandleLocked::erase_item(const char* key) {
Lock lock;
return handle->erase_item(key);

20
components/nvs_flash/src/nvs_handle_locked.hpp
View File

@@ -1,16 +1,8 @@
// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef NVS_HANDLE_LOCKED_HPP_
#define NVS_HANDLE_LOCKED_HPP_
@@ -49,6 +41,8 @@ public:
esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) override;
esp_err_t find_key(const char* key, nvs_type_t &nvstype) override;
esp_err_t erase_item(const char* key) override;
esp_err_t erase_all() override;

35
components/nvs_flash/src/nvs_handle_simple.cpp
View File

@@ -1,16 +1,8 @@
// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <cstdlib>
#include "nvs_handle.hpp"
#include "nvs_partition_manager.hpp"
@@ -73,6 +65,23 @@ esp_err_t NVSHandleSimple::get_item_size(ItemType datatype, const char *key, siz
return mStoragePtr->getItemDataSize(mNsIndex, datatype, key, size);
}
esp_err_t NVSHandleSimple::find_key(const char* key, nvs_type_t &nvstype)
{
if (!valid) return ESP_ERR_NVS_INVALID_HANDLE;
nvs::ItemType datatype;
esp_err_t err = mStoragePtr->findKey(mNsIndex, key, &datatype);
if(err != ESP_OK)
return err;
if(datatype == ItemType::BLOB_IDX || datatype == ItemType::BLOB)
datatype = ItemType::BLOB_DATA;
nvstype = (nvs_type_t) datatype;
return err;
}
esp_err_t NVSHandleSimple::erase_item(const char* key)
{
if (!valid) return ESP_ERR_NVS_INVALID_HANDLE;

2
components/nvs_flash/src/nvs_handle_simple.hpp
View File

@@ -51,6 +51,8 @@ public:
esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) override;
esp_err_t find_key(const char *key, nvs_type_t &nvstype) override;
esp_err_t erase_item(const char *key) override;
esp_err_t erase_all() override;

87
components/nvs_flash/src/nvs_page.cpp
View File

@@ -246,6 +246,43 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c
return ESP_OK;
}
// Reads the data entries of the variable length item.
// The metadata entry is already read in the item object.
// index is the index of the metadata entry on the page.
// data is pointer to the buffer where the data will be copied to. It has to be at least
// item.varLength.dataSize bytes long.
// The function returns ESP_OK if the data was read successfully, or an error code if there was an error.
esp_err_t Page::readVariableLengthItemData(const Item& item, const size_t index, void* data)
{
if (mState == PageState::INVALID) {
return ESP_ERR_NVS_INVALID_STATE;
}
esp_err_t rc;
uint8_t* dst = reinterpret_cast<uint8_t*>(data);
size_t left = item.varLength.dataSize;
for (size_t i = index + 1; i < index + item.span; ++i) {
Item ditem;
rc = readEntry(i, ditem);
if (rc != ESP_OK) {
return rc;
}
size_t willCopy = ENTRY_SIZE;
willCopy = (left < willCopy) ? left : willCopy;
memcpy(dst, ditem.rawData, willCopy);
left -= willCopy;
dst += willCopy;
}
if (Item::calculateCrc32(reinterpret_cast<uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
rc = eraseEntryAndSpan(index);
if (rc != ESP_OK) {
return rc;
}
return ESP_ERR_NVS_NOT_FOUND;
}
return ESP_OK;
}
esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart)
{
size_t index = 0;
@@ -273,28 +310,7 @@ esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, vo
return ESP_ERR_NVS_INVALID_LENGTH;
}
uint8_t* dst = reinterpret_cast<uint8_t*>(data);
size_t left = item.varLength.dataSize;
for (size_t i = index + 1; i < index + item.span; ++i) {
Item ditem;
rc = readEntry(i, ditem);
if (rc != ESP_OK) {
return rc;
}
size_t willCopy = ENTRY_SIZE;
willCopy = (left < willCopy) ? left : willCopy;
memcpy(dst, ditem.rawData, willCopy);
left -= willCopy;
dst += willCopy;
}
if (Item::calculateCrc32(reinterpret_cast<uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
rc = eraseEntryAndSpan(index);
if (rc != ESP_OK) {
return rc;
}
return ESP_ERR_NVS_NOT_FOUND;
}
return ESP_OK;
return readVariableLengthItemData(item, index, data);
}
esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart)
@@ -326,9 +342,23 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con
return ESP_ERR_NVS_INVALID_LENGTH;
}
// We have metadata of the variable length data chunk. It contains the length of the data and the crc32.
// As a first step we can calculate the crc32 of the data buffer to be compared with the crc32 of the item in the flash.
// If they are not equal, immediately return ESP_ERR_NVS_CONTENT_DIFFERS.
// If they are equal, to avoid crc32 collision false positive, we will read the data from the flash entry by entry and compare
// it with the respective chunk of input data buffer. The crc32 of the data read from the flash will be calculated on the fly.
// At the end, we will compare the crc32 of the data read from the flash with the crc32 of the metadata item in the flash to make sure
// that the data in the flash is not corrupted.
if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
return ESP_ERR_NVS_CONTENT_DIFFERS;
}
const uint8_t* dst = reinterpret_cast<const uint8_t*>(data);
size_t left = item.varLength.dataSize;
for (size_t i = index + 1; i < index + item.span; ++i) {
uint32_t accumulatedCRC32;
size_t initial_index = index + 1;
for (size_t i = initial_index; i < index + item.span; ++i) {
Item ditem;
rc = readEntry(i, ditem);
if (rc != ESP_OK) {
@@ -339,11 +369,18 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con
if (memcmp(dst, ditem.rawData, willCopy)) {
return ESP_ERR_NVS_CONTENT_DIFFERS;
}
// Calculate the crc32 of the actual ditem.rawData buffer. Do not pass accumulatedCRC32 in the first call.
// In the first call, calculateCrc32 will use its default. In the subsequent calls, accumulatedCRC32 is the crc32 of the previous buffer.
accumulatedCRC32 = Item::calculateCrc32(ditem.rawData, willCopy, (i == initial_index) ? nullptr : &accumulatedCRC32);
left -= willCopy;
dst += willCopy;
}
if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
return ESP_ERR_NVS_NOT_FOUND;
// Check if the CRC32 calculated on the fly matches the variable length data CRC32 indicated in the metadata entry.
// If they are not equal, it means the data in the flash is corrupt, we will return ESP_ERR_NVS_CONTENT_DIFFERS.
if (accumulatedCRC32 != item.varLength.dataCrc32) {
return ESP_ERR_NVS_CONTENT_DIFFERS;
}
return ESP_OK;

2
components/nvs_flash/src/nvs_page.hpp
View File

@@ -88,6 +88,8 @@ public:
esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY);
esp_err_t readVariableLengthItemData(const Item& item, const size_t index, void* data);
esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);
esp_err_t cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);

668
components/nvs_flash/src/nvs_storage.cpp
View File

File diff suppressed because it is too large Load Diff

4
components/nvs_flash/src/nvs_storage.hpp
View File

@@ -74,6 +74,8 @@ public:
esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize);
esp_err_t findKey(const uint8_t nsIndex, const char* key, ItemType* datatype);
esp_err_t getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize);
esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key);
@@ -149,7 +151,7 @@ protected:
void fillEntryInfo(Item &item, nvs_entry_info_t &info);
esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);
esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY, size_t* itemIndex = NULL);
protected:
Partition *mPartition;

5
components/nvs_flash/src/nvs_types.cpp
View File

@@ -33,9 +33,12 @@ uint32_t Item::calculateCrc32WithoutValue() const
return result;
}
uint32_t Item::calculateCrc32(const uint8_t* data, size_t size)
uint32_t Item::calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32)
{
uint32_t result = 0xffffffff;
if(initial_crc32) {
result = *initial_crc32;
}
result = esp_rom_crc32_le(result, data, size);
return result;
}

2
components/nvs_flash/src/nvs_types.hpp
View File

@@ -93,7 +93,7 @@ public:
uint32_t calculateCrc32() const;
uint32_t calculateCrc32WithoutValue() const;
static uint32_t calculateCrc32(const uint8_t* data, size_t size);
static uint32_t calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32 = nullptr);
void getKey(char* dst, size_t dstSize)
{

7
docs/en/api-reference/storage/nvs_flash.rst
View File

@@ -35,12 +35,9 @@ NVS operates on key-value pairs. Keys are ASCII strings; the maximum key length
Additional types, such as ``float`` and ``double`` might be added later.
Keys are required to be unique. Assigning a new value to an existing key works as follows:
Keys are required to be unique. Assigning a new value to an existing key replaces the old value and data type with the value and data type specified by a write operation.
- If the new value is of the same type as the old one, value is updated.
- If the new value has a different data type, an error is returned.
Data type check is also performed when reading a value. An error is returned if the data type of the read operation does not match the data type of the value.
A data type check is performed when reading a value. An error is returned if the data type expected by read operation does not match the data type of entry found for the key provided.
Namespaces

7
docs/zh_CN/api-reference/storage/nvs_flash.rst
View File

@@ -35,12 +35,9 @@ NVS 的操作对象为键值对，其中键是 ASCII 字符串，当前支持的
后续可能会增加对 ``float`` 和 ``double`` 等其他类型数据的支持。
键必须唯一。为现有的键写入新的值可能产生如下结果：
键必须唯一。为现有的键写入新值时，会将旧的值及数据类型更新为写入操作指定的值和数据类型。
- 如果新旧值数据类型相同，则更新值；
- 如果新旧值数据类型不同，则返回错误。
读取值时也会执行数据类型检查。如果读取操作的数据类型与该值的数据类型不匹配，则返回错误。
读取值时会执行数据类型检查。如果读取操作预期的数据类型与对应键的数据类型不匹配，则返回错误。
命名空间

3
tools/ci/check_copyright_ignore.txt
View File

@@ -700,9 +700,6 @@ components/nvs_flash/include/nvs_flash.h
components/nvs_flash/include/nvs_handle.hpp
components/nvs_flash/src/nvs_cxx_api.cpp
components/nvs_flash/src/nvs_encrypted_partition.hpp
components/nvs_flash/src/nvs_handle_locked.cpp
components/nvs_flash/src/nvs_handle_locked.hpp
components/nvs_flash/src/nvs_handle_simple.cpp
components/nvs_flash/src/nvs_item_hash_list.cpp
components/nvs_flash/src/nvs_pagemanager.hpp
components/nvs_flash/src/nvs_partition.cpp