espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-10-04 11:00:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(nvs_flash): nvs_set with same key and different data type

Browse Source 
Function now always rewrites old value under same key
regardless existing data type. Users requiring old API behaviour
can enable it by kconfig option CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY
This commit is contained in:
radek.tandler
2023-08-25 17:15:18 +02:00
parent d4b1bfafba
commit 521463503f
16 changed files with 391 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
components/nvs_flash/.build-test-rules.yml Normal file
View File

@@ -0,0 +1,3 @@
components/nvs_flash/host_test:
enable:
- if: IDF_TARGET == "linux"

9
components/nvs_flash/Kconfig
View File

@@ -25,4 +25,13 @@ menu "NVS"
default n
help
This option switches error checking type between assertions (y) or return codes (n).
config NVS_LEGACY_DUP_KEYS_COMPATIBILITY
bool "Enable legacy nvs_set function behavior when same key is reused with different data types"
default n
help
Enabling this will switch the nvs_set family of functions to the legacy mode. When called with same
key and different data type, existing value stored in NVS remains active and as a side effect, the
new value is also stored into NVS, although not accessible using respective nvs_get function. Use only
if your application relies on this NVS API behaviour.
endmenu

1
components/nvs_flash/host_test/nvs_host_test/sdkconfig.ci.default_set_key Normal file
View File

@@ -0,0 +1 @@
# CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY=n

1
components/nvs_flash/host_test/nvs_host_test/sdkconfig.ci.legacy_set_key Normal file
View File

@@ -0,0 +1 @@
CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY=y

21
components/nvs_flash/include/nvs.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -464,6 +464,25 @@ esp_err_t nvs_get_str (nvs_handle_t handle, const char* key, char* out_value, si
esp_err_t nvs_get_blob(nvs_handle_t handle, const char* key, void* out_value, size_t* length);
/**@}*/
/**
* @brief Lookup key-value pair with given key name.
*
* Note that function may indicate both existence of the key as well as the data type of NVS entry if it is found.
*
* @param[in] handle Storage handle obtained with nvs_open.
* @param[in] key Key name. Maximum length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty.
* @param[out] out_type Pointer to the output variable populated with data type of NVS entry in case key was found.
* May be NULL, respective data type is then not provided.
* @return
* - ESP_OK if NVS entry for key provided was found
* - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist
* - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL
* - ESP_FAIL if there is an internal error; most likely due to corrupted
* NVS partition (only if NVS assertion checks are disabled)
* - other error codes from the underlying storage driver
*/
esp_err_t nvs_find_key(nvs_handle_t handle, const char* key, nvs_type_t* out_type);
/**
* @brief Erase key-value pair with given key name.
*

15
components/nvs_flash/include/nvs_handle.hpp
View File

@@ -164,6 +164,21 @@ public:
*/
virtual esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) = 0;
/**
* @brief Checks whether key exists and optionally returns also data type of associated entry.
*
* @param[in] key Key name. Maximum length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty.
* @param[out] nvstype Nvs data type to of entry, if it exists.
*
* @return - ESP_OK if NVS entry for key provided was found. Data type will be returned via \c nvstype.
* - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist.
* - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL.
* - ESP_FAIL if there is an internal error; most likely due to corrupted
* NVS partition (only if NVS assertion checks are disabled).
* - other error codes from the underlying storage driver.
*/
virtual esp_err_t find_key(const char* key, nvs_type_t &nvstype) = 0;
/**
* @brief Erases an entry.
*/

21
components/nvs_flash/src/nvs_api.cpp
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -310,6 +310,25 @@ extern "C" void nvs_close(nvs_handle_t handle)
delete static_cast<NVSHandleEntry*>(it);
}
extern "C" esp_err_t nvs_find_key(nvs_handle_t c_handle, const char* key, nvs_type_t* out_type)
{
Lock lock;
ESP_LOGD(TAG, "%s %s", __func__, key);
NVSHandleSimple *handle;
auto err = nvs_find_ns_handle(c_handle, &handle);
if (err != ESP_OK) {
return err;
}
nvs_type_t nvstype;
err = handle->find_key(key, nvstype);
if(err == ESP_OK && out_type != nullptr)
*out_type = nvstype;
return err;
}
extern "C" esp_err_t nvs_erase_key(nvs_handle_t c_handle, const char* key)
{
Lock lock;

24
components/nvs_flash/src/nvs_handle_locked.cpp
View File

@@ -1,16 +1,8 @@
// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "nvs_handle_locked.hpp"
namespace nvs {
@@ -47,6 +39,12 @@ esp_err_t NVSHandleLocked::get_item_size(ItemType datatype, const char *key, siz
return handle->get_item_size(datatype, key, size);
}
esp_err_t NVSHandleLocked::find_key(const char* key, nvs_type_t &nvstype)
{
Lock lock;
return handle->find_key(key, nvstype);
}
esp_err_t NVSHandleLocked::erase_item(const char* key) {
Lock lock;
return handle->erase_item(key);

20
components/nvs_flash/src/nvs_handle_locked.hpp
View File

@@ -1,16 +1,8 @@
// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef NVS_HANDLE_LOCKED_HPP_
#define NVS_HANDLE_LOCKED_HPP_
@@ -49,6 +41,8 @@ public:
esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) override;
esp_err_t find_key(const char* key, nvs_type_t &nvstype) override;
esp_err_t erase_item(const char* key) override;
esp_err_t erase_all() override;

35
components/nvs_flash/src/nvs_handle_simple.cpp
View File

@@ -1,16 +1,8 @@
// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <cstdlib>
#include "nvs_handle.hpp"
#include "nvs_partition_manager.hpp"
@@ -73,6 +65,23 @@ esp_err_t NVSHandleSimple::get_item_size(ItemType datatype, const char *key, siz
return mStoragePtr->getItemDataSize(mNsIndex, datatype, key, size);
}
esp_err_t NVSHandleSimple::find_key(const char* key, nvs_type_t &nvstype)
{
if (!valid) return ESP_ERR_NVS_INVALID_HANDLE;
nvs::ItemType datatype;
esp_err_t err = mStoragePtr->findKey(mNsIndex, key, &datatype);
if(err != ESP_OK)
return err;
if(datatype == ItemType::BLOB_IDX || datatype == ItemType::BLOB)
datatype = ItemType::BLOB_DATA;
nvstype = (nvs_type_t) datatype;
return err;
}
esp_err_t NVSHandleSimple::erase_item(const char* key)
{
if (!valid) return ESP_ERR_NVS_INVALID_HANDLE;

2
components/nvs_flash/src/nvs_handle_simple.hpp
View File

@@ -51,6 +51,8 @@ public:
esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) override;
esp_err_t find_key(const char *key, nvs_type_t &nvstype) override;
esp_err_t erase_item(const char *key) override;
esp_err_t erase_all() override;

200
components/nvs_flash/src/nvs_storage.cpp
View File

@@ -359,87 +359,133 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
return err;
}
// datatype BLOB is written as BLOB_INDEX and BLOB_DATA and is searched for previous value as BLOB_INDEX and/or BLOB
// datatype BLOB_IDX and BLOB_DATA are not supported as input parameters, the layer above should always use BLOB
esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize)
{
if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED;
}
// pointer to the page where the existing item was found
Page* findPage = nullptr;
// page sequence number helping to detect whether the page with old value was relocated during the new write
uint32_t findPageSeqNumber = UINT32_MAX;
// indicates the datatype representation match between the old value and the new one
bool matchedTypePageFound = false;
// contains the item with the old value, if found
Item item;
esp_err_t err;
esp_err_t err = ESP_OK;
// Try to find existing item with the same key and namespace index
// We are performing the findItem with datatype specified (it is not ANY) to ensure the hash list lookup is done.
if(datatype == ItemType::BLOB) {
// Specific lookup if performed for input datatype BLOB. The searched datatype for exact match is BLOB_INDEX.
// The BLOB_INDEX is used to store the metadata of the (originally typed) BLOB data in current V2 implementation.
err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
if(err == ESP_OK && findPage != nullptr) {
matchedTypePageFound = true;
}
} else {
// Handle all other data types than BLOB
err = findItem(nsIndex, datatype, key, findPage, item);
if(err == ESP_OK && findPage != nullptr) {
matchedTypePageFound = true;
// keep the sequence number of the page where the item was found for later check of relocation
err = findPage->getSeqNumber(findPageSeqNumber);
if(err != ESP_OK) {
return err;
}
}
}
#ifndef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY
// If the item was not found, try to find it with any other datatype. Omit the BLOB_DATA and handle BLOB with respect
// to the V1 and V2 representation.
const ItemType dataTypes[] = {
ItemType::U8,
ItemType::U16,
ItemType::U32,
ItemType::U64,
ItemType::I8,
ItemType::I16,
ItemType::I32,
ItemType::I64,
ItemType::SZ,
ItemType::BLOB_IDX,
ItemType::BLOB
};
if(findPage == nullptr) {
// Iterate over potential data types to allow findItem() search using the hash list instead of bruteforce search.
for(const auto& currType : dataTypes) {
// Skip search for BLOB_IDX if the requested datatype is BLOB. BLOB_IDX was already searched above.
if(datatype == ItemType::BLOB && currType == ItemType::BLOB_IDX) continue;
// Skip search if requested datatype is not BLOB and the current datatype is equal to the requested one. This was already searched above.
if(datatype != ItemType::BLOB && currType == datatype) continue;
err = findItem(nsIndex, currType, key, findPage, item);
if(err == ESP_OK && findPage != nullptr) {
// keep the sequence number of the page where the item was found for later check of relocation
err = findPage->getSeqNumber(findPageSeqNumber);
if(err != ESP_OK) {
return err;
}
// item was found with the same key and namespace index but data type is different
matchedTypePageFound = false;
break;
}
}
}
#endif
// Here the findPage is either nullptr or points to the page where the item was found.
// The matchedTypePageFound is true if the old value item was found and its datatype representation matches the new one.
// This flag is used to determine if the item should be checked for same value.
if(err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
return err;
}
// Handle value update
if(datatype == ItemType::BLOB) {
VerOffset prevStart, nextStart;
prevStart = nextStart = VerOffset::VER_0_OFFSET;
if (findPage) {
// Do a sanity check that the item in question is actually being modified.
if(matchedTypePageFound) {
// Do a check that the item in question is actually being modified.
// If it isn't, it is cheaper to purposefully not write out new data.
// since it may invoke an erasure of flash.
if(cmpMultiPageBlob(nsIndex, key, data, dataSize) == ESP_OK) {
return ESP_OK;
}
if (findPage->state() == Page::PageState::UNINITIALIZED ||
findPage->state() == Page::PageState::INVALID) {
err = findItem(nsIndex, datatype, key, findPage, item);
if (err != ESP_OK) {
return err;
}
}
/* Get the version of the previous index with same <ns,key> */
// Get the version of the previous index with same <ns,key>
prevStart = item.blobIndex.chunkStart;
NVS_ASSERT_OR_RETURN(prevStart == VerOffset::VER_0_OFFSET || prevStart == VerOffset::VER_1_OFFSET, ESP_FAIL);
/* Toggle the version by changing the offset */
// Toggle the version by changing the offset
nextStart
= (prevStart == VerOffset::VER_1_OFFSET) ? VerOffset::VER_0_OFFSET : VerOffset::VER_1_OFFSET;
}
/* Write the blob with new version*/
// Write the blob with new version
err = writeMultiPageBlob(nsIndex, key, data, dataSize, nextStart);
if(err == ESP_ERR_NVS_PAGE_FULL) {
return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
}
if(err != ESP_OK) {
return err;
}
if (findPage) {
/* Erase the blob with earlier version*/
err = eraseMultiPageBlob(nsIndex, key, prevStart);
if (err == ESP_ERR_FLASH_OP_FAIL) {
return ESP_ERR_NVS_REMOVE_FAILED;
}
if (err != ESP_OK) {
return err;
}
findPage = nullptr;
} else {
/* Support for earlier versions where BLOBS were stored without index */
err = findItem(nsIndex, datatype, key, findPage, item);
if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
return err;
}
}
} else {
// Do a sanity check that the item in question is actually being modified.
// Do a check that the item in question is actually being modified.
// If it isn't, it is cheaper to purposefully not write out new data.
// since it may invoke an erasure of flash.
if (findPage != nullptr &&
if(matchedTypePageFound &&
findPage->cmpItem(nsIndex, datatype, key, data, dataSize) == ESP_OK) {
return ESP_OK;
}
@@ -462,34 +508,74 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key
if(err == ESP_ERR_NVS_PAGE_FULL) {
return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
}
if (err != ESP_OK) {
return err;
}
} else if (err != ESP_OK) {
if(err != ESP_OK) {
return err;
}
}
if (findPage) {
if (findPage->state() == Page::PageState::UNINITIALIZED ||
findPage->state() == Page::PageState::INVALID) {
err = findItem(nsIndex, datatype, key, findPage, item);
if (err != ESP_OK) {
// Delete previous value
// Note: The old entry won't be deleted if the new value is the same as the old value - code won't reach here in that case.
// If findPage is null then previous value was not present in NVS and nothig is to be deleted.
if(findPage == nullptr) {
return err;
}
}
err = findPage->eraseItem(nsIndex, datatype, key);
if(item.datatype == ItemType::BLOB_IDX) {
// If the item found was BLOB_INDEX, the eraseMultiPageBlob is used to erase the whole multi-page blob.
// It is not necessary to check the potential page relocation as the function will find the blob again anyway.
VerOffset prevStart = item.blobIndex.chunkStart;
err = eraseMultiPageBlob(nsIndex, key, prevStart);
if(err == ESP_ERR_FLASH_OP_FAIL) {
return ESP_ERR_NVS_REMOVE_FAILED;
}
} else {
// For all other data types, we have to check the potential page relocation.
// The findPage might have been relocated as a part of space reclaiming.
// First indication is the page state. It might become the "spare" page thus changing the state from FULL or ACTIVE.
bool wasRelocated = false;
if( findPage->state() != Page::PageState::ACTIVE &&
findPage->state() != Page::PageState::FULL) {
wasRelocated = true;
}
// Other indication of the multi step relocation is the page sequence number. If the sequence number is different than page
// sequence number at the moment initial item was found, the page was relocated.
if(!wasRelocated) {
uint32_t newPageSeqNumber;
err = findPage->getSeqNumber(newPageSeqNumber);
if(err != ESP_OK) {
return err;
}
if(newPageSeqNumber != findPageSeqNumber) {
wasRelocated = true;
}
}
if(wasRelocated) {
// The page was relocated. We have to find the old value again from the beginning.
// As the item was already found before relocation, we can use the exact datatype from item.
err = findItem(nsIndex, item.datatype, key, findPage, item);
if(err != ESP_OK) {
return err;
}
}
// Page containing the old value is now refreshed. We can erase the old value.
err = findPage->eraseItem(nsIndex, item.datatype, key);
if(err == ESP_ERR_FLASH_OP_FAIL) {
return ESP_ERR_NVS_REMOVE_FAILED;
}
}
#ifdef DEBUG_STORAGE
debugCheck();
#endif
return ESP_OK;
return err;
}
esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uint8_t& nsIndex)
@@ -773,6 +859,26 @@ esp_err_t Storage::eraseNamespace(uint8_t nsIndex)
}
esp_err_t Storage::findKey(const uint8_t nsIndex, const char* key, ItemType* datatype)
{
if (mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED;
}
Item item;
Page* findPage = nullptr;
auto err = findItem(nsIndex, ItemType::ANY, key, findPage, item);
if (err != ESP_OK) {
return err;
}
if(datatype != nullptr) {
*datatype = item.datatype;
}
return err;
}
esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize)
{
if(mState != StorageState::ACTIVE) {

2
components/nvs_flash/src/nvs_storage.hpp
View File

@@ -74,6 +74,8 @@ public:
esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize);
esp_err_t findKey(const uint8_t nsIndex, const char* key, ItemType* datatype);
esp_err_t getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize);
esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key);

7
docs/en/api-reference/storage/nvs_flash.rst
View File

@@ -35,12 +35,9 @@ NVS operates on key-value pairs. Keys are ASCII strings; the maximum key length
Additional types, such as ``float`` and ``double`` might be added later.
Keys are required to be unique. Assigning a new value to an existing key works as follows:
Keys are required to be unique. Assigning a new value to an existing key replaces the old value and data type with the value and data type specified by a write operation.
- If the new value is of the same type as the old one, value is updated.
- If the new value has a different data type, an error is returned.
Data type check is also performed when reading a value. An error is returned if the data type of the read operation does not match the data type of the value.
A data type check is performed when reading a value. An error is returned if the data type expected by read operation does not match the data type of entry found for the key provided.
Namespaces

7
docs/zh_CN/api-reference/storage/nvs_flash.rst
View File

@@ -35,12 +35,9 @@ NVS 的操作对象为键值对，其中键是 ASCII 字符串，当前支持的
后续可能会增加对 ``float`` 和 ``double`` 等其他类型数据的支持。
键必须唯一。为现有的键写入新的值可能产生如下结果：
键必须唯一。为现有的键写入新值时，会将旧的值及数据类型更新为写入操作指定的值和数据类型。
- 如果新旧值数据类型相同，则更新值；
- 如果新旧值数据类型不同，则返回错误。
读取值时也会执行数据类型检查。如果读取操作的数据类型与该值的数据类型不匹配，则返回错误。
读取值时会执行数据类型检查。如果读取操作预期的数据类型与对应键的数据类型不匹配，则返回错误。
命名空间

3
tools/ci/check_copyright_ignore.txt
View File

@@ -700,9 +700,6 @@ components/nvs_flash/include/nvs_flash.h
components/nvs_flash/include/nvs_handle.hpp
components/nvs_flash/src/nvs_cxx_api.cpp
components/nvs_flash/src/nvs_encrypted_partition.hpp
components/nvs_flash/src/nvs_handle_locked.cpp
components/nvs_flash/src/nvs_handle_locked.hpp
components/nvs_flash/src/nvs_handle_simple.cpp
components/nvs_flash/src/nvs_item_hash_list.cpp
components/nvs_flash/src/nvs_pagemanager.hpp
components/nvs_flash/src/nvs_partition.cpp