feedc0de/esp-idf
1
0
Fork 0
forked from espressif/esp-idf
Code Pull Requests Activity

fix(nvs): fixed erasing of old values if new data type is not the same

Browse Source 
Closes https://github.com/espressif/esp-idf/issues/15559
This commit is contained in:
radek.tandler
2025-03-19 19:43:55 +01:00
parent 13953d5e59
commit d9bc77e422
2 changed files with 330 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp
View File

@@ -1,5 +1,5 @@
/* /*
* SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
* *
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
*/ */
@@ -3806,6 +3806,82 @@ TEST_CASE("nvs multiple write with same key but different types", "[nvs]")
TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME));
} }
TEST_CASE("nvs multiple write with same key blob and string involved", "[nvs]")
{
PartitionEmulationFixture f(0, 10);
nvs_handle_t handle_1;
const uint32_t NVS_FLASH_SECTOR = 6;
const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3;
TEMPORARILY_DISABLED(f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN);)
for (uint16_t j = NVS_FLASH_SECTOR; j < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; ++j) {
f.erase(j);
}
TEST_ESP_OK(nvs::NVSPartitionManager::get_instance()->init_custom(f.part(),
NVS_FLASH_SECTOR,
NVS_FLASH_SECTOR_COUNT_MIN));
TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle_1));
nvs_erase_all(handle_1);
const char key_name[] = "foo";
// integer variables
int32_t v32;
int8_t v8;
// string
#define str_data_len 64
const char str_data[] = "string data";
char str_buf[str_data_len] = {0};
size_t str_len = str_data_len;
// blob
#define blob_data_len 64
uint8_t blob_data[blob_data_len] = {0};
uint8_t blob_buf[blob_data_len] = {0};
size_t blob_read_size;
// first write is i32
TEST_ESP_OK(nvs_set_i32(handle_1, key_name, (int32_t)12345678));
TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_OK(nvs_get_i32(handle_1, key_name, &v32));
TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND);
// second write is string
TEST_ESP_OK(nvs_set_str(handle_1, key_name, str_data));
TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_OK(nvs_get_str(handle_1, key_name, str_buf, &str_len));
TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND);
// third write is blob
TEST_ESP_OK(nvs_set_blob(handle_1, key_name, blob_data, blob_data_len));
TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_OK(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size));
// fourth write is i8
TEST_ESP_OK(nvs_set_i8(handle_1, key_name, (int8_t)12));
TEST_ESP_OK(nvs_get_i8(handle_1, key_name, &v8));
TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND);
TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND);
nvs_close(handle_1);
TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME));
}
TEST_CASE("nvs find key tests", "[nvs]") TEST_CASE("nvs find key tests", "[nvs]")
{ {
const size_t buff_len = 4096; const size_t buff_len = 4096;

442
components/nvs_flash/src/nvs_storage.cpp
View File

@@ -1,5 +1,5 @@
/* /*
* SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
* *
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
*/ */
@@ -51,7 +51,7 @@ void Storage::clearNamespaces()
esp_err_t Storage::populateBlobIndices(TBlobIndexList& blobIdxList) esp_err_t Storage::populateBlobIndices(TBlobIndexList& blobIdxList)
{ {
for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) {
Page& p = *it; Page& p = *it;
size_t itemIndex = 0; size_t itemIndex = 0;
Item item; Item item;
@@ -60,10 +60,10 @@ esp_err_t Storage::populateBlobIndices(TBlobIndexList& blobIdxList)
* logic in pagemanager will remove the earlier index. So we should never find a * logic in pagemanager will remove the earlier index. So we should never find a
* duplicate index at this point */ * duplicate index at this point */
while (p.findItem(Page::NS_ANY, ItemType::BLOB_IDX, nullptr, itemIndex, item) == ESP_OK) { while(p.findItem(Page::NS_ANY, ItemType::BLOB_IDX, nullptr, itemIndex, item) == ESP_OK) {
BlobIndexNode* entry = new (std::nothrow) BlobIndexNode; BlobIndexNode* entry = new (std::nothrow) BlobIndexNode;
if (!entry) return ESP_ERR_NO_MEM; if(!entry) return ESP_ERR_NO_MEM;
item.getKey(entry->key, sizeof(entry->key)); item.getKey(entry->key, sizeof(entry->key));
entry->nsIndex = item.nsIndex; entry->nsIndex = item.nsIndex;
@@ -89,7 +89,7 @@ esp_err_t Storage::populateBlobIndices(TBlobIndexList& blobIdxList)
// later by the call to eraseOrphanDataBlobs(). // later by the call to eraseOrphanDataBlobs().
void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList) void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList)
{ {
for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) {
Page& p = *it; Page& p = *it;
size_t itemIndex = 0; size_t itemIndex = 0;
Item item; Item item;
@@ -98,7 +98,7 @@ void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList)
* 1) VER_0_OFFSET <= chunkIndex < VER_1_OFFSET-1 => Version0 chunks * 1) VER_0_OFFSET <= chunkIndex < VER_1_OFFSET-1 => Version0 chunks
* 2) VER_1_OFFSET <= chunkIndex < VER_ANY => Version1 chunks * 2) VER_1_OFFSET <= chunkIndex < VER_ANY => Version1 chunks
*/ */
while (p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) { while(p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) {
auto iter = std::find_if(blobIdxList.begin(), auto iter = std::find_if(blobIdxList.begin(),
blobIdxList.end(), blobIdxList.end(),
@@ -107,7 +107,7 @@ void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList)
&& (item.nsIndex == e.nsIndex) && (item.nsIndex == e.nsIndex)
&& (item.chunkIndex >= static_cast<uint8_t> (e.chunkStart)) && (item.chunkIndex >= static_cast<uint8_t> (e.chunkStart))
&& (item.chunkIndex < static_cast<uint8_t> ((e.chunkStart == nvs::VerOffset::VER_0_OFFSET) ? nvs::VerOffset::VER_1_OFFSET : nvs::VerOffset::VER_ANY));}); && (item.chunkIndex < static_cast<uint8_t> ((e.chunkStart == nvs::VerOffset::VER_0_OFFSET) ? nvs::VerOffset::VER_1_OFFSET : nvs::VerOffset::VER_ANY));});
if (iter != std::end(blobIdxList)) { if(iter != std::end(blobIdxList)) {
// accumulate the size // accumulate the size
iter->observedDataSize += item.varLength.dataSize; iter->observedDataSize += item.varLength.dataSize;
iter->observedChunkCount++; iter->observedChunkCount++;
@@ -117,15 +117,15 @@ void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList)
} }
auto iter = blobIdxList.begin(); auto iter = blobIdxList.begin();
while (iter != blobIdxList.end()) while(iter != blobIdxList.end())
{ {
if ( (iter->observedDataSize != iter->dataSize) || (iter->observedChunkCount != iter->chunkCount) ) if( (iter->observedDataSize != iter->dataSize) || (iter->observedChunkCount != iter->chunkCount) )
{ {
// Delete blob_index from flash // Delete blob_index from flash
// This is very rare case, so we can loop over all pages // This is very rare case, so we can loop over all pages
for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) {
// skip pages in non eligible states // skip pages in non eligible states
if (it->state() == nvs::Page::PageState::CORRUPT if(it->state() == nvs::Page::PageState::CORRUPT
|| it->state() == nvs::Page::PageState::INVALID || it->state() == nvs::Page::PageState::INVALID
|| it->state() == nvs::Page::PageState::UNINITIALIZED){ || it->state() == nvs::Page::PageState::UNINITIALIZED){
continue; continue;
@@ -153,7 +153,7 @@ void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList)
void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList) void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList)
{ {
for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) {
Page& p = *it; Page& p = *it;
size_t itemIndex = 0; size_t itemIndex = 0;
Item item; Item item;
@@ -162,7 +162,7 @@ void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList)
* 1) VER_0_OFFSET <= chunkIndex < VER_1_OFFSET-1 => Version0 chunks * 1) VER_0_OFFSET <= chunkIndex < VER_1_OFFSET-1 => Version0 chunks
* 2) VER_1_OFFSET <= chunkIndex < VER_ANY => Version1 chunks * 2) VER_1_OFFSET <= chunkIndex < VER_ANY => Version1 chunks
*/ */
while (p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) { while(p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) {
auto iter = std::find_if(blobIdxList.begin(), auto iter = std::find_if(blobIdxList.begin(),
blobIdxList.end(), blobIdxList.end(),
@@ -171,7 +171,7 @@ void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList)
&& (item.nsIndex == e.nsIndex) && (item.nsIndex == e.nsIndex)
&& (item.chunkIndex >= static_cast<uint8_t> (e.chunkStart)) && (item.chunkIndex >= static_cast<uint8_t> (e.chunkStart))
&& (item.chunkIndex < static_cast<uint8_t> (e.chunkStart) + e.chunkCount);}); && (item.chunkIndex < static_cast<uint8_t> (e.chunkStart) + e.chunkCount);});
if (iter == std::end(blobIdxList)) { if(iter == std::end(blobIdxList)) {
p.eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex); p.eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex);
} }
@@ -183,7 +183,7 @@ void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList)
esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount)
{ {
auto err = mPageManager.load(mPartition, baseSector, sectorCount); auto err = mPageManager.load(mPartition, baseSector, sectorCount);
if (err != ESP_OK) { if(err != ESP_OK) {
mState = StorageState::INVALID; mState = StorageState::INVALID;
return err; return err;
} }
@@ -191,25 +191,25 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount)
// load namespaces list // load namespaces list
clearNamespaces(); clearNamespaces();
std::fill_n(mNamespaceUsage.data(), mNamespaceUsage.byteSize() / 4, 0); std::fill_n(mNamespaceUsage.data(), mNamespaceUsage.byteSize() / 4, 0);
for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) {
Page& p = *it; Page& p = *it;
size_t itemIndex = 0; size_t itemIndex = 0;
Item item; Item item;
while (p.findItem(Page::NS_INDEX, ItemType::U8, nullptr, itemIndex, item) == ESP_OK) { while(p.findItem(Page::NS_INDEX, ItemType::U8, nullptr, itemIndex, item) == ESP_OK) {
NamespaceEntry* entry = new (std::nothrow) NamespaceEntry; NamespaceEntry* entry = new (std::nothrow) NamespaceEntry;
if (!entry) { if(!entry) {
mState = StorageState::INVALID; mState = StorageState::INVALID;
return ESP_ERR_NO_MEM; return ESP_ERR_NO_MEM;
} }
item.getKey(entry->mName, sizeof(entry->mName)); item.getKey(entry->mName, sizeof(entry->mName));
err = item.getValue(entry->mIndex); err = item.getValue(entry->mIndex);
if (err != ESP_OK) { if(err != ESP_OK) {
delete entry; delete entry;
return err; return err;
} }
if (mNamespaceUsage.set(entry->mIndex, true) != ESP_OK) { if(mNamespaceUsage.set(entry->mIndex, true) != ESP_OK) {
delete entry; delete entry;
return ESP_FAIL; return ESP_FAIL;
} }
@@ -217,17 +217,17 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount)
itemIndex += item.span; itemIndex += item.span;
} }
} }
if (mNamespaceUsage.set(0, true) != ESP_OK) { if(mNamespaceUsage.set(0, true) != ESP_OK) {
return ESP_FAIL; return ESP_FAIL;
} }
if (mNamespaceUsage.set(255, true) != ESP_OK) { if(mNamespaceUsage.set(255, true) != ESP_OK) {
return ESP_FAIL; return ESP_FAIL;
} }
// Populate list of multi-page index entries. // Populate list of multi-page index entries.
TBlobIndexList blobIdxList; TBlobIndexList blobIdxList;
err = populateBlobIndices(blobIdxList); err = populateBlobIndices(blobIdxList);
if (err != ESP_OK) { if(err != ESP_OK) {
mState = StorageState::INVALID; mState = StorageState::INVALID;
return ESP_ERR_NO_MEM; return ESP_ERR_NO_MEM;
} }
@@ -256,10 +256,10 @@ bool Storage::isValid() const
esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart) esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart)
{ {
for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
size_t itemIndex = 0; size_t itemIndex = 0;
auto err = it->findItem(nsIndex, datatype, key, itemIndex, item, chunkIdx, chunkStart); auto err = it->findItem(nsIndex, datatype, key, itemIndex, item, chunkIdx, chunkStart);
if (err == ESP_OK) { if(err == ESP_OK) {
page = it; page = it;
return ESP_OK; return ESP_OK;
} }
@@ -282,7 +282,7 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
max_pages = (Page::CHUNK_ANY-1)/2; max_pages = (Page::CHUNK_ANY-1)/2;
} }
if (dataSize > max_pages * Page::CHUNK_MAX_SIZE) { if(dataSize > max_pages * Page::CHUNK_MAX_SIZE) {
return ESP_ERR_NVS_VALUE_TOO_LONG; return ESP_ERR_NVS_VALUE_TOO_LONG;
} }
@@ -290,16 +290,16 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
Page& page = getCurrentPage(); Page& page = getCurrentPage();
size_t tailroom = page.getVarDataTailroom(); size_t tailroom = page.getVarDataTailroom();
size_t chunkSize = 0; size_t chunkSize = 0;
if (chunkCount == 0U && ((tailroom < dataSize) || (tailroom == 0 && dataSize == 0)) && tailroom < Page::CHUNK_MAX_SIZE/10) { if(chunkCount == 0U && ((tailroom < dataSize) || (tailroom == 0 && dataSize == 0)) && tailroom < Page::CHUNK_MAX_SIZE/10) {
/** This is the first chunk and tailroom is too small ***/ /** This is the first chunk and tailroom is too small ***/
if (page.state() != Page::PageState::FULL) { if(page.state() != Page::PageState::FULL) {
err = page.markFull(); err = page.markFull();
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
} }
err = mPageManager.requestNewPage(); err = mPageManager.requestNewPage();
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} else if(getCurrentPage().getVarDataTailroom() == tailroom) { } else if(getCurrentPage().getVarDataTailroom() == tailroom) {
/* We got the same page or we are not improving.*/ /* We got the same page or we are not improving.*/
@@ -307,7 +307,7 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
} else { } else {
continue; continue;
} }
} else if (!tailroom) { } else if(!tailroom) {
err = ESP_ERR_NVS_NOT_ENOUGH_SPACE; err = ESP_ERR_NVS_NOT_ENOUGH_SPACE;
break; break;
} }
@@ -322,32 +322,32 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
static_cast<const uint8_t*> (data) + offset, chunkSize, static_cast<uint8_t> (chunkStart) + chunkCount); static_cast<const uint8_t*> (data) + offset, chunkSize, static_cast<uint8_t> (chunkStart) + chunkCount);
chunkCount++; chunkCount++;
if (err != ESP_OK) { if(err != ESP_OK) {
NVS_ASSERT_OR_RETURN(err != ESP_ERR_NVS_PAGE_FULL, err); NVS_ASSERT_OR_RETURN(err != ESP_ERR_NVS_PAGE_FULL, err);
break; break;
} else { } else {
UsedPageNode* node = new (std::nothrow) UsedPageNode(); UsedPageNode* node = new (std::nothrow) UsedPageNode();
if (!node) { if(!node) {
err = ESP_ERR_NO_MEM; err = ESP_ERR_NO_MEM;
break; break;
} }
node->mPage = &page; node->mPage = &page;
usedPages.push_back(node); usedPages.push_back(node);
if (remainingSize || (tailroom - chunkSize) < Page::ENTRY_SIZE) { if(remainingSize || (tailroom - chunkSize) < Page::ENTRY_SIZE) {
if (page.state() != Page::PageState::FULL) { if(page.state() != Page::PageState::FULL) {
err = page.markFull(); err = page.markFull();
if (err != ESP_OK) { if(err != ESP_OK) {
break; break;
} }
} }
err = mPageManager.requestNewPage(); err = mPageManager.requestNewPage();
if (err != ESP_OK) { if(err != ESP_OK) {
break; break;
} }
} }
} }
offset += chunkSize; offset += chunkSize;
if (!remainingSize) { if(!remainingSize) {
/* All pages are stored. Now store the index.*/ /* All pages are stored. Now store the index.*/
Item item; Item item;
std::fill_n(item.data, sizeof(item.data), 0xff); std::fill_n(item.data, sizeof(item.data), 0xff);
@@ -359,12 +359,12 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
NVS_ASSERT_OR_RETURN(err != ESP_ERR_NVS_PAGE_FULL, err); NVS_ASSERT_OR_RETURN(err != ESP_ERR_NVS_PAGE_FULL, err);
break; break;
} }
} while (1); } while(1);
if (err != ESP_OK) { if(err != ESP_OK) {
/* Anything failed, then we should erase all the written chunks*/ /* Anything failed, then we should erase all the written chunks*/
int ii=0; int ii=0;
for (auto it = std::begin(usedPages); it != std::end(usedPages); it++) { for(auto it = std::begin(usedPages); it != std::end(usedPages); it++) {
it->mPage->eraseItem(nsIndex, ItemType::BLOB_DATA, key, ii++); it->mPage->eraseItem(nsIndex, ItemType::BLOB_DATA, key, ii++);
} }
} }
@@ -372,200 +372,264 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
return err; return err;
} }
// datatype BLOB is written as BLOB_INDEX and BLOB_DATA and is searched for previous value as BLOB_INDEX and/or BLOB
// datatype BLOB_IDX and BLOB_DATA are not supported as input parameters, the layer above should always use BLOB
esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize)
{ {
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
// pointer to the page where the existing item was found
Page* findPage = nullptr; Page* findPage = nullptr;
// page sequence number helping to detect whether the page with old value was relocated during the new write
uint32_t findPageSeqNumber = UINT32_MAX;
// indicates the datatype representation match between the old value and the new one
bool matchedTypePageFound = false; bool matchedTypePageFound = false;
// contains the item with the old value, if found
Item item; Item item;
esp_err_t err; esp_err_t err = ESP_OK;
if (datatype == ItemType::BLOB) {
// Try to find existing item with the same key and namespace index
// We are performing the findItem with datatype specified (it is not ANY) to ensure the hash list lookup is done.
if(datatype == ItemType::BLOB) {
// Specific lookup if performed for input datatype BLOB. The searched datatype for exact match is BLOB_INDEX.
// The BLOB_INDEX is used to store the metadata of the (originally typed) BLOB data in current V2 implementation.
err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
if(err == ESP_OK) {
matchedTypePageFound = true;
}
} else {
#ifdef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY
err = findItem(nsIndex, datatype, key, findPage, item);
if(err == ESP_OK && findPage != nullptr) { if(err == ESP_OK && findPage != nullptr) {
matchedTypePageFound = true; matchedTypePageFound = true;
} }
#else } else {
err = findItem(nsIndex, ItemType::ANY, key, findPage, item); // Handle all other data types than BLOB
if(err == ESP_OK && datatype == item.datatype) { err = findItem(nsIndex, datatype, key, findPage, item);
if(err == ESP_OK && findPage != nullptr) {
matchedTypePageFound = true; matchedTypePageFound = true;
// keep the sequence number of the page where the item was found for later check of relocation
err = findPage->getSeqNumber(findPageSeqNumber);
if(err != ESP_OK) {
return err;
}
} }
#endif
} }
if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { #ifndef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY
// If the item was not found, try to find it with any other datatype. Omit the BLOB_DATA and handle BLOB with respect
// to the V1 and V2 representation.
const ItemType dataTypes[] = {
ItemType::U8,
ItemType::U16,
ItemType::U32,
ItemType::U64,
ItemType::I8,
ItemType::I16,
ItemType::I32,
ItemType::I64,
ItemType::SZ,
ItemType::BLOB_IDX,
ItemType::BLOB
};
if(findPage == nullptr) {
// Iterate over potential data types to allow findItem() search using the hash list instead of bruteforce search.
for(const auto& currType : dataTypes) {
// Skip search for BLOB_IDX if the requested datatype is BLOB. BLOB_IDX was already searched above.
if(datatype == ItemType::BLOB && currType == ItemType::BLOB_IDX) continue;
// Skip search if requested datatype is not BLOB and the current datatype is equal to the requested one. This was already searched above.
if(datatype != ItemType::BLOB && currType == datatype) continue;
err = findItem(nsIndex, currType, key, findPage, item);
if(err == ESP_OK && findPage != nullptr) {
// keep the sequence number of the page where the item was found for later check of relocation
err = findPage->getSeqNumber(findPageSeqNumber);
if(err != ESP_OK) {
return err;
}
// item was found with the same key and namespace index but data type is different
matchedTypePageFound = false;
break;
}
}
}
#endif
// Here the findPage is either nullptr or points to the page where the item was found.
// The matchedTypePageFound is true if the old value item was found and its datatype representation matches the new one.
// This flag is used to determine if the item should be checked for same value.
if(err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
return err; return err;
} }
if (datatype == ItemType::BLOB) { // Handle value update
if(datatype == ItemType::BLOB) {
VerOffset prevStart, nextStart; VerOffset prevStart, nextStart;
prevStart = nextStart = VerOffset::VER_0_OFFSET; prevStart = nextStart = VerOffset::VER_0_OFFSET;
if (matchedTypePageFound) { if(matchedTypePageFound) {
// Do a sanity check that the item in question is actually being modified. // Do a check that the item in question is actually being modified.
// If it isn't, it is cheaper to purposefully not write out new data. // If it isn't, it is cheaper to purposefully not write out new data.
// since it may invoke an erasure of flash. // since it may invoke an erasure of flash.
if (cmpMultiPageBlob(nsIndex, key, data, dataSize) == ESP_OK) { if(cmpMultiPageBlob(nsIndex, key, data, dataSize) == ESP_OK) {
return ESP_OK; return ESP_OK;
} }
if (findPage->state() == Page::PageState::UNINITIALIZED || // Get the version of the previous index with same <ns,key>
findPage->state() == Page::PageState::INVALID) {
err = findItem(nsIndex, datatype, key, findPage, item);
if (err != ESP_OK) {
return err;
}
}
/* Get the version of the previous index with same <ns,key> */
prevStart = item.blobIndex.chunkStart; prevStart = item.blobIndex.chunkStart;
NVS_ASSERT_OR_RETURN(prevStart == VerOffset::VER_0_OFFSET || prevStart == VerOffset::VER_1_OFFSET, ESP_FAIL); NVS_ASSERT_OR_RETURN(prevStart == VerOffset::VER_0_OFFSET || prevStart == VerOffset::VER_1_OFFSET, ESP_FAIL);
// Toggle the version by changing the offset
/* Toggle the version by changing the offset */
nextStart nextStart
= (prevStart == VerOffset::VER_1_OFFSET) ? VerOffset::VER_0_OFFSET : VerOffset::VER_1_OFFSET; = (prevStart == VerOffset::VER_1_OFFSET) ? VerOffset::VER_0_OFFSET : VerOffset::VER_1_OFFSET;
} }
/* Write the blob with new version*/ // Write the blob with new version
err = writeMultiPageBlob(nsIndex, key, data, dataSize, nextStart); err = writeMultiPageBlob(nsIndex, key, data, dataSize, nextStart);
if (err == ESP_ERR_NVS_PAGE_FULL) { if(err == ESP_ERR_NVS_PAGE_FULL) {
return ESP_ERR_NVS_NOT_ENOUGH_SPACE; return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
} }
if (err != ESP_OK) {
if(err != ESP_OK) {
return err; return err;
} }
if (matchedTypePageFound) {
/* Erase the blob with earlier version*/
err = eraseMultiPageBlob(nsIndex, key, prevStart);
if (err == ESP_ERR_FLASH_OP_FAIL) {
return ESP_ERR_NVS_REMOVE_FAILED;
}
if (err != ESP_OK) {
return err;
}
findPage = nullptr;
} else {
/* Support for earlier versions where BLOBS were stored without index */
err = findItem(nsIndex, datatype, key, findPage, item);
if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
return err;
}
}
} else { } else {
// Do a sanity check that the item in question is actually being modified. // Do a check that the item in question is actually being modified.
// If it isn't, it is cheaper to purposefully not write out new data. // If it isn't, it is cheaper to purposefully not write out new data.
// since it may invoke an erasure of flash. // since it may invoke an erasure of flash.
if (matchedTypePageFound && if(matchedTypePageFound &&
findPage->cmpItem(nsIndex, datatype, key, data, dataSize) == ESP_OK) { findPage->cmpItem(nsIndex, datatype, key, data, dataSize) == ESP_OK) {
return ESP_OK; return ESP_OK;
} }
Page& page = getCurrentPage(); Page& page = getCurrentPage();
err = page.writeItem(nsIndex, datatype, key, data, dataSize); err = page.writeItem(nsIndex, datatype, key, data, dataSize);
if (err == ESP_ERR_NVS_PAGE_FULL) { if(err == ESP_ERR_NVS_PAGE_FULL) {
if (page.state() != Page::PageState::FULL) { if(page.state() != Page::PageState::FULL) {
err = page.markFull(); err = page.markFull();
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
} }
err = mPageManager.requestNewPage(); err = mPageManager.requestNewPage();
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
err = getCurrentPage().writeItem(nsIndex, datatype, key, data, dataSize); err = getCurrentPage().writeItem(nsIndex, datatype, key, data, dataSize);
if (err == ESP_ERR_NVS_PAGE_FULL) { if(err == ESP_ERR_NVS_PAGE_FULL) {
return ESP_ERR_NVS_NOT_ENOUGH_SPACE; return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
} }
if (err != ESP_OK) { }
return err;
} if(err != ESP_OK) {
} else if (err != ESP_OK) {
return err; return err;
} }
} }
if (findPage) { // Delete previous value
if (findPage->state() == Page::PageState::UNINITIALIZED || // Note: The old entry won't be deleted if the new value is the same as the old value - code won't reach here in that case.
findPage->state() == Page::PageState::INVALID) {
#ifdef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY // If findPage is null then previous value was not present in NVS and nothig is to be deleted.
err = findItem(nsIndex, datatype, key, findPage, item); if(findPage == nullptr) {
#else return err;
err = findItem(nsIndex, ItemType::ANY, key, findPage, item); }
#endif
if (err != ESP_OK) { if(item.datatype == ItemType::BLOB_IDX) {
// If the item found was BLOB_INDEX, the eraseMultiPageBlob is used to erase the whole multi-page blob.
// It is not necessary to check the potential page relocation as the function will find the blob again anyway.
VerOffset prevStart = item.blobIndex.chunkStart;
err = eraseMultiPageBlob(nsIndex, key, prevStart);
if(err == ESP_ERR_FLASH_OP_FAIL) {
return ESP_ERR_NVS_REMOVE_FAILED;
}
} else {
// For all other data types, we have to check the potential page relocation.
// The findPage might have been relocated as a part of space reclaiming.
// First indication is the page state. It might become the "spare" page thus changing the state from FULL or ACTIVE.
bool wasRelocated = false;
if( findPage->state() != Page::PageState::ACTIVE &&
findPage->state() != Page::PageState::FULL) {
wasRelocated = true;
}
// Other indication of the multi step relocation is the page sequence number. If the sequence number is different than page
// sequence number at the moment initial item was found, the page was relocated.
if(!wasRelocated) {
uint32_t newPageSeqNumber;
err = findPage->getSeqNumber(newPageSeqNumber);
if(err != ESP_OK) {
return err;
}
if(newPageSeqNumber != findPageSeqNumber) {
wasRelocated = true;
}
}
if(wasRelocated) {
// The page was relocated. We have to find the old value again from the beginning.
// As the item was already found before relocation, we can use the exact datatype from item.
err = findItem(nsIndex, item.datatype, key, findPage, item);
if(err != ESP_OK) {
return err; return err;
} }
} }
#ifdef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY // Page containing the old value is now refreshed. We can erase the old value.
err = findPage->eraseItem(nsIndex, datatype, key); err = findPage->eraseItem(nsIndex, item.datatype, key);
#else if(err == ESP_ERR_FLASH_OP_FAIL) {
err = findPage->eraseItem(nsIndex, ItemType::ANY, key);
#endif
if (err == ESP_ERR_FLASH_OP_FAIL) {
return ESP_ERR_NVS_REMOVE_FAILED; return ESP_ERR_NVS_REMOVE_FAILED;
} }
if (err != ESP_OK) {
return err;
}
} }
#ifdef DEBUG_STORAGE #ifdef DEBUG_STORAGE
debugCheck(); debugCheck();
#endif #endif
return ESP_OK; return err;
} }
esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uint8_t& nsIndex) esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uint8_t& nsIndex)
{ {
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
auto it = std::find_if(mNamespaces.begin(), mNamespaces.end(), [=] (const NamespaceEntry& e) -> bool { auto it = std::find_if(mNamespaces.begin(), mNamespaces.end(), [=] (const NamespaceEntry& e) -> bool {
return strncmp(nsName, e.mName, sizeof(e.mName) - 1) == 0; return strncmp(nsName, e.mName, sizeof(e.mName) - 1) == 0;
}); });
if (it == std::end(mNamespaces)) { if(it == std::end(mNamespaces)) {
if (!canCreate) { if(!canCreate) {
return ESP_ERR_NVS_NOT_FOUND; return ESP_ERR_NVS_NOT_FOUND;
} }
uint8_t ns; uint8_t ns;
bool ns_state; bool ns_state;
for (ns = 1; ns < 255; ++ns) { for(ns = 1; ns < 255; ++ns) {
if (mNamespaceUsage.get(ns, &ns_state) != ESP_OK) { if(mNamespaceUsage.get(ns, &ns_state) != ESP_OK) {
return ESP_FAIL; return ESP_FAIL;
} }
if (!ns_state) { if(!ns_state) {
break; break;
} }
} }
if (ns == 255) { if(ns == 255) {
return ESP_ERR_NVS_NOT_ENOUGH_SPACE; return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
} }
auto err = writeItem(Page::NS_INDEX, ItemType::U8, nsName, &ns, sizeof(ns)); auto err = writeItem(Page::NS_INDEX, ItemType::U8, nsName, &ns, sizeof(ns));
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
if (mNamespaceUsage.set(ns, true) != ESP_OK) { if(mNamespaceUsage.set(ns, true) != ESP_OK) {
return ESP_FAIL; return ESP_FAIL;
} }
nsIndex = ns; nsIndex = ns;
NamespaceEntry* entry = new (std::nothrow) NamespaceEntry; NamespaceEntry* entry = new (std::nothrow) NamespaceEntry;
if (!entry) { if(!entry) {
return ESP_ERR_NO_MEM; return ESP_ERR_NO_MEM;
} }
@@ -587,7 +651,7 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat
/* First read the blob index */ /* First read the blob index */
auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
@@ -598,21 +662,21 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat
NVS_ASSERT_OR_RETURN(dataSize == item.blobIndex.dataSize, ESP_FAIL); NVS_ASSERT_OR_RETURN(dataSize == item.blobIndex.dataSize, ESP_FAIL);
/* Now read corresponding chunks */ /* Now read corresponding chunks */
for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) {
err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum); err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum);
if (err != ESP_OK) { if(err != ESP_OK) {
if (err == ESP_ERR_NVS_NOT_FOUND) { if(err == ESP_ERR_NVS_NOT_FOUND) {
break; break;
} }
return err; return err;
} }
if (item.varLength.dataSize > dataSize - offset) { if(item.varLength.dataSize > dataSize - offset) {
/* The size of the entry in the index is inconsistent with the sum of the sizes of chunks */ /* The size of the entry in the index is inconsistent with the sum of the sizes of chunks */
err = ESP_ERR_NVS_INVALID_LENGTH; err = ESP_ERR_NVS_INVALID_LENGTH;
break; break;
} }
err = findPage->readItem(nsIndex, ItemType::BLOB_DATA, key, static_cast<uint8_t*>(data) + offset, item.varLength.dataSize, static_cast<uint8_t> (chunkStart) + chunkNum); err = findPage->readItem(nsIndex, ItemType::BLOB_DATA, key, static_cast<uint8_t*>(data) + offset, item.varLength.dataSize, static_cast<uint8_t> (chunkStart) + chunkNum);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
NVS_ASSERT_OR_RETURN(static_cast<uint8_t> (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL); NVS_ASSERT_OR_RETURN(static_cast<uint8_t> (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL);
@@ -620,7 +684,7 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat
offset += item.varLength.dataSize; offset += item.varLength.dataSize;
} }
if (err == ESP_ERR_NVS_NOT_FOUND || err == ESP_ERR_NVS_INVALID_LENGTH) { if(err == ESP_ERR_NVS_NOT_FOUND || err == ESP_ERR_NVS_INVALID_LENGTH) {
// cleanup if a chunk is not found or the size is inconsistent // cleanup if a chunk is not found or the size is inconsistent
eraseMultiPageBlob(nsIndex, key); eraseMultiPageBlob(nsIndex, key);
} }
@@ -637,7 +701,7 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void
/* First read the blob index */ /* First read the blob index */
auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
@@ -646,21 +710,21 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void
size_t readSize = item.blobIndex.dataSize; size_t readSize = item.blobIndex.dataSize;
size_t offset = 0; size_t offset = 0;
if (dataSize != readSize) { if(dataSize != readSize) {
return ESP_ERR_NVS_CONTENT_DIFFERS; return ESP_ERR_NVS_CONTENT_DIFFERS;
} }
/* Now read corresponding chunks */ /* Now read corresponding chunks */
for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) {
err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum); err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum);
if (err != ESP_OK) { if(err != ESP_OK) {
if (err == ESP_ERR_NVS_NOT_FOUND) { if(err == ESP_ERR_NVS_NOT_FOUND) {
break; break;
} }
return err; return err;
} }
err = findPage->cmpItem(nsIndex, ItemType::BLOB_DATA, key, static_cast<const uint8_t*>(data) + offset, item.varLength.dataSize, static_cast<uint8_t> (chunkStart) + chunkNum); err = findPage->cmpItem(nsIndex, ItemType::BLOB_DATA, key, static_cast<const uint8_t*>(data) + offset, item.varLength.dataSize, static_cast<uint8_t> (chunkStart) + chunkNum);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
NVS_ASSERT_OR_RETURN(static_cast<uint8_t> (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL); NVS_ASSERT_OR_RETURN(static_cast<uint8_t> (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL);
@@ -674,21 +738,21 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void
esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize) esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize)
{ {
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
Item item; Item item;
Page* findPage = nullptr; Page* findPage = nullptr;
if (datatype == ItemType::BLOB) { if(datatype == ItemType::BLOB) {
auto err = readMultiPageBlob(nsIndex, key, data, dataSize); auto err = readMultiPageBlob(nsIndex, key, data, dataSize);
if (err != ESP_ERR_NVS_NOT_FOUND) { if(err != ESP_ERR_NVS_NOT_FOUND) {
return err; return err;
} // else check if the blob is stored with earlier version format without index } // else check if the blob is stored with earlier version format without index
} }
auto err = findItem(nsIndex, datatype, key, findPage, item); auto err = findItem(nsIndex, datatype, key, findPage, item);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
return findPage->readItem(nsIndex, datatype, key, data, dataSize); return findPage->readItem(nsIndex, datatype, key, data, dataSize);
@@ -697,19 +761,19 @@ esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key,
esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffset chunkStart) esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffset chunkStart)
{ {
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
Item item; Item item;
Page* findPage = nullptr; Page* findPage = nullptr;
auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart); auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
// Erase the index first and make children blobs orphan // Erase the index first and make children blobs orphan
err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart); err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
@@ -719,12 +783,12 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse
if(chunkStart == VerOffset::VER_ANY) if(chunkStart == VerOffset::VER_ANY)
{ {
err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart); err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart);
if (err == ESP_OK) { if(err == ESP_OK) {
err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart); err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
} else if (err != ESP_ERR_NVS_NOT_FOUND) { } else if(err != ESP_ERR_NVS_NOT_FOUND) {
return err; return err;
} }
} }
@@ -735,17 +799,17 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse
if(chunkStart == VerOffset::VER_0_OFFSET) { if(chunkStart == VerOffset::VER_0_OFFSET) {
maxChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET; maxChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET;
} else if (chunkStart == VerOffset::VER_1_OFFSET) { } else if(chunkStart == VerOffset::VER_1_OFFSET) {
minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET; minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET;
} }
for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
size_t itemIndex = 0; size_t itemIndex = 0;
do { do {
err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item); err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item);
if (err == ESP_ERR_NVS_NOT_FOUND) { if(err == ESP_ERR_NVS_NOT_FOUND) {
break; break;
} else if (err == ESP_OK) { } else if(err == ESP_OK) {
// check if item.chunkIndex is within the version range indicated by chunkStart, if so, delete it // check if item.chunkIndex is within the version range indicated by chunkStart, if so, delete it
if((item.chunkIndex >= minChunkIndex) && (item.chunkIndex < maxChunkIndex)) { if((item.chunkIndex >= minChunkIndex) && (item.chunkIndex < maxChunkIndex)) {
err = it->eraseEntryAndSpan(itemIndex); err = it->eraseEntryAndSpan(itemIndex);
@@ -757,7 +821,7 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse
if(err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
} while (err == ESP_OK && itemIndex < Page::ENTRY_COUNT); } while(err == ESP_OK && itemIndex < Page::ENTRY_COUNT);
} }
return ESP_OK; return ESP_OK;
@@ -765,22 +829,22 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse
esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key) esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key)
{ {
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
if (datatype == ItemType::BLOB) { if(datatype == ItemType::BLOB) {
return eraseMultiPageBlob(nsIndex, key); return eraseMultiPageBlob(nsIndex, key);
} }
Item item; Item item;
Page* findPage = nullptr; Page* findPage = nullptr;
auto err = findItem(nsIndex, datatype, key, findPage, item); auto err = findItem(nsIndex, datatype, key, findPage, item);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
if (item.datatype == ItemType::BLOB_DATA || item.datatype == ItemType::BLOB_IDX) { if(item.datatype == ItemType::BLOB_DATA || item.datatype == ItemType::BLOB_IDX) {
return eraseMultiPageBlob(nsIndex, key); return eraseMultiPageBlob(nsIndex, key);
} }
@@ -789,17 +853,17 @@ esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key
esp_err_t Storage::eraseNamespace(uint8_t nsIndex) esp_err_t Storage::eraseNamespace(uint8_t nsIndex)
{ {
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
while (true) { while(true) {
auto err = it->eraseItem(nsIndex, ItemType::ANY, nullptr); auto err = it->eraseItem(nsIndex, ItemType::ANY, nullptr);
if (err == ESP_ERR_NVS_NOT_FOUND) { if(err == ESP_ERR_NVS_NOT_FOUND) {
break; break;
} }
else if (err != ESP_OK) { else if(err != ESP_OK) {
return err; return err;
} }
} }
@@ -810,14 +874,14 @@ esp_err_t Storage::eraseNamespace(uint8_t nsIndex)
esp_err_t Storage::findKey(const uint8_t nsIndex, const char* key, ItemType* datatype) esp_err_t Storage::findKey(const uint8_t nsIndex, const char* key, ItemType* datatype)
{ {
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
Item item; Item item;
Page* findPage = nullptr; Page* findPage = nullptr;
auto err = findItem(nsIndex, ItemType::ANY, key, findPage, item); auto err = findItem(nsIndex, ItemType::ANY, key, findPage, item);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
@@ -830,19 +894,19 @@ esp_err_t Storage::findKey(const uint8_t nsIndex, const char* key, ItemType* dat
esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize) esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize)
{ {
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
Item item; Item item;
Page* findPage = nullptr; Page* findPage = nullptr;
auto err = findItem(nsIndex, datatype, key, findPage, item); auto err = findItem(nsIndex, datatype, key, findPage, item);
if (err != ESP_OK) { if(err != ESP_OK) {
if (datatype != ItemType::BLOB) { if(datatype != ItemType::BLOB) {
return err; return err;
} }
err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
if (err != ESP_OK) { if(err != ESP_OK) {
return err; return err;
} }
dataSize = item.blobIndex.dataSize; dataSize = item.blobIndex.dataSize;
@@ -855,7 +919,7 @@ esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const cha
void Storage::debugDump() void Storage::debugDump()
{ {
for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { for(auto p = mPageManager.begin(); p != mPageManager.end(); ++p) {
p->debugDump(); p->debugDump();
} }
} }
@@ -865,15 +929,15 @@ void Storage::debugCheck()
{ {
std::map<std::string, Page*> keys; std::map<std::string, Page*> keys;
for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { for(auto p = mPageManager.begin(); p != mPageManager.end(); ++p) {
size_t itemIndex = 0; size_t itemIndex = 0;
size_t usedCount = 0; size_t usedCount = 0;
Item item; Item item;
while (p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { while(p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) {
std::stringstream keyrepr; std::stringstream keyrepr;
keyrepr << static_cast<unsigned>(item.nsIndex) << "_" << static_cast<unsigned>(item.datatype) << "_" << item.key <<"_"<<static_cast<unsigned>(item.chunkIndex); keyrepr << static_cast<unsigned>(item.nsIndex) << "_" << static_cast<unsigned>(item.datatype) << "_" << item.key <<"_"<<static_cast<unsigned>(item.chunkIndex);
std::string keystr = keyrepr.str(); std::string keystr = keyrepr.str();
if (keys.find(keystr) != std::end(keys)) { if(keys.find(keystr) != std::end(keys)) {
printf("Duplicate key: %s\n", keystr.c_str()); printf("Duplicate key: %s\n", keystr.c_str());
debugDump(); debugDump();
assert(0); assert(0);
@@ -897,24 +961,24 @@ esp_err_t Storage::calcEntriesInNamespace(uint8_t nsIndex, size_t& usedEntries)
{ {
usedEntries = 0; usedEntries = 0;
if (mState != StorageState::ACTIVE) { if(mState != StorageState::ACTIVE) {
return ESP_ERR_NVS_NOT_INITIALIZED; return ESP_ERR_NVS_NOT_INITIALIZED;
} }
for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
size_t itemIndex = 0; size_t itemIndex = 0;
Item item; Item item;
while (true) { while(true) {
auto err = it->findItem(nsIndex, ItemType::ANY, nullptr, itemIndex, item); auto err = it->findItem(nsIndex, ItemType::ANY, nullptr, itemIndex, item);
if (err == ESP_ERR_NVS_NOT_FOUND) { if(err == ESP_ERR_NVS_NOT_FOUND) {
break; break;
} }
else if (err != ESP_OK) { else if(err != ESP_OK) {
return err; return err;
} }
usedEntries += item.span; usedEntries += item.span;
itemIndex += item.span; itemIndex += item.span;
if (itemIndex >= it->ENTRY_COUNT) break; if(itemIndex >= it->ENTRY_COUNT) break;
} }
} }
return ESP_OK; return ESP_OK;
@@ -926,7 +990,7 @@ void Storage::fillEntryInfo(Item &item, nvs_entry_info_t &info)
strncpy(info.key, item.key, sizeof(info.key) - 1); strncpy(info.key, item.key, sizeof(info.key) - 1);
info.key[sizeof(info.key) - 1] = '\0'; info.key[sizeof(info.key) - 1] = '\0';
for (auto &name : mNamespaces) { for(auto &name : mNamespaces) {
if(item.nsIndex == name.mIndex) { if(item.nsIndex == name.mIndex) {
strlcpy(info.namespace_name, name.mName, sizeof(info.namespace_name)); strlcpy(info.namespace_name, name.mName, sizeof(info.namespace_name));
break; break;
@@ -940,7 +1004,7 @@ bool Storage::findEntry(nvs_opaque_iterator_t* it, const char* namespace_name)
it->nsIndex = Page::NS_ANY; it->nsIndex = Page::NS_ANY;
it->page = mPageManager.begin(); it->page = mPageManager.begin();
if (namespace_name != nullptr) { if(namespace_name != nullptr) {
if(createOrOpenNamespace(namespace_name, false, it->nsIndex) != ESP_OK) { if(createOrOpenNamespace(namespace_name, false, it->nsIndex) != ESP_OK) {
return false; return false;
} }
@@ -977,7 +1041,7 @@ bool Storage::nextEntry(nvs_opaque_iterator_t* it)
Item item; Item item;
esp_err_t err; esp_err_t err;
for (auto page = it->page; page != mPageManager.end(); ++page) { for(auto page = it->page; page != mPageManager.end(); ++page) {
do { do {
err = page->findItem(it->nsIndex, (ItemType)it->type, nullptr, it->entryIndex, item); err = page->findItem(it->nsIndex, (ItemType)it->type, nullptr, it->entryIndex, item);
it->entryIndex += item.span; it->entryIndex += item.span;
@@ -986,7 +1050,7 @@ bool Storage::nextEntry(nvs_opaque_iterator_t* it)
it->page = page; it->page = page;
return true; return true;
} }
} while (err != ESP_ERR_NVS_NOT_FOUND); } while(err != ESP_ERR_NVS_NOT_FOUND);
it->entryIndex = 0; it->entryIndex = 0;
} }