From c23c21f3c07ffa1cdcbd37fb11f127f02ea2e761 Mon Sep 17 00:00:00 2001 From: "radek.tandler" Date: Fri, 9 May 2025 21:42:25 +0200 Subject: [PATCH] refactor(nvs_flash): Improved Blob performance The findItem method was improved to use a hash list in RAM when searching for BLOB data chunks The findItem method was extended with a parameter that returns the position of an item on the page, if it is found The algorithm for matching existing variable-length data (such as strings and BLOBs) with new values was enhanced by comparing the CRC32 of the data chunks before reading the data from flash --- components/nvs_flash/src/nvs_page.cpp | 87 ++++++--- components/nvs_flash/src/nvs_page.hpp | 2 + components/nvs_flash/src/nvs_storage.cpp | 232 +++++++++++++---------- components/nvs_flash/src/nvs_storage.hpp | 2 +- components/nvs_flash/src/nvs_types.cpp | 5 +- components/nvs_flash/src/nvs_types.hpp | 2 +- 6 files changed, 206 insertions(+), 124 deletions(-) diff --git a/components/nvs_flash/src/nvs_page.cpp b/components/nvs_flash/src/nvs_page.cpp index f16c4a3b83..d0049c4f08 100644 --- a/components/nvs_flash/src/nvs_page.cpp +++ b/components/nvs_flash/src/nvs_page.cpp @@ -250,6 +250,43 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c return ESP_OK; } +// Reads the data entries of the variable length item. +// The metadata entry is already read in the item object. +// index is the index of the metadata entry on the page. +// data is pointer to the buffer where the data will be copied to. It has to be at least +// item.varLength.dataSize bytes long. +// The function returns ESP_OK if the data was read successfully, or an error code if there was an error. +esp_err_t Page::readVariableLengthItemData(const Item& item, const size_t index, void* data) +{ + if (mState == PageState::INVALID) { + return ESP_ERR_NVS_INVALID_STATE; + } + + esp_err_t rc; + uint8_t* dst = reinterpret_cast(data); + size_t left = item.varLength.dataSize; + for (size_t i = index + 1; i < index + item.span; ++i) { + Item ditem; + rc = readEntry(i, ditem); + if (rc != ESP_OK) { + return rc; + } + size_t willCopy = ENTRY_SIZE; + willCopy = (left < willCopy) ? left : willCopy; + memcpy(dst, ditem.rawData, willCopy); + left -= willCopy; + dst += willCopy; + } + if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { + rc = eraseEntryAndSpan(index); + if (rc != ESP_OK) { + return rc; + } + return ESP_ERR_NVS_NOT_FOUND; + } + return ESP_OK; +} + esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart) { size_t index = 0; @@ -277,28 +314,7 @@ esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, vo return ESP_ERR_NVS_INVALID_LENGTH; } - uint8_t* dst = reinterpret_cast(data); - size_t left = item.varLength.dataSize; - for (size_t i = index + 1; i < index + item.span; ++i) { - Item ditem; - rc = readEntry(i, ditem); - if (rc != ESP_OK) { - return rc; - } - size_t willCopy = ENTRY_SIZE; - willCopy = (left < willCopy) ? left : willCopy; - memcpy(dst, ditem.rawData, willCopy); - left -= willCopy; - dst += willCopy; - } - if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { - rc = eraseEntryAndSpan(index); - if (rc != ESP_OK) { - return rc; - } - return ESP_ERR_NVS_NOT_FOUND; - } - return ESP_OK; + return readVariableLengthItemData(item, index, data); } esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart) @@ -330,9 +346,23 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con return ESP_ERR_NVS_INVALID_LENGTH; } + // We have metadata of the variable length data chunk. It contains the length of the data and the crc32. + // As a first step we can calculate the crc32 of the data buffer to be compared with the crc32 of the item in the flash. + // If they are not equal, immediately return ESP_ERR_NVS_CONTENT_DIFFERS. + // If they are equal, to avoid crc32 collision false positive, we will read the data from the flash entry by entry and compare + // it with the respective chunk of input data buffer. The crc32 of the data read from the flash will be calculated on the fly. + // At the end, we will compare the crc32 of the data read from the flash with the crc32 of the metadata item in the flash to make sure + // that the data in the flash is not corrupted. + if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + const uint8_t* dst = reinterpret_cast(data); size_t left = item.varLength.dataSize; - for (size_t i = index + 1; i < index + item.span; ++i) { + uint32_t accumulatedCRC32; + size_t initial_index = index + 1; + + for (size_t i = initial_index; i < index + item.span; ++i) { Item ditem; rc = readEntry(i, ditem); if (rc != ESP_OK) { @@ -343,11 +373,18 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con if (memcmp(dst, ditem.rawData, willCopy)) { return ESP_ERR_NVS_CONTENT_DIFFERS; } + + // Calculate the crc32 of the actual ditem.rawData buffer. Do not pass accumulatedCRC32 in the first call. + // In the first call, calculateCrc32 will use its default. In the subsequent calls, accumulatedCRC32 is the crc32 of the previous buffer. + accumulatedCRC32 = Item::calculateCrc32(ditem.rawData, willCopy, (i == initial_index) ? nullptr : &accumulatedCRC32); + left -= willCopy; dst += willCopy; } - if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { - return ESP_ERR_NVS_NOT_FOUND; + // Check if the CRC32 calculated on the fly matches the variable length data CRC32 indicated in the metadata entry. + // If they are not equal, it means the data in the flash is corrupt, we will return ESP_ERR_NVS_CONTENT_DIFFERS. + if (accumulatedCRC32 != item.varLength.dataCrc32) { + return ESP_ERR_NVS_CONTENT_DIFFERS; } return ESP_OK; diff --git a/components/nvs_flash/src/nvs_page.hpp b/components/nvs_flash/src/nvs_page.hpp index 7adb75bef7..953682b157 100644 --- a/components/nvs_flash/src/nvs_page.hpp +++ b/components/nvs_flash/src/nvs_page.hpp @@ -88,6 +88,8 @@ public: esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY); + esp_err_t readVariableLengthItemData(const Item& item, const size_t index, void* data); + esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); esp_err_t cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); diff --git a/components/nvs_flash/src/nvs_storage.cpp b/components/nvs_flash/src/nvs_storage.cpp index 1cb35d8a38..374e176757 100644 --- a/components/nvs_flash/src/nvs_storage.cpp +++ b/components/nvs_flash/src/nvs_storage.cpp @@ -254,13 +254,16 @@ bool Storage::isValid() const return mState == StorageState::ACTIVE; } -esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart) +esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart, size_t* itemIndex) { for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { - size_t itemIndex = 0; - auto err = it->findItem(nsIndex, datatype, key, itemIndex, item, chunkIdx, chunkStart); + size_t tmpItemIndex = 0; + auto err = it->findItem(nsIndex, datatype, key, tmpItemIndex, item, chunkIdx, chunkStart); if(err == ESP_OK) { page = it; + if(itemIndex) { + *itemIndex = tmpItemIndex; + } return ESP_OK; } } @@ -373,7 +376,7 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo } // datatype BLOB is written as BLOB_INDEX and BLOB_DATA and is searched for previous value as BLOB_INDEX and/or BLOB -// datatype BLOB_IDX and BLOB_DATA are not supported as input parameters, the layer above should always use BLOB +// datatype BLOB_INDEX and BLOB_DATA are not supported as input parameters, the layer above should always use BLOB esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) { if(mState != StorageState::ACTIVE) { @@ -382,6 +385,8 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key // pointer to the page where the existing item was found Page* findPage = nullptr; + // index of the item in the page where the existing item was found + size_t itemIndex = 0; // page sequence number helping to detect whether the page with old value was relocated during the new write uint32_t findPageSeqNumber = UINT32_MAX; @@ -398,13 +403,13 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key if(datatype == ItemType::BLOB) { // Specific lookup if performed for input datatype BLOB. The searched datatype for exact match is BLOB_INDEX. // The BLOB_INDEX is used to store the metadata of the (originally typed) BLOB data in current V2 implementation. - err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); + err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); if(err == ESP_OK && findPage != nullptr) { matchedTypePageFound = true; } } else { // Handle all other data types than BLOB - err = findItem(nsIndex, datatype, key, findPage, item); + err = findItem(nsIndex, datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); if(err == ESP_OK && findPage != nullptr) { matchedTypePageFound = true; @@ -417,42 +422,19 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key } #ifndef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY - // If the item was not found, try to find it with any other datatype. Omit the BLOB_DATA and handle BLOB with respect - // to the V1 and V2 representation. - const ItemType dataTypes[] = { - ItemType::U8, - ItemType::U16, - ItemType::U32, - ItemType::U64, - ItemType::I8, - ItemType::I16, - ItemType::I32, - ItemType::I64, - ItemType::SZ, - ItemType::BLOB_IDX, - ItemType::BLOB - }; - + // If the item was not found under assumed datatype, try to find it as ANY. if(findPage == nullptr) { - // Iterate over potential data types to allow findItem() search using the hash list instead of bruteforce search. - for(const auto& currType : dataTypes) { - // Skip search for BLOB_IDX if the requested datatype is BLOB. BLOB_IDX was already searched above. - if(datatype == ItemType::BLOB && currType == ItemType::BLOB_IDX) continue; - // Skip search if requested datatype is not BLOB and the current datatype is equal to the requested one. This was already searched above. - if(datatype != ItemType::BLOB && currType == datatype) continue; - - err = findItem(nsIndex, currType, key, findPage, item); - if(err == ESP_OK && findPage != nullptr) { - // keep the sequence number of the page where the item was found for later check of relocation - err = findPage->getSeqNumber(findPageSeqNumber); - if(err != ESP_OK) { - return err; - } - // item was found with the same key and namespace index but data type is different - matchedTypePageFound = false; - break; + // We should not find BLOB_DATA chunks as CHUNK_ANY is never used by the BLOB_DATA. + err = findItem(nsIndex, nvs::ItemType::ANY, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); + if(err == ESP_OK && findPage != nullptr) { + // keep the sequence number of the page where the item was found for later check of relocation + err = findPage->getSeqNumber(findPageSeqNumber); + if(err != ESP_OK) { + return err; } + // item was found with the same key and namespace index but data type is different + matchedTypePageFound = false; } } #endif @@ -573,13 +555,14 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key if(wasRelocated) { // The page was relocated. We have to find the old value again from the beginning. // As the item was already found before relocation, we can use the exact datatype from item. - err = findItem(nsIndex, item.datatype, key, findPage, item); + err = findItem(nsIndex, item.datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); if(err != ESP_OK) { return err; } } + // Page containing the old value is now refreshed. We can erase the old value. - err = findPage->eraseItem(nsIndex, item.datatype, key); + err = findPage->eraseEntryAndSpan(itemIndex); if(err == ESP_ERR_FLASH_OP_FAIL) { return ESP_ERR_NVS_REMOVE_FAILED; } @@ -648,8 +631,10 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat { Item item; Page* findPage = nullptr; + size_t itemIndex = 0; - /* First read the blob index */ + + // First read the blob index auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); if(err != ESP_OK) { return err; @@ -661,24 +646,28 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat NVS_ASSERT_OR_RETURN(dataSize == item.blobIndex.dataSize, ESP_FAIL); - /* Now read corresponding chunks */ + // Now read related blob data chunks + // Remember the itemIndex as it is used to fast locate the entry in the page for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { - err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum, nvs::VerOffset::VER_ANY, &itemIndex); if(err != ESP_OK) { if(err == ESP_ERR_NVS_NOT_FOUND) { break; } return err; } + + // Check if the blob data chunk length indicated for actual item still fits into the total length of the buffer if(item.varLength.dataSize > dataSize - offset) { - /* The size of the entry in the index is inconsistent with the sum of the sizes of chunks */ err = ESP_ERR_NVS_INVALID_LENGTH; break; } - err = findPage->readItem(nsIndex, ItemType::BLOB_DATA, key, static_cast(data) + offset, item.varLength.dataSize, static_cast (chunkStart) + chunkNum); + + err = findPage->readVariableLengthItemData(item, itemIndex, static_cast(data) + offset); if(err != ESP_OK) { return err; } + NVS_ASSERT_OR_RETURN(static_cast (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL); offset += item.varLength.dataSize; @@ -698,8 +687,9 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void { Item item; Page* findPage = nullptr; + size_t itemIndex = 0; - /* First read the blob index */ + // First read the blob index auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); if(err != ESP_OK) { return err; @@ -714,15 +704,28 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void return ESP_ERR_NVS_CONTENT_DIFFERS; } - /* Now read corresponding chunks */ + // Now read corresponding chunks for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { - err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum, nvs::VerOffset::VER_ANY, &itemIndex); if(err != ESP_OK) { if(err == ESP_ERR_NVS_NOT_FOUND) { break; } return err; } + + if(item.varLength.dataSize > dataSize - offset) { + // The size of the entry in the index is bigger than the size of the remaining data to be compared + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + + // calculate crc32 of the incoming data window related to the BLOB_DATA chunk and compare it with the crc32 from the BLOB_DATA metadata entry + // Different crc32 indicates data mismatch. + // If crc32 matches, we have to compare the data in the chunk with the buffer data to exclude crc32 collision. + if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + err = findPage->cmpItem(nsIndex, ItemType::BLOB_DATA, key, static_cast(data) + offset, item.varLength.dataSize, static_cast (chunkStart) + chunkNum); if(err != ESP_OK) { return err; @@ -766,13 +769,18 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse } Item item; Page* findPage = nullptr; + size_t itemIndex = 0; + uint8_t chunkCount = 0; - auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart); + auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart, &itemIndex); if(err != ESP_OK) { return err; } + + chunkCount = item.blobIndex.chunkCount; + // Erase the index first and make children blobs orphan - err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart); + err = findPage->eraseEntryAndSpan(itemIndex); if(err != ESP_OK) { return err; } @@ -780,48 +788,76 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse // If caller requires delete of VER_ANY // We may face dirty NVS partition and version duplicates can be there // Make second attempt to delete index and ignore eventual not found - if(chunkStart == VerOffset::VER_ANY) - { - err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart); + if(chunkStart == VerOffset::VER_ANY) { + // Specific case called during initialisation of the storage + // We need to delete all chunks with the same key and namespace index + + // If there exists another BLOB_IDX with the same key and namespace index, delete it + // Ignore potential error if the item is not found + err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart, &itemIndex); if(err == ESP_OK) { - err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart); + err = findPage->eraseEntryAndSpan(itemIndex); if(err != ESP_OK) { return err; } } else if(err != ESP_ERR_NVS_NOT_FOUND) { return err; } - } - // setup limits for chunkIndex-es to be deleted - uint8_t minChunkIndex = (uint8_t) VerOffset::VER_0_OFFSET; - uint8_t maxChunkIndex = (uint8_t) VerOffset::VER_ANY; + // To delete all chunks, we will visit every page and delete all chunks regardless of chunkIndex + // This approach cannot use the hash list as the chunkIndex is not known. + for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { + // reset itemIndex to zero for each page to search from the beginning + itemIndex = 0; + do { + // (Re)Try to find the item at the position starting at the itemIndex + err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item); - if(chunkStart == VerOffset::VER_0_OFFSET) { - maxChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET; - } else if(chunkStart == VerOffset::VER_1_OFFSET) { - minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET; - } - - for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { - size_t itemIndex = 0; - do { - err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item); - if(err == ESP_ERR_NVS_NOT_FOUND) { - break; - } else if(err == ESP_OK) { - // check if item.chunkIndex is within the version range indicated by chunkStart, if so, delete it - if((item.chunkIndex >= minChunkIndex) && (item.chunkIndex < maxChunkIndex)) { + // If the item is not found, we can break the actual loop and continue with the next page + if(err == ESP_ERR_NVS_NOT_FOUND) { + break; + } else if(err == ESP_OK) { err = it->eraseEntryAndSpan(itemIndex); - } - // continue findItem until end of page - itemIndex += item.span; - } + // advance itemIndex to the next potential entry on the page + // findItem checks the consistency of the entry metadata so we can safely assume the span is non-zero + itemIndex += item.span; + } + if(err != ESP_OK) { + return err; + } + // Continue the loop until all items on the current page are found and erased + } while(err == ESP_OK && itemIndex < Page::ENTRY_COUNT); + } + + } else { + // Most common condition + // The caller has specified the chunk version, delete all chunks within the chunk index range indicated by the BLOB_IDX entry + // The loop will iterate the chunk index, page will be found and chunk index will be erased + // This approach uses the hash list to find the item on the page, so it is efficient. + uint8_t minChunkIndex = (uint8_t) VerOffset::VER_ANY; + uint8_t maxChunkIndex = (uint8_t) VerOffset::VER_ANY; + + if(chunkStart == VerOffset::VER_0_OFFSET) { + minChunkIndex = (uint8_t) VerOffset::VER_0_OFFSET; + maxChunkIndex = minChunkIndex + chunkCount; + } else if(chunkStart == VerOffset::VER_1_OFFSET) { + minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET; + maxChunkIndex = minChunkIndex + chunkCount; + } + + for(uint8_t chunkIndex = minChunkIndex; chunkIndex < maxChunkIndex; chunkIndex++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, chunkIndex, nvs::VerOffset::VER_ANY, &itemIndex); if(err != ESP_OK) { return err; } - } while(err == ESP_OK && itemIndex < Page::ENTRY_COUNT); + + // Erase the entry + err = findPage->eraseEntryAndSpan(itemIndex); + if(err != ESP_OK) { + return err; + } + } } return ESP_OK; @@ -833,22 +869,21 @@ esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key return ESP_ERR_NVS_NOT_INITIALIZED; } - if(datatype == ItemType::BLOB) { - return eraseMultiPageBlob(nsIndex, key); - } - Item item; Page* findPage = nullptr; - auto err = findItem(nsIndex, datatype, key, findPage, item); + esp_err_t err = ESP_OK; + size_t itemIndex = 0; + + err = findItem(nsIndex, datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); if(err != ESP_OK) { return err; } - - if(item.datatype == ItemType::BLOB_DATA || item.datatype == ItemType::BLOB_IDX) { - return eraseMultiPageBlob(nsIndex, key); + // If the item found is BLOB_IDX, the eraseMultiPageBlob is used to erase the whole multi-page blob. + if (item.datatype == ItemType::BLOB_IDX) { + return eraseMultiPageBlob(nsIndex, key, item.blobIndex.chunkStart); } - return findPage->eraseItem(nsIndex, datatype, key); + return findPage->eraseEntryAndSpan(itemIndex); } esp_err_t Storage::eraseNamespace(uint8_t nsIndex) @@ -900,19 +935,24 @@ esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const cha Item item; Page* findPage = nullptr; - auto err = findItem(nsIndex, datatype, key, findPage, item); - if(err != ESP_OK) { - if(datatype != ItemType::BLOB) { - return err; - } + esp_err_t err = ESP_OK; + + // If requested datatype is BLOB, first try to find the item with datatype BLOB_IDX - new format + // If not found, try to find the item with datatype BLOB - old format. + if(datatype == ItemType::BLOB) { err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); - if(err != ESP_OK) { + if(err == ESP_OK) { + dataSize = item.blobIndex.dataSize; + return err; + } else if(err != ESP_ERR_NVS_NOT_FOUND) { return err; } - dataSize = item.blobIndex.dataSize; - return ESP_OK; } + err = findItem(nsIndex, datatype, key, findPage, item); + if(err != ESP_OK) { + return err; + } dataSize = item.varLength.dataSize; return ESP_OK; } diff --git a/components/nvs_flash/src/nvs_storage.hpp b/components/nvs_flash/src/nvs_storage.hpp index 3ea642e773..1c63d1c8f0 100644 --- a/components/nvs_flash/src/nvs_storage.hpp +++ b/components/nvs_flash/src/nvs_storage.hpp @@ -153,7 +153,7 @@ protected: void fillEntryInfo(Item &item, nvs_entry_info_t &info); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY, size_t* itemIndex = NULL); protected: Partition *mPartition; diff --git a/components/nvs_flash/src/nvs_types.cpp b/components/nvs_flash/src/nvs_types.cpp index 24073fc306..81379c8591 100644 --- a/components/nvs_flash/src/nvs_types.cpp +++ b/components/nvs_flash/src/nvs_types.cpp @@ -33,9 +33,12 @@ uint32_t Item::calculateCrc32WithoutValue() const return result; } -uint32_t Item::calculateCrc32(const uint8_t* data, size_t size) +uint32_t Item::calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32) { uint32_t result = 0xffffffff; + if(initial_crc32) { + result = *initial_crc32; + } result = esp_rom_crc32_le(result, data, size); return result; } diff --git a/components/nvs_flash/src/nvs_types.hpp b/components/nvs_flash/src/nvs_types.hpp index 1761fa0354..33591a9112 100644 --- a/components/nvs_flash/src/nvs_types.hpp +++ b/components/nvs_flash/src/nvs_types.hpp @@ -91,7 +91,7 @@ public: uint32_t calculateCrc32() const; uint32_t calculateCrc32WithoutValue() const; - static uint32_t calculateCrc32(const uint8_t* data, size_t size); + static uint32_t calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32 = nullptr); void getKey(char* dst, size_t dstSize) {