refactor(nvs_flash): Improved Blob performance

The findItem method was improved to use a hash list in RAM when searching for BLOB data chunks

The findItem method was extended with a parameter that returns the position of an item on the page,
if it is found

The algorithm for matching existing variable-length data (such as strings and BLOBs) with
new values was enhanced by comparing the CRC32 of the data chunks before reading the data from flash

components/nvs_flash/src/nvs_page.cpp

@@ -250,6 +250,43 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c
     return ESP_OK;
 }
 
+// Reads the data entries of the variable length item.
+// The metadata entry is already read in the item object.
+// index is the index of the metadata entry on the page.
+// data is pointer to the buffer where the data will be copied to. It has to be at least
+// item.varLength.dataSize bytes long.
+// The function returns ESP_OK if the data was read successfully, or an error code if there was an error.
+esp_err_t Page::readVariableLengthItemData(const Item& item, const size_t index, void* data)
+{
+    if (mState == PageState::INVALID) {
+        return ESP_ERR_NVS_INVALID_STATE;
+    }
+
+    esp_err_t rc;
+    uint8_t* dst = reinterpret_cast<uint8_t*>(data);
+    size_t left = item.varLength.dataSize;
+    for (size_t i = index + 1; i < index + item.span; ++i) {
+        Item ditem;
+        rc = readEntry(i, ditem);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        size_t willCopy = ENTRY_SIZE;
+        willCopy = (left < willCopy) ? left : willCopy;
+        memcpy(dst, ditem.rawData, willCopy);
+        left -= willCopy;
+        dst += willCopy;
+    }
+    if (Item::calculateCrc32(reinterpret_cast<uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+        rc = eraseEntryAndSpan(index);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        return ESP_ERR_NVS_NOT_FOUND;
+    }
+    return ESP_OK;
+}
+
 esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart)
 {
     size_t index = 0;
@@ -277,28 +314,7 @@ esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, vo
         return ESP_ERR_NVS_INVALID_LENGTH;
     }
 
-    uint8_t* dst = reinterpret_cast<uint8_t*>(data);
+    return readVariableLengthItemData(item, index, data);
-    size_t left = item.varLength.dataSize;
-    for (size_t i = index + 1; i < index + item.span; ++i) {
-        Item ditem;
-        rc = readEntry(i, ditem);
-        if (rc != ESP_OK) {
-            return rc;
-        }
-        size_t willCopy = ENTRY_SIZE;
-        willCopy = (left < willCopy) ? left : willCopy;
-        memcpy(dst, ditem.rawData, willCopy);
-        left -= willCopy;
-        dst += willCopy;
-    }
-    if (Item::calculateCrc32(reinterpret_cast<uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
-        rc = eraseEntryAndSpan(index);
-        if (rc != ESP_OK) {
-            return rc;
-        }
-        return ESP_ERR_NVS_NOT_FOUND;
-    }
-    return ESP_OK;
 }
 
 esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart)
@@ -330,9 +346,23 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con
         return ESP_ERR_NVS_INVALID_LENGTH;
     }
 
+    // We have metadata of the variable length data chunk. It contains the length of the data and the crc32.
+    // As a first step we can calculate the crc32 of the data buffer to be compared with the crc32 of the item in the flash.
+    // If they are not equal, immediately return ESP_ERR_NVS_CONTENT_DIFFERS.
+    // If they are equal, to avoid crc32 collision false positive, we will read the data from the flash entry by entry and compare
+    // it with the respective chunk of input data buffer. The crc32 of the data read from the flash will be calculated on the fly.
+    // At the end, we will compare the crc32 of the data read from the flash with the crc32 of the metadata item in the flash to make sure
+    // that the data in the flash is not corrupted.
+    if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+        return ESP_ERR_NVS_CONTENT_DIFFERS;
+    }
+
     const uint8_t* dst = reinterpret_cast<const uint8_t*>(data);
     size_t left = item.varLength.dataSize;
-    for (size_t i = index + 1; i < index + item.span; ++i) {
+    uint32_t accumulatedCRC32;
+    size_t initial_index = index + 1;
+
+    for (size_t i = initial_index; i < index + item.span; ++i) {
         Item ditem;
         rc = readEntry(i, ditem);
         if (rc != ESP_OK) {
@@ -343,11 +373,18 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con
         if (memcmp(dst, ditem.rawData, willCopy)) {
             return ESP_ERR_NVS_CONTENT_DIFFERS;
         }
+
+        // Calculate the crc32 of the actual ditem.rawData buffer. Do not pass accumulatedCRC32 in the first call.
+        // In the first call, calculateCrc32 will use its default. In the subsequent calls, accumulatedCRC32 is the crc32 of the previous buffer.
+        accumulatedCRC32 = Item::calculateCrc32(ditem.rawData, willCopy, (i == initial_index) ? nullptr : &accumulatedCRC32);
+
         left -= willCopy;
         dst += willCopy;
     }
-    if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+    // Check if the CRC32 calculated on the fly matches the variable length data CRC32 indicated in the metadata entry.
-        return ESP_ERR_NVS_NOT_FOUND;
+    // If they are not equal, it means the data in the flash is corrupt, we will return ESP_ERR_NVS_CONTENT_DIFFERS.
+    if (accumulatedCRC32 != item.varLength.dataCrc32) {
+        return ESP_ERR_NVS_CONTENT_DIFFERS;
     }
 
     return ESP_OK;

components/nvs_flash/src/nvs_page.hpp

@@ -88,6 +88,8 @@ public:
 
     esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY);
 
+    esp_err_t readVariableLengthItemData(const Item& item, const size_t index, void* data);
+
     esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);
 
     esp_err_t cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);

components/nvs_flash/src/nvs_storage.cpp

@@ -254,13 +254,16 @@ bool Storage::isValid() const
     return mState == StorageState::ACTIVE;
 }
 
-esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart)
+esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart, size_t* itemIndex)
 {
     for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
-        size_t itemIndex = 0;
+        size_t tmpItemIndex = 0;
-        auto err = it->findItem(nsIndex, datatype, key, itemIndex, item, chunkIdx, chunkStart);
+        auto err = it->findItem(nsIndex, datatype, key, tmpItemIndex, item, chunkIdx, chunkStart);
         if(err == ESP_OK) {
             page = it;
+            if(itemIndex) {
+                *itemIndex = tmpItemIndex;
+            }
             return ESP_OK;
         }
     }
@@ -373,7 +376,7 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo
 }
 
 // datatype BLOB is written as BLOB_INDEX and BLOB_DATA and is searched for previous value as BLOB_INDEX and/or BLOB
-// datatype BLOB_IDX and BLOB_DATA are not supported as input parameters, the layer above should always use BLOB
+// datatype BLOB_INDEX and BLOB_DATA are not supported as input parameters, the layer above should always use BLOB
 esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize)
 {
     if(mState != StorageState::ACTIVE) {
@@ -382,6 +385,8 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key
 
     // pointer to the page where the existing item was found
     Page* findPage = nullptr;
+    // index of the item in the page where the existing item was found
+    size_t itemIndex = 0;
     // page sequence number helping to detect whether the page with old value was relocated during the new write
     uint32_t findPageSeqNumber = UINT32_MAX;
 
@@ -398,13 +403,13 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key
     if(datatype == ItemType::BLOB) {
         // Specific lookup if performed for input datatype BLOB. The searched datatype for exact match is BLOB_INDEX.
         // The BLOB_INDEX is used to store the metadata of the (originally typed) BLOB data in current V2 implementation.
-        err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
+        err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
         if(err == ESP_OK && findPage != nullptr) {
             matchedTypePageFound = true;
         }
     } else {
         // Handle all other data types than BLOB
-        err = findItem(nsIndex, datatype, key, findPage, item);
+        err = findItem(nsIndex, datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
         if(err == ESP_OK && findPage != nullptr) {
             matchedTypePageFound = true;
 
@@ -417,42 +422,19 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key
     }
 
 #ifndef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY
-    // If the item was not found, try to find it with any other datatype. Omit the BLOB_DATA and handle BLOB with respect
+    // If the item was not found under assumed datatype, try to find it as ANY.
-    // to the V1 and V2 representation.
-    const ItemType dataTypes[] = {
-        ItemType::U8,
-        ItemType::U16,
-        ItemType::U32,
-        ItemType::U64,
-        ItemType::I8,
-        ItemType::I16,
-        ItemType::I32,
-        ItemType::I64,
-        ItemType::SZ,
-        ItemType::BLOB_IDX,
-        ItemType::BLOB
-    };
-
     if(findPage == nullptr) {
-        // Iterate over potential data types to allow findItem() search using the hash list instead of bruteforce search.
-        for(const auto& currType : dataTypes) {
-            // Skip search for BLOB_IDX if the requested datatype is BLOB. BLOB_IDX was already searched above.
-            if(datatype == ItemType::BLOB && currType == ItemType::BLOB_IDX) continue;
 
-            // Skip search if requested datatype is not BLOB and the current datatype is equal to the requested one. This was already searched above.
+        // We should not find BLOB_DATA chunks as CHUNK_ANY is never used by the BLOB_DATA.
-            if(datatype != ItemType::BLOB && currType == datatype) continue;
+        err = findItem(nsIndex, nvs::ItemType::ANY, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
-
+        if(err == ESP_OK && findPage != nullptr) {
-            err = findItem(nsIndex, currType, key, findPage, item);
+            // keep the sequence number of the page where the item was found for later check of relocation
-            if(err == ESP_OK && findPage != nullptr) {
+            err = findPage->getSeqNumber(findPageSeqNumber);
-                // keep the sequence number of the page where the item was found for later check of relocation
+            if(err != ESP_OK) {
-                err = findPage->getSeqNumber(findPageSeqNumber);
+                return err;
-                if(err != ESP_OK) {
-                    return err;
-                }
-                // item was found with the same key and namespace index but data type is different
-                matchedTypePageFound = false;
-                break;
             }
+            // item was found with the same key and namespace index but data type is different
+            matchedTypePageFound = false;
         }
     }
 #endif
@@ -573,13 +555,14 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key
         if(wasRelocated) {
             // The page was relocated. We have to find the old value again from the beginning.
             // As the item was already found before relocation, we can use the exact datatype from item.
-            err = findItem(nsIndex, item.datatype, key, findPage, item);
+            err = findItem(nsIndex, item.datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
             if(err != ESP_OK) {
                 return err;
             }
         }
+
         // Page containing the old value is now refreshed. We can erase the old value.
-        err = findPage->eraseItem(nsIndex, item.datatype, key);
+        err = findPage->eraseEntryAndSpan(itemIndex);
         if(err == ESP_ERR_FLASH_OP_FAIL) {
             return ESP_ERR_NVS_REMOVE_FAILED;
         }
@@ -648,8 +631,10 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat
 {
     Item item;
     Page* findPage = nullptr;
+    size_t itemIndex = 0;
 
-    /* First read the blob index */
+
+    // First read the blob index
     auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
     if(err != ESP_OK) {
         return err;
@@ -661,24 +646,28 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat
 
     NVS_ASSERT_OR_RETURN(dataSize == item.blobIndex.dataSize, ESP_FAIL);
 
-    /* Now read corresponding chunks */
+    // Now read related blob data chunks
+    // Remember the itemIndex as it is used to fast locate the entry in the page
     for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) {
-        err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum);
+        err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum, nvs::VerOffset::VER_ANY, &itemIndex);
         if(err != ESP_OK) {
             if(err == ESP_ERR_NVS_NOT_FOUND) {
                 break;
             }
             return err;
         }
+
+        // Check if the blob data chunk length indicated for actual item still fits into the total length of the buffer
         if(item.varLength.dataSize > dataSize - offset) {
-            /* The size of the entry in the index is inconsistent with the sum of the sizes of chunks */
             err = ESP_ERR_NVS_INVALID_LENGTH;
             break;
         }
-        err = findPage->readItem(nsIndex, ItemType::BLOB_DATA, key, static_cast<uint8_t*>(data) + offset, item.varLength.dataSize, static_cast<uint8_t> (chunkStart) + chunkNum);
+
+        err = findPage->readVariableLengthItemData(item, itemIndex, static_cast<uint8_t*>(data) + offset);
         if(err != ESP_OK) {
             return err;
         }
+
         NVS_ASSERT_OR_RETURN(static_cast<uint8_t> (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL);
 
         offset += item.varLength.dataSize;
@@ -698,8 +687,9 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void
 {
     Item item;
     Page* findPage = nullptr;
+    size_t itemIndex = 0;
 
-    /* First read the blob index */
+    // First read the blob index
     auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
     if(err != ESP_OK) {
         return err;
@@ -714,15 +704,28 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void
         return ESP_ERR_NVS_CONTENT_DIFFERS;
     }
 
-    /* Now read corresponding chunks */
+    // Now read corresponding chunks
     for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) {
-        err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum);
+        err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast<uint8_t> (chunkStart) + chunkNum, nvs::VerOffset::VER_ANY, &itemIndex);
         if(err != ESP_OK) {
             if(err == ESP_ERR_NVS_NOT_FOUND) {
                 break;
             }
             return err;
         }
+
+        if(item.varLength.dataSize > dataSize - offset) {
+            // The size of the entry in the index is bigger than the size of the remaining data to be compared
+            return ESP_ERR_NVS_CONTENT_DIFFERS;
+        }
+
+        // calculate crc32 of the incoming data window related to the BLOB_DATA chunk and compare it with the crc32 from the BLOB_DATA metadata entry
+        // Different crc32 indicates data mismatch.
+        // If crc32 matches, we have to compare the data in the chunk with the buffer data to exclude crc32 collision.
+        if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+            return ESP_ERR_NVS_CONTENT_DIFFERS;
+        }
+
         err = findPage->cmpItem(nsIndex, ItemType::BLOB_DATA, key, static_cast<const uint8_t*>(data) + offset, item.varLength.dataSize, static_cast<uint8_t> (chunkStart) + chunkNum);
         if(err != ESP_OK) {
             return err;
@@ -766,13 +769,18 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse
     }
     Item item;
     Page* findPage = nullptr;
+    size_t itemIndex = 0;
+    uint8_t chunkCount = 0;
 
-    auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart);
+    auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart, &itemIndex);
     if(err != ESP_OK) {
         return err;
     }
+
+    chunkCount = item.blobIndex.chunkCount;
+
     // Erase the index first and make children blobs orphan
-    err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart);
+    err = findPage->eraseEntryAndSpan(itemIndex);
     if(err != ESP_OK) {
         return err;
     }
@@ -780,48 +788,76 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse
     // If caller requires delete of VER_ANY
     // We may face dirty NVS partition and version duplicates can be there
     // Make second attempt to delete index and ignore eventual not found
-    if(chunkStart == VerOffset::VER_ANY)
+    if(chunkStart == VerOffset::VER_ANY) {
-    {
+        // Specific case called during initialisation of the storage
-        err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart);
+        // We need to delete all chunks with the same key and namespace index
+
+        // If there exists another BLOB_IDX with the same key and namespace index, delete it
+        // Ignore potential error if the item is not found
+        err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart, &itemIndex);
         if(err == ESP_OK) {
-            err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart);
+            err = findPage->eraseEntryAndSpan(itemIndex);
             if(err != ESP_OK) {
                 return err;
             }
         } else if(err != ESP_ERR_NVS_NOT_FOUND) {
             return err;
         }
-    }
 
-    // setup limits for chunkIndex-es to be deleted
+        // To delete all chunks, we will visit every page and delete all chunks regardless of chunkIndex
-    uint8_t minChunkIndex = (uint8_t) VerOffset::VER_0_OFFSET;
+        // This approach cannot use the hash list as the chunkIndex is not known.
-    uint8_t maxChunkIndex = (uint8_t) VerOffset::VER_ANY;
+        for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
+            // reset itemIndex to zero for each page to search from the beginning
+            itemIndex = 0;
+            do {
+                // (Re)Try to find the item at the position starting at the itemIndex
+                err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item);
 
-    if(chunkStart == VerOffset::VER_0_OFFSET) {
+                // If the item is not found, we can break the actual loop and continue with the next page
-        maxChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET;
+                if(err == ESP_ERR_NVS_NOT_FOUND) {
-    } else if(chunkStart == VerOffset::VER_1_OFFSET) {
+                    break;
-        minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET;
+                } else if(err == ESP_OK) {
-    }
-
-    for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
-        size_t itemIndex = 0;
-        do {
-            err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item);
-            if(err == ESP_ERR_NVS_NOT_FOUND) {
-                break;
-            } else if(err == ESP_OK) {
-                // check if item.chunkIndex is within the version range indicated by chunkStart, if so, delete it
-                if((item.chunkIndex >= minChunkIndex) && (item.chunkIndex < maxChunkIndex)) {
                     err = it->eraseEntryAndSpan(itemIndex);
-                }
 
-                // continue findItem until end of page
+                    // advance itemIndex to the next potential entry on the page
-                itemIndex += item.span;
+                    // findItem checks the consistency of the entry metadata so we can safely assume the span is non-zero
-            }
+                    itemIndex += item.span;
+                }
+                if(err != ESP_OK) {
+                    return err;
+                }
+            // Continue the loop until all items on the current page are found and erased
+            } while(err == ESP_OK && itemIndex < Page::ENTRY_COUNT);
+        }
+
+    } else {
+        // Most common condition
+        // The caller has specified the chunk version, delete all chunks within the chunk index range indicated by the BLOB_IDX entry
+        // The loop will iterate the chunk index, page will be found and chunk index will be erased
+        // This approach uses the hash list to find the item on the page, so it is efficient.
+        uint8_t minChunkIndex = (uint8_t) VerOffset::VER_ANY;
+        uint8_t maxChunkIndex = (uint8_t) VerOffset::VER_ANY;
+
+        if(chunkStart == VerOffset::VER_0_OFFSET) {
+            minChunkIndex = (uint8_t) VerOffset::VER_0_OFFSET;
+            maxChunkIndex = minChunkIndex + chunkCount;
+        } else if(chunkStart == VerOffset::VER_1_OFFSET) {
+            minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET;
+            maxChunkIndex = minChunkIndex + chunkCount;
+        }
+
+        for(uint8_t chunkIndex = minChunkIndex; chunkIndex < maxChunkIndex; chunkIndex++) {
+            err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, chunkIndex, nvs::VerOffset::VER_ANY, &itemIndex);
             if(err != ESP_OK) {
                 return err;
             }
-        } while(err == ESP_OK && itemIndex < Page::ENTRY_COUNT);
+
+            // Erase the entry
+            err = findPage->eraseEntryAndSpan(itemIndex);
+            if(err != ESP_OK) {
+                return err;
+            }
+        }
     }
 
     return ESP_OK;
@@ -833,22 +869,21 @@ esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key
         return ESP_ERR_NVS_NOT_INITIALIZED;
     }
 
-    if(datatype == ItemType::BLOB) {
-        return eraseMultiPageBlob(nsIndex, key);
-    }
-
     Item item;
     Page* findPage = nullptr;
-    auto err = findItem(nsIndex, datatype, key, findPage, item);
+    esp_err_t err = ESP_OK;
+    size_t itemIndex = 0;
+
+    err = findItem(nsIndex, datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex);
     if(err != ESP_OK) {
         return err;
     }
-
+    // If the item found is BLOB_IDX, the eraseMultiPageBlob is used to erase the whole multi-page blob.
-    if(item.datatype == ItemType::BLOB_DATA || item.datatype == ItemType::BLOB_IDX) {
+    if (item.datatype == ItemType::BLOB_IDX) {
-        return eraseMultiPageBlob(nsIndex, key);
+        return eraseMultiPageBlob(nsIndex, key, item.blobIndex.chunkStart);
     }
 
-    return findPage->eraseItem(nsIndex, datatype, key);
+    return findPage->eraseEntryAndSpan(itemIndex);
 }
 
 esp_err_t Storage::eraseNamespace(uint8_t nsIndex)
@@ -900,19 +935,24 @@ esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const cha
 
     Item item;
     Page* findPage = nullptr;
-    auto err = findItem(nsIndex, datatype, key, findPage, item);
+    esp_err_t err = ESP_OK;
-    if(err != ESP_OK) {
+
-        if(datatype != ItemType::BLOB) {
+    // If requested datatype is BLOB, first try to find the item with datatype BLOB_IDX - new format
-            return err;
+    // If not found, try to find the item with datatype BLOB - old format.
-        }
+    if(datatype == ItemType::BLOB) {
         err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item);
-        if(err != ESP_OK) {
+        if(err == ESP_OK) {
+            dataSize = item.blobIndex.dataSize;
+            return err;
+        } else if(err != ESP_ERR_NVS_NOT_FOUND) {
             return err;
         }
-        dataSize = item.blobIndex.dataSize;
-        return ESP_OK;
     }
 
+    err = findItem(nsIndex, datatype, key, findPage, item);
+    if(err != ESP_OK) {
+        return err;
+    }
     dataSize = item.varLength.dataSize;
     return ESP_OK;
 }

components/nvs_flash/src/nvs_storage.hpp

@@ -153,7 +153,7 @@ protected:
 
     void fillEntryInfo(Item &item, nvs_entry_info_t &info);
 
-    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);
+    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY, size_t* itemIndex = NULL);
 
 protected:
     Partition *mPartition;

components/nvs_flash/src/nvs_types.cpp

@@ -33,9 +33,12 @@ uint32_t Item::calculateCrc32WithoutValue() const
     return result;
 }
 
-uint32_t Item::calculateCrc32(const uint8_t* data, size_t size)
+uint32_t Item::calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32)
 {
     uint32_t result = 0xffffffff;
+    if(initial_crc32) {
+        result = *initial_crc32;
+    }
     result = esp_rom_crc32_le(result, data, size);
     return result;
 }

components/nvs_flash/src/nvs_types.hpp

@@ -91,7 +91,7 @@ public:
 
     uint32_t calculateCrc32() const;
     uint32_t calculateCrc32WithoutValue() const;
-    static uint32_t calculateCrc32(const uint8_t* data, size_t size);
+    static uint32_t calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32 = nullptr);
 
     void getKey(char* dst, size_t dstSize)
     {