refactor(nvs_flash): Host test migrated to esp_partition emulation for linux

components/esp_partition/CMakeLists.txt

@@ -9,10 +9,11 @@ if(${target} STREQUAL "linux")
     list(APPEND srcs "partition_linux.c")
     set(priv_reqs partition_table)
 
-    # Steal some include directories from bootloader_support and hal components:
+    # Steal some include directories from bootloader_support hal and spi_flash components:
     idf_component_get_property(hal_dir hal COMPONENT_DIR)
     idf_component_get_property(bootloader_support_dir bootloader_support COMPONENT_DIR)
-    list(APPEND include_dirs include ${hal_dir}/include ${bootloader_support_dir}/include)
+    idf_component_get_property(spi_flash_dir spi_flash COMPONENT_DIR)
+    list(APPEND include_dirs include ${hal_dir}/include ${bootloader_support_dir}/include ${spi_flash_dir}/include)
 else()
     list(APPEND srcs "partition_target.c")
 endif()

components/esp_partition/partition_linux.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -22,6 +22,7 @@
 #include "esp_flash_partitions.h"
 #include "esp_private/partition_linux.h"
 #include "esp_log.h"
+#include "spi_flash_mmap.h"
 
 static const char *TAG = "linux_spiflash";
 
@@ -390,7 +391,7 @@ esp_err_t esp_partition_write(const esp_partition_t *partition, size_t dst_offse
     // in case of power - off it decreases new_size to the number of bytes written
     // before power event occured
     if (!ESP_PARTITION_HOOK_WRITE(dst_addr, &new_size)) {
-        ret =  ESP_ERR_FLASH_BASE + 1;
+        ret =  ESP_ERR_FLASH_OP_FAIL;
     }
 
     for (size_t x = 0; x < new_size; x++) {
@@ -398,7 +399,7 @@ esp_err_t esp_partition_write(const esp_partition_t *partition, size_t dst_offse
         // Check if address to be written was erased first
         if((~((uint8_t *)dst_addr)[x] & ((uint8_t *)src)[x]) != 0) {
             ESP_LOGW(TAG, "invalid flash operation detected");
-            ret = ESP_ERR_FLASH_BASE + 1;
+            ret = ESP_ERR_FLASH_OP_FAIL;
             break;
         }
 
@@ -466,7 +467,7 @@ esp_err_t esp_partition_erase_range(const esp_partition_t *partition, size_t off
     esp_err_t ret = ESP_OK;
 
     if(!ESP_PARTITION_HOOK_ERASE(target_addr, &new_size)) {
-        ret =  ESP_ERR_FLASH_BASE + 1;
+        ret =  ESP_ERR_FLASH_OP_FAIL;
     }
 
     //set all bits to 1 (NOR FLASH default)
@@ -654,16 +655,13 @@ static bool esp_partition_hook_erase(const void *dstAddr, size_t *size)
             sector_count = s_esp_partition_emulated_power_off_counter;
             // disable power on cycles for further calls
             s_esp_partition_emulated_power_off_counter = SIZE_MAX;
+            // update number of bytes to be really erased before power-off event
+            *size = sector_count * ESP_PARTITION_EMULATED_SECTOR_SIZE;
             // final result value will be false
             ret_val = false;
         }
     }
 
-    if(!ret_val) {
-        // update number of bytes to be really erased before power-off event
-        *size = sector_count * ESP_PARTITION_EMULATED_SECTOR_SIZE;
-    }
-
     // update statistcs for all sectors until power down cycle
     for (size_t sector_index = first_sector_idx; sector_index < first_sector_idx + sector_count; sector_index++) {
         ++s_esp_partition_stat_erase_ops;

components/nvs_flash/host_test/nvs_host_test/main/CMakeLists.txt

@@ -12,3 +12,7 @@ idf_component_register(SRCS "main.cpp"
                             "../../../../../tools/catch"
                        WHOLE_ARCHIVE
                        REQUIRES nvs_flash)
+
+if(CMAKE_C_COMPILER_ID MATCHES "Clang")
+    target_compile_options(${COMPONENT_LIB} PRIVATE -std=gnu++20)
+endif()

components/nvs_flash/host_test/nvs_host_test/main/test_fixtures.hpp

@@ -1,11 +1,14 @@
 /*
- * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 #include "nvs_partition.hpp"
 #include "esp_private/partition_linux.h"
 #include "nvs.h"
+#include <random>
+#include <fcntl.h>
+#include <unistd.h>
 
 class PartitionEmulationFixture {
 public:
@@ -16,20 +19,132 @@ public:
     {
 
         if (esp_partition_file_mmap((const uint8_t **) &p_part_desc_addr_start) != ESP_OK) {
-            throw ("Failed to initialize esp_partition_file_mmap");
+            FAIL("Failed to initialize esp_partition_file_mmap");
         }
 
         esp_partition.address = start_sector * SPI_FLASH_SEC_SIZE;
-        esp_partition.size = sector_size * SPI_FLASH_SEC_SIZE;
+        esp_partition.size = (start_sector + sector_size) * SPI_FLASH_SEC_SIZE;
         esp_partition.erase_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
+        esp_partition.type = ESP_PARTITION_TYPE_DATA;
+        esp_partition.subtype = ESP_PARTITION_SUBTYPE_DATA_NVS;
         strncpy(esp_partition.label, partition_name, PART_NAME_MAX_SIZE);
         p_part = new (std::nothrow) nvs::NVSPartition(&esp_partition);
-        CHECK(p_part != nullptr);
+        REQUIRE(p_part != nullptr);
+    }
+
+    // initializes the partition and loads partition binary file into it
+    PartitionEmulationFixture(  uint32_t start_sector,
+                                uint32_t sector_size,
+                                const char *partition_name,
+                                const char *partition_binary) : PartitionEmulationFixture(start_sector, sector_size, partition_name)
+    {
+        int file_fd = -1;
+        off_t size = -1;
+        void *p_buff = nullptr;
+        char const *fail_msg = nullptr;
+
+        do {
+            // get file size
+            file_fd = open(partition_binary, O_RDONLY);
+            if (file_fd == -1) {
+                fail_msg = "Failed to open file with partition content";
+                break;
+            }
+            size = lseek(file_fd, 0L, SEEK_END);
+            if (size < 0) {
+                fail_msg = "falied to seek in file with partition content";
+                break;
+            }
+
+            // check if file fits into the partitiion
+            if (size > sector_size * SPI_FLASH_SEC_SIZE) {
+                fail_msg = "file with partition content doesn't fit into the partition";
+                break;
+            }
+
+            // allocate local buffer
+            p_buff = malloc((size_t) size);
+            if (p_buff == nullptr) {
+                fail_msg = "unable to allocate buffer for reading file with partition content";
+                break;
+            }
+
+            // laoad file into local buffer
+            int res = lseek(file_fd, 0L, SEEK_SET);
+            if (res < 0) {
+                fail_msg = "falied to seek in file with partition content";
+                break;
+            }
+            size = read(file_fd, p_buff, size);
+            if (size < 0) {
+                fail_msg = "cannot read file with partition content";
+                break;
+            }
+
+            // erase whole partition
+            if (ESP_OK != esp_partition_erase_range(&esp_partition, 0, sector_size * SPI_FLASH_SEC_SIZE)) {
+                fail_msg = "cannot erase partition prior to write partition binary from file";
+                break;
+            }
+
+            // write local buffer to the partition
+            if (ESP_OK != esp_partition_write_raw(&esp_partition, 0, p_buff, size)) {
+                fail_msg = "cannot write to the partition";
+                break;
+            }
+        } while (false);
+
+        // close file
+        if (file_fd >= 0) {
+            close(file_fd);
+        }
+
+        // deallocate buffer
+        if (p_buff != nullptr) {
+            free(p_buff);
+        }
+
+        if(fail_msg != nullptr) {
+            FAIL(fail_msg);
+        }
+    }
+
+    void randomize(uint32_t seed)
+    {
+        std::random_device rd;
+        std::mt19937 gen(rd());
+        gen.seed(seed);
+
+        esp_partition_file_mmap_ctrl_t *p_ctrl = esp_partition_get_file_mmap_ctrl_act();
+        REQUIRE(p_ctrl != nullptr);
+        std::generate_n(p_part_desc_addr_start, p_ctrl->flash_file_size, gen);
+    }
+
+    // absolute sectorNumber is used here
+    bool erase(size_t sectorNumber)
+    {
+        size_t offset = sectorNumber * SPI_FLASH_SEC_SIZE;
+
+        // check the upper bound
+        esp_partition_file_mmap_ctrl_t *p_ctrl = esp_partition_get_file_mmap_ctrl_act();
+        REQUIRE(p_ctrl != nullptr);
+        if (offset > p_ctrl->flash_file_size) {
+            return false;
+        }
+
+        // esp_partition_erase_range uses offset relative to the begining of partition
+        return (esp_partition_erase_range(&esp_partition,
+                                          offset - esp_partition.address,
+                                          SPI_FLASH_SEC_SIZE) == ESP_OK);
     }
 
     ~PartitionEmulationFixture()
     {
         delete p_part;
+
+        // ensure underlying mmaped file gets deleted after unmap.
+        esp_partition_file_mmap_ctrl_t *p_ctrl = esp_partition_get_file_mmap_ctrl_input();
+        p_ctrl->remove_dump = true;
         esp_partition_file_munmap();
     }
 
@@ -38,7 +153,49 @@ public:
         return p_part;
     }
 
+    const esp_partition_t *get_esp_partition() const
+    {
+        return &esp_partition;
+    }
+
     nvs::NVSPartition *p_part;
     esp_partition_t esp_partition;
     uint8_t *p_part_desc_addr_start;
 };
+
+// fixture with 2 partitions
+class PartitionEmulationFixture2 : public PartitionEmulationFixture {
+public:
+    PartitionEmulationFixture2( uint32_t start_sector1 = 0,
+                                uint32_t sector_size1 = 1,
+                                const char *partition_name1 = "nvs1",
+                                uint32_t start_sector2 = 1,
+                                uint32_t sector_size2 = 1,
+                                const char *partition_name2 = "nvs2"
+                              ) :
+        PartitionEmulationFixture(start_sector1, sector_size1, partition_name1), esp_partition2()
+    {
+        // for 2nd partition
+        esp_partition2.address = start_sector2 * SPI_FLASH_SEC_SIZE;
+        esp_partition2.size = (start_sector2 + sector_size2) * SPI_FLASH_SEC_SIZE;
+        esp_partition2.erase_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
+        esp_partition2.type = ESP_PARTITION_TYPE_DATA;
+        esp_partition2.subtype = ESP_PARTITION_SUBTYPE_DATA_NVS;
+        strncpy(esp_partition2.label, partition_name2, PART_NAME_MAX_SIZE);
+        p_part2 = new (std::nothrow) nvs::NVSPartition(&esp_partition2);
+        REQUIRE(p_part2 != nullptr);
+    }
+
+    ~PartitionEmulationFixture2()
+    {
+        delete p_part2;
+    }
+
+    nvs::NVSPartition *part2()
+    {
+        return p_part2;
+    }
+
+    nvs::NVSPartition *p_part2;
+    esp_partition_t esp_partition2;
+};

components/nvs_flash/host_test/nvs_host_test/main/test_partition_manager.cpp

@@ -68,3 +68,40 @@ TEST_CASE("Partition manager invalidates handle on partition de-init", "[partiti
 
     delete handle;
 }
+
+TEST_CASE("Partition manager initializes multiple partitions", "[partition_mgr]")
+{
+    const uint32_t NVS_FLASH_SECTOR_BEGIN1 = 0;
+    const uint32_t NVS_FLASH_SECTOR_SIZE1 = 3;
+    const char* NVS_FLASH_PARTITION1 = "test1";
+    const uint32_t NVS_FLASH_SECTOR_BEGIN2 = 3;
+    const uint32_t NVS_FLASH_SECTOR_SIZE2 = 3;
+    const char* NVS_FLASH_PARTITION2 = "test2";
+
+    PartitionEmulationFixture2 f(NVS_FLASH_SECTOR_BEGIN1,
+        NVS_FLASH_SECTOR_SIZE1,
+        NVS_FLASH_PARTITION1,
+        NVS_FLASH_SECTOR_BEGIN2,
+        NVS_FLASH_SECTOR_SIZE2,
+        NVS_FLASH_PARTITION2
+        );
+
+    REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(f.part(),
+        NVS_FLASH_SECTOR_BEGIN1,
+        NVS_FLASH_SECTOR_SIZE1)
+            == ESP_OK);
+
+    REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(f.part2(),
+        NVS_FLASH_SECTOR_BEGIN2,
+        NVS_FLASH_SECTOR_SIZE2)
+            == ESP_OK);
+
+    nvs::Storage *storage1 = nvs::NVSPartitionManager::get_instance()->lookup_storage_from_name(NVS_FLASH_PARTITION1);
+    REQUIRE(storage1 != nullptr);
+    nvs::Storage *storage2 = nvs::NVSPartitionManager::get_instance()->lookup_storage_from_name(NVS_FLASH_PARTITION2);
+    REQUIRE(storage2 != nullptr);
+
+    CHECK(storage1 != storage2);
+    REQUIRE(nvs::NVSPartitionManager::get_instance()->deinit_partition(NVS_FLASH_PARTITION1) == ESP_OK);
+    REQUIRE(nvs::NVSPartitionManager::get_instance()->deinit_partition(NVS_FLASH_PARTITION2) == ESP_OK);
+}

components/nvs_flash/host_test/nvs_host_test/sdkconfig.defaults

@@ -1,3 +1,8 @@
 CONFIG_IDF_TARGET="linux"
 CONFIG_COMPILER_CXX_EXCEPTIONS=y
 CONFIG_UNITY_ENABLE_IDF_TEST_RUNNER=n
+CONFIG_ESP_PARTITION_ENABLE_STATS=y
+CONFIG_PARTITION_TABLE_SINGLE_APP=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
+CONFIG_PARTITION_TABLE_FILENAME="partitions_singleapp.csv"
+CONFIG_PARTITION_TABLE_OFFSET=0x8000

components/nvs_flash/test_nvs_host/test_partition_manager.cpp

@@ -13,6 +13,7 @@
 #include "nvs_test_api.h"
 #include "test_fixtures.hpp"
 
+/*
 TEST_CASE("Partition manager initializes multiple partitions", "[partition_mgr]")
 {
     const uint32_t NVS_FLASH_SECTOR = 6;
@@ -35,3 +36,4 @@ TEST_CASE("Partition manager initializes multiple partitions", "[partition_mgr]"
     REQUIRE(nvs::NVSPartitionManager::get_instance()->deinit_partition(part_0.get_partition_name()) == ESP_OK);
     REQUIRE(nvs::NVSPartitionManager::get_instance()->deinit_partition(part_1.get_partition_name()) == ESP_OK);
 }
+*/