Merge branch 'test/add_cache2phys_xip_tests_v5.3' into 'release/v5.3'

fix(mmap): fixed spi_flash_phys2cache, spi_flash_cache2phys return addr in PSRAM issue (v5.3)

See merge request espressif/esp-idf!33628

components/app_update/test_apps/.build-test-rules.yml

@@ -1,7 +0,0 @@
-# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
-
-components/app_update/test_apps:
-  disable:
-    - if: IDF_TARGET in ["esp32c6", "esp32h2", "esp32c5"]
-      temporary: true
-      reason: target esp32c6, esp32h2 esp32c5 is not supported yet # TODO: [ESP32C5] IDF-8638

components/app_update/test_apps/pytest_app_update_ut.py

@@ -1,32 +0,0 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Unlicense OR CC0-1.0
-import re
-
-import pytest
-from pytest_embedded import Dut
-
-DEFAULT_TIMEOUT = 20
-TEST_SUBMENU_PATTERN_PYTEST = re.compile(rb'\s+\((\d+)\)\s+"([^"]+)"\r?\n')
-
-
-def run_multiple_stages(dut: Dut, test_case_num: int, stages: int) -> None:
-    for stage in range(1, stages + 1):
-        dut.write(str(test_case_num))
-        dut.expect(TEST_SUBMENU_PATTERN_PYTEST, timeout=DEFAULT_TIMEOUT)
-        dut.write(str(stage))
-        if stage != stages:
-            dut.expect_exact('Press ENTER to see the list of tests.')
-
-
-@pytest.mark.supported_targets
-@pytest.mark.temp_skip_ci(targets=['esp32c6', 'esp32h2'], reason='c6/h2 support TBD')
-@pytest.mark.generic
-def test_app_update(dut: Dut) -> None:
-    extra_data = dut.parse_test_menu()
-    for test_case in extra_data:
-        if test_case.type != 'multi_stage':
-            dut.write(str(test_case.index))
-        else:
-            run_multiple_stages(dut, test_case.index, len(test_case.subcases))
-        dut.expect_unity_test_output(timeout=90)
-        dut.expect_exact("Enter next test, or 'enter' to see menu")

components/app_update/test_apps/test_app_update/.build-test-rules.yml

@@ -0,0 +1,12 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/app_update/test_apps:
+  enable:
+    - if: CONFIG_NAME == "defaults" and IDF_TARGET != "linux"
+    - if: CONFIG_NAME == "xip_psram" and IDF_TARGET in ["esp32s2", "esp32s3", "esp32p4"]
+    # S2 doesn't have ROM for flash
+    - if: CONFIG_NAME == "xip_psram_with_rom_impl" and IDF_TARGET in ["esp32s3", "esp32p4"]
+  disable:
+    - if: IDF_TARGET in ["esp32c6", "esp32h2", "esp32c5", "esp32c61"]
+      temporary: true
+      reason: target esp32c6, esp32h2 esp32c5 is not supported yet # TODO: [ESP32C5] IDF-8638

components/app_update/test_apps/test_app_update/main/CMakeLists.txt

@@ -1,4 +1,4 @@
 idf_component_register(SRC_DIRS "."
                        PRIV_INCLUDE_DIRS "."
-                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver spi_flash
+                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver spi_flash esp_psram
                       WHOLE_ARCHIVE)

components/app_update/test_apps/test_app_update/main/test_ota_ops.c

@@ -6,6 +6,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include "esp_log.h"
 #include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
 #include <freertos/semphr.h>
@@ -113,3 +114,11 @@ TEST_CASE("esp_ota_get_partition_description", "[ota]")
     };
     TEST_ESP_ERR(ESP_ERR_NOT_FOUND, bootloader_common_get_partition_description(&not_app_pos, &app_desc1));
 }
+
+TEST_CASE("esp_ota_get_running_partition points to correct address", "[spi_flash]")
+{
+    const esp_partition_t *factory = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_ANY, "factory");
+    const esp_partition_t* part = esp_ota_get_running_partition();
+    ESP_LOGI("running bin", "0x%p", (void*)part->address);
+    TEST_ASSERT_EQUAL_HEX32(factory->address, part->address);
+}

components/app_update/test_apps/test_app_update/main/test_switch_ota.c

@@ -842,7 +842,7 @@ static void test_flow6(void)
 TEST_CASE_MULTIPLE_STAGES("Switching between factory, OTA0 using esp_ota_write_with_offset", "[app_update][timeout=90][reset=DEEPSLEEP_RESET, DEEPSLEEP_RESET]", start_test, test_flow6, test_flow6);
 
 //IDF-5145
-TEST_CASE("Test bootloader_common_get_sha256_of_partition returns ESP_ERR_IMAGE_INVALID when image is ivalid", "[partitions]")
+TEST_CASE("Test bootloader_common_get_sha256_of_partition returns ESP_ERR_IMAGE_INVALID when image is invalid", "[partitions]")
 {
     const esp_partition_t *cur_app = esp_ota_get_running_partition();
     ESP_LOGI(TAG, "copy current app to next part");

components/app_update/test_apps/test_app_update/pytest_app_update_ut.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import re
+
+import pytest
+from pytest_embedded import Dut
+
+DEFAULT_TIMEOUT = 20
+TEST_SUBMENU_PATTERN_PYTEST = re.compile(rb'\s+\((\d+)\)\s+"([^"]+)"\r?\n')
+
+
+@pytest.mark.supported_targets
+@pytest.mark.temp_skip_ci(targets=['esp32c6', 'esp32h2'], reason='c6/h2 support TBD')
+@pytest.mark.generic
+@pytest.mark.parametrize(
+    'config',
+    [
+        'defaults',
+    ],
+    indirect=True,
+)
+def test_app_update(dut: Dut) -> None:
+    dut.run_all_single_board_cases(timeout=90)
+
+
+@pytest.mark.supported_targets
+@pytest.mark.temp_skip_ci(targets=['esp32c6', 'esp32h2'], reason='c6/h2 support TBD')
+@pytest.mark.generic
+@pytest.mark.parametrize(
+    'config',
+    [
+        'xip_psram',
+    ],
+    indirect=True,
+)
+def test_app_update_xip_psram(dut: Dut) -> None:
+    dut.run_all_single_board_cases(timeout=90)
+
+
+@pytest.mark.supported_targets
+@pytest.mark.temp_skip_ci(targets=['esp32c6', 'esp32h2'], reason='c6/h2 support TBD')
+@pytest.mark.generic
+@pytest.mark.parametrize(
+    'config',
+    [
+        'xip_psram_with_rom_impl',
+    ],
+    indirect=True,
+)
+def test_app_update_xip_psram_rom_impl(dut: Dut) -> None:
+    dut.run_all_single_board_cases(timeout=90)

components/app_update/test_apps/test_app_update/sdkconfig.ci.defaults

@@ -0,0 +1,2 @@
+# don't delete.
+# used for CI to compile a default config when 'sdkconfig.ci.xxxx' is exist

components/app_update/test_apps/test_app_update/sdkconfig.ci.xip_psram

@@ -0,0 +1,2 @@
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_XIP_FROM_PSRAM=y

components/app_update/test_apps/test_app_update/sdkconfig.ci.xip_psram_with_rom_impl

@@ -0,0 +1,3 @@
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_XIP_FROM_PSRAM=y
+CONFIG_SPI_FLASH_ROM_IMPL=y

components/app_update/test_apps/test_app_update/sdkconfig.defaults.esp32s3

@@ -0,0 +1,2 @@
+CONFIG_IDF_TARGET="esp32s3"
+CONFIG_BOOTLOADER_NUM_PIN_APP_TEST=18

components/spi_flash/flash_mmap.c

@@ -270,81 +270,6 @@ uint32_t spi_flash_mmap_get_free_pages(spi_flash_mmap_memory_t memory)
     return len / CONFIG_MMU_PAGE_SIZE;
 }
 
-
-size_t spi_flash_cache2phys(const void *cached)
-{
-    if (cached == NULL) {
-        return SPI_FLASH_CACHE2PHYS_FAIL;
-    }
-
-    esp_err_t ret = ESP_FAIL;
-    uint32_t paddr = 0;
-    mmu_target_t target = 0;
-
-#if CONFIG_SPIRAM_FLASH_LOAD_TO_PSRAM //TODO: IDF-9049
-    paddr = mmu_xip_psram_flash_vaddr_to_paddr(cached);
-    //SPI_FLASH_CACHE2PHYS_FAIL is UINT32_MAX
-    if (paddr != SPI_FLASH_CACHE2PHYS_FAIL) {
-        return paddr;
-    }
-#endif
-
-    ret = esp_mmu_vaddr_to_paddr((void *)cached, &paddr, &target);
-    if (ret != ESP_OK) {
-        return SPI_FLASH_CACHE2PHYS_FAIL;
-    }
-
-    int offset = 0;
-
-#if !SOC_MMU_PER_EXT_MEM_TARGET //TODO: IDF-9049
-#if CONFIG_SPIRAM_RODATA
-    if ((uint32_t)cached >= (uint32_t)&_rodata_reserved_start && (uint32_t)cached <= (uint32_t)&_rodata_reserved_end) {
-        offset = rodata_flash2spiram_offset();
-    }
-#endif
-#if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
-    if ((uint32_t)cached >= (uint32_t)&_instruction_reserved_start && (uint32_t)cached <= (uint32_t)&_instruction_reserved_end) {
-        offset = instruction_flash2spiram_offset();
-    }
-#endif
-#endif  //#if !SOC_MMU_PER_EXT_MEM_TARGET
-
-    return paddr + offset * CONFIG_MMU_PAGE_SIZE;
-}
-
-
-const void * spi_flash_phys2cache(size_t phys_offs, spi_flash_mmap_memory_t memory)
-{
-    esp_err_t ret = ESP_FAIL;
-    void *ptr = NULL;
-    mmu_target_t target = MMU_TARGET_FLASH0;
-
-    __attribute__((unused)) uint32_t phys_page = phys_offs / CONFIG_MMU_PAGE_SIZE;
-#if !SOC_MMU_PER_EXT_MEM_TARGET
-#if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
-    if (phys_page >= instruction_flash_start_page_get() && phys_page <= instruction_flash_end_page_get()) {
-        target = MMU_TARGET_PSRAM0;
-        phys_offs -= instruction_flash2spiram_offset() * CONFIG_MMU_PAGE_SIZE;
-    }
-#endif
-
-#if CONFIG_SPIRAM_RODATA
-    if (phys_page >= rodata_flash_start_page_get() && phys_page <= rodata_flash_start_page_get()) {
-        target = MMU_TARGET_PSRAM0;
-        phys_offs -= rodata_flash2spiram_offset() * CONFIG_MMU_PAGE_SIZE;
-    }
-#endif
-#endif  //#if !SOC_MMU_PER_EXT_MEM_TARGET
-
-    mmu_vaddr_t type = (memory == SPI_FLASH_MMAP_DATA) ? MMU_VADDR_DATA : MMU_VADDR_INSTRUCTION;
-    ret = esp_mmu_paddr_to_vaddr(phys_offs, target, type, &ptr);
-    if (ret == ESP_ERR_NOT_FOUND) {
-        return NULL;
-    }
-    assert(ret == ESP_OK);
-    return (const void *)ptr;
-}
-
 static bool IRAM_ATTR is_page_mapped_in_cache(uint32_t phys_addr, const void **out_ptr)
 {
     *out_ptr = NULL;
@@ -397,3 +322,80 @@ IRAM_ATTR bool spi_flash_check_and_flush_cache(size_t start_addr, size_t length)
     return ret;
 }
 #endif //!CONFIG_SPI_FLASH_ROM_IMPL
+
+#if !CONFIG_SPI_FLASH_ROM_IMPL || CONFIG_SPIRAM_FETCH_INSTRUCTIONS || CONFIG_SPIRAM_RODATA
+//The ROM implementation returns physical address of the PSRAM when the .text or .rodata is in the PSRAM.
+//Always patch it when SPIRAM_FETCH_INSTRUCTIONS or SPIRAM_RODATA is set.
+size_t spi_flash_cache2phys(const void *cached)
+{
+    if (cached == NULL) {
+        return SPI_FLASH_CACHE2PHYS_FAIL;
+    }
+
+    esp_err_t ret = ESP_FAIL;
+    uint32_t paddr = 0;
+    mmu_target_t target = 0;
+
+#if CONFIG_SPIRAM_FLASH_LOAD_TO_PSRAM //TODO: IDF-9049
+    paddr = mmu_xip_psram_flash_vaddr_to_paddr(cached);
+    //SPI_FLASH_CACHE2PHYS_FAIL is UINT32_MAX
+    if (paddr != SPI_FLASH_CACHE2PHYS_FAIL) {
+        return paddr;
+    }
+#endif
+
+    ret = esp_mmu_vaddr_to_paddr((void *)cached, &paddr, &target);
+    if (ret != ESP_OK) {
+        return SPI_FLASH_CACHE2PHYS_FAIL;
+    }
+
+    int offset = 0;
+
+#if !SOC_MMU_PER_EXT_MEM_TARGET //TODO: IDF-9049
+#if CONFIG_SPIRAM_RODATA
+    if ((uint32_t)cached >= (uint32_t)&_rodata_reserved_start && (uint32_t)cached <= (uint32_t)&_rodata_reserved_end) {
+        offset = rodata_flash2spiram_offset();
+    }
+#endif
+#if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
+    if ((uint32_t)cached >= (uint32_t)&_instruction_reserved_start && (uint32_t)cached <= (uint32_t)&_instruction_reserved_end) {
+        offset = instruction_flash2spiram_offset();
+    }
+#endif
+#endif  //#if !SOC_MMU_PER_EXT_MEM_TARGET
+
+    return paddr + offset * CONFIG_MMU_PAGE_SIZE;
+}
+
+const void * spi_flash_phys2cache(size_t phys_offs, spi_flash_mmap_memory_t memory)
+{
+    esp_err_t ret = ESP_FAIL;
+    void *ptr = NULL;
+    mmu_target_t target = MMU_TARGET_FLASH0;
+
+    __attribute__((unused)) uint32_t phys_page = phys_offs / CONFIG_MMU_PAGE_SIZE;
+#if !SOC_MMU_PER_EXT_MEM_TARGET
+#if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
+    if (phys_page >= instruction_flash_start_page_get() && phys_page <= instruction_flash_end_page_get()) {
+        target = MMU_TARGET_PSRAM0;
+        phys_offs -= instruction_flash2spiram_offset() * CONFIG_MMU_PAGE_SIZE;
+    }
+#endif
+
+#if CONFIG_SPIRAM_RODATA
+    if (phys_page >= rodata_flash_start_page_get() && phys_page <= rodata_flash_start_page_get()) {
+        target = MMU_TARGET_PSRAM0;
+        phys_offs -= rodata_flash2spiram_offset() * CONFIG_MMU_PAGE_SIZE;
+    }
+#endif
+#endif  //#if !SOC_MMU_PER_EXT_MEM_TARGET
+
+    mmu_vaddr_t type = (memory == SPI_FLASH_MMAP_DATA) ? MMU_VADDR_DATA : MMU_VADDR_INSTRUCTION;
+    ret = esp_mmu_paddr_to_vaddr(phys_offs, target, type, &ptr);
+    if (ret == ESP_ERR_NOT_FOUND) {
+        return NULL;
+    }
+    assert(ret == ESP_OK);
+    return (const void *)ptr;
+}
+#endif //!CONFIG_SPI_FLASH_ROM_IMPL || CONFIG_SPIRAM_FETCH_INSTRUCTIONS || CONFIG_SPIRAM_RODATA

components/spi_flash/test_apps/.build-test-rules.yml

@@ -39,6 +39,11 @@ components/spi_flash/test_apps/flash_mmap:
   depends_components:
     - esp_mm
     - spi_flash
+  enable:
+    - if: CONFIG_NAME in ["release", "rom_impl"] and IDF_TARGET not in ["linux", "esp32c61"]
+    - if: CONFIG_NAME == "xip_psram" and IDF_TARGET in ["esp32s2", "esp32s3", "esp32p4"]
+    # S2 doesn't have ROM for flash
+    - if: CONFIG_NAME == "xip_psram_with_rom_impl" and IDF_TARGET in ["esp32s3", "esp32p4"]
 
 components/spi_flash/test_apps/flash_suspend:
   disable:

components/spi_flash/test_apps/flash_mmap/main/test_flash_mmap.c

@@ -7,6 +7,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <inttypes.h>
+#include "esp_log.h"
 #include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
 #include <freertos/semphr.h>
@@ -517,3 +518,14 @@ TEST_CASE("no stale data read post mmap and write partition", "[spi_flash][mmap]
     TEST_ASSERT_EQUAL(0, memcmp(buf, read_data, sizeof(buf)));
 #endif
 }
+
+TEST_CASE("spi_flash_cache2phys points to correct address", "[spi_flash]")
+{
+    //_rodata_start, which begins with appdesc, is always the first segment of the bin.
+    extern int _rodata_start;
+    size_t addr = spi_flash_cache2phys(&_rodata_start);
+
+    const esp_partition_t *factory = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_ANY, "factory");
+    ESP_LOGI("running bin", "0x%p", (void*)addr);
+    TEST_ASSERT_HEX32_WITHIN(CONFIG_MMU_PAGE_SIZE/2, factory->address + CONFIG_MMU_PAGE_SIZE/2, addr);
+}

components/spi_flash/test_apps/flash_mmap/partitions.csv

@@ -1,5 +1,6 @@
 # Name,     Type, SubType, Offset,   Size, Flags
 # Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
 nvs,        data, nvs,      0x9000,  0x6000,
-factory,    0,    0,        0x10000, 1M
-flash_test, data, fat,      ,        528K
+flash_test, data, fat,      0x10000, 528K
+# This partition is placed to this weird address intentionally to test spi_flash_cache2phys
+factory,    0,    0,        0xF0000, 1M

components/spi_flash/test_apps/flash_mmap/pytest_flash_mmap.py

@@ -1,6 +1,5 @@
 # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import pytest
 from pytest_embedded import Dut
 
@@ -35,19 +34,20 @@ def test_flash_mmap_rom_impl(dut: Dut) -> None:
     dut.run_all_single_board_cases(timeout=30)
 
 
-XIP_CONFIGS = [
-    pytest.param('xip_psram_esp32s2', marks=[pytest.mark.esp32s2]),
-    pytest.param('xip_psram_esp32s3', marks=[pytest.mark.esp32s3]),
-]
-
-
+@pytest.mark.supported_targets
 @pytest.mark.generic
-@pytest.mark.parametrize('config', XIP_CONFIGS, indirect=True)
+@pytest.mark.parametrize(
+    'config',
+    [
+        'xip_psram',
+    ],
+    indirect=True,
+)
 def test_flash_mmap_xip_psram(dut: Dut) -> None:
     dut.run_all_single_board_cases(timeout=30)
 
 
-@pytest.mark.esp32s3
+@pytest.mark.supported_targets
 @pytest.mark.generic
 @pytest.mark.parametrize(
     'config',

components/spi_flash/test_apps/flash_mmap/sdkconfig.ci.xip_psram

@@ -0,0 +1,2 @@
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_XIP_FROM_PSRAM=y

components/spi_flash/test_apps/flash_mmap/sdkconfig.ci.xip_psram_esp32s2

@@ -1,3 +0,0 @@
-CONFIG_IDF_TARGET="esp32s2"
-CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y
-CONFIG_SPIRAM_RODATA=y

components/spi_flash/test_apps/flash_mmap/sdkconfig.ci.xip_psram_esp32s3

@@ -1,3 +0,0 @@
-CONFIG_IDF_TARGET="esp32s3"
-CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y
-CONFIG_SPIRAM_RODATA=y

components/spi_flash/test_apps/flash_mmap/sdkconfig.ci.xip_psram_with_rom_impl

@@ -1,4 +1,3 @@
-CONFIG_IDF_TARGET="esp32s3"
-CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y
-CONFIG_SPIRAM_RODATA=y
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_XIP_FROM_PSRAM=y
 CONFIG_SPI_FLASH_ROM_IMPL=y