Merge branch 'refactor/change_mmap_cache_lock_type' into 'master'

mmu: use cache freeze for mmap APIs

See merge request espressif/esp-idf!39199

components/esp_mm/CMakeLists.txt

@@ -14,14 +14,15 @@ set(srcs)
 if(NOT CONFIG_APP_BUILD_TYPE_PURE_RAM_APP)
     set(srcs "esp_mmu_map.c"
              "port/${target}/ext_mem_layout.c"
-             "esp_cache.c")
+             "esp_cache_msync.c"
+             "esp_cache_utils.c")
 
     if(CONFIG_IDF_TARGET_ESP32)
         list(APPEND srcs "cache_esp32.c")
     endif()
 else()
     if(CONFIG_SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE)
-        list(APPEND srcs "esp_cache.c")
+        list(APPEND srcs "esp_cache_msync.c")
     endif()
 endif()
 

components/esp_mm/esp_cache_msync.c

@@ -147,42 +147,6 @@ esp_err_t esp_cache_msync(void *addr, size_t size, int flags)
     return ESP_OK;
 }
 
-void esp_cache_suspend_ext_mem_cache(void)
-{
-#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE)
-    /**
-     * before suspending the external mem cache, writeback internal mem cache content back to external mem cache
-     * to avoid stuck issue caused by internal mem cache auto-writeback
-     */
-    cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL);
-#endif
-    cache_hal_suspend(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL);
-}
-
-void esp_cache_resume_ext_mem_cache(void)
-{
-    cache_hal_resume(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL);
-}
-
-#if SOC_CACHE_FREEZE_SUPPORTED
-void esp_cache_freeze_ext_mem_cache(void)
-{
-#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE)
-    /**
-     * before freezing the external mem cache, writeback internal mem cache content back to external mem cache
-     * to avoid stuck issue caused by internal mem cache auto-writeback
-     */
-    cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL);
-#endif
-    cache_hal_freeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL);
-}
-
-void esp_cache_unfreeze_ext_mem_cache(void)
-{
-    cache_hal_unfreeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL);
-}
-#endif  //#if SOC_CACHE_FREEZE_SUPPORTED
-
 //The esp_cache_aligned_malloc function is marked deprecated but also called by other
 //(also deprecated) functions in this file. In order to work around that generating warnings, it's
 //split into a non-deprecated internal function and the stubbed external deprecated function.

components/esp_mm/esp_cache_utils.c

@@ -0,0 +1,114 @@
+/*
+ * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
+*
+* SPDX-License-Identifier: Apache-2.0
+*/
+
+#include <sys/param.h>
+#include <inttypes.h>
+#include <string.h>
+#include "sys/lock.h"
+#include "sdkconfig.h"
+#include "esp_check.h"
+#include "esp_log.h"
+#include "freertos/FreeRTOS.h"
+#include "esp_memory_utils.h"
+#include "soc/soc_caps.h"
+#include "hal/cache_hal.h"
+#include "hal/cache_ll.h"
+#include "esp_private/esp_cache_private.h"
+#include "esp_private/critical_section.h"
+#if __riscv
+#include "riscv/rv_utils.h"
+#endif
+
+#define ALIGN_UP_BY(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
+
+void esp_cache_suspend_ext_mem_cache(void)
+{
+#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE)
+    /**
+     * before suspending the external mem cache, writeback internal mem cache content back to external mem cache
+     * to avoid stuck issue caused by internal mem cache auto-writeback
+     */
+    cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL);
+#endif
+    cache_hal_suspend(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL);
+}
+
+void esp_cache_resume_ext_mem_cache(void)
+{
+    cache_hal_resume(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL);
+}
+
+/*-----------------------------------------------------------------------------
+ *                      Cache Freeze Related
+ *----------------------------------------------------------------------------*/
+#if SOC_CACHE_FREEZE_SUPPORTED
+DEFINE_CRIT_SECTION_LOCK_STATIC(s_spinlock);
+
+void esp_cache_freeze_ext_mem_cache(void)
+{
+#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE)
+#if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE
+    int cpuid = xPortGetCoreID();
+    uint32_t other_cpuid = (cpuid == 0) ? 1 : 0;
+    esp_cpu_stall(other_cpuid);
+#else
+    //single core mode, don't need to stall other core
+#endif
+    /**
+     * before freezing the external mem cache, writeback internal mem cache content back to external mem cache
+     * to avoid stuck issue caused by internal mem cache auto-writeback
+     */
+    cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL);
+#endif
+
+    cache_hal_freeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL);
+
+#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE)
+#if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE
+    esp_cpu_unstall(other_cpuid);
+#else
+    //single core mode, don't need to unstall other core
+#endif
+#endif
+}
+
+void esp_cache_unfreeze_ext_mem_cache(void)
+{
+    cache_hal_unfreeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL);
+}
+
+static inline bool s_task_stack_is_sane_when_cache_frozen(void)
+{
+    const void *sp = (const void *)esp_cpu_get_sp();
+
+    return esp_ptr_in_dram(sp)
+#if CONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP
+           || esp_ptr_in_rtc_dram_fast(sp)
+#endif
+           ;
+}
+
+void esp_cache_freeze_caches_disable_interrupts(void)
+{
+    assert(s_task_stack_is_sane_when_cache_frozen());
+    esp_os_enter_critical_safe(&s_spinlock);
+
+    /**
+     * - disable non-iram interrupt on current core
+     * - current core call cache freeze
+     * - external access from other cores will hang on cache
+     */
+    esp_intr_noniram_disable();
+    esp_cache_freeze_ext_mem_cache();
+}
+
+void esp_cache_unfreeze_caches_enable_interrupts(void)
+{
+    esp_cache_unfreeze_ext_mem_cache();
+    esp_intr_noniram_enable();
+    esp_os_exit_critical_safe(&s_spinlock);
+}
+#endif

components/esp_mm/esp_mmu_map.c

@@ -27,6 +27,7 @@
 
 #include "esp_private/cache_utils.h"
 #include "esp_private/esp_cache_esp32_private.h"
+#include "esp_private/esp_cache_private.h"
 #include "esp_private/esp_mmu_map_private.h"
 #include "ext_mem_layout.h"
 #include "esp_mmu_map.h"
@@ -373,6 +374,26 @@ IRAM_ATTR esp_err_t esp_mmu_paddr_find_caps(const esp_paddr_t paddr, mmu_mem_cap
     return ESP_OK;
 }
 
+static void IRAM_ATTR NOINLINE_ATTR s_stop_cache(void)
+{
+#if SOC_CACHE_FREEZE_SUPPORTED && !CONFIG_IDF_TARGET_ESP32P4
+    // On P4, due to limitations on stalling another core, we temporarily use cache disable/enable
+    esp_cache_freeze_caches_disable_interrupts();
+#else
+    spi_flash_disable_interrupts_caches_and_other_cpu();
+#endif
+}
+
+static void IRAM_ATTR NOINLINE_ATTR s_start_cache(void)
+{
+#if SOC_CACHE_FREEZE_SUPPORTED && !CONFIG_IDF_TARGET_ESP32P4
+    // On P4, due to limitations on stalling another core, we temporarily use cache disable/enable
+    esp_cache_unfreeze_caches_enable_interrupts();
+#else
+    spi_flash_enable_interrupts_caches_and_other_cpu();
+#endif
+}
+
 static void IRAM_ATTR NOINLINE_ATTR s_do_cache_invalidate(uint32_t vaddr_start, uint32_t size)
 {
 #if CONFIG_IDF_TARGET_ESP32
@@ -416,10 +437,8 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_mapping(mmu_target_t target, uint32_t v
 {
     /**
      * Disable Cache, after this function, involved code and data should be placed in internal RAM.
-     *
-     * @note we call this for now, but this will be refactored to move out of `spi_flash`
      */
-    spi_flash_disable_interrupts_caches_and_other_cpu();
+    s_stop_cache();
 
     uint32_t actual_mapped_len = s_mapping_operation(target, vaddr_start, paddr_start, size);
 
@@ -433,7 +452,7 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_mapping(mmu_target_t target, uint32_t v
     s_do_cache_invalidate(vaddr_start, size);
 
     //enable Cache, after this function, internal RAM access is no longer mandatory
-    spi_flash_enable_interrupts_caches_and_other_cpu();
+    s_start_cache();
 
     ESP_EARLY_LOGV(TAG, "actual_mapped_len is 0x%"PRIx32, actual_mapped_len);
 }
@@ -613,15 +632,13 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_unmapping(uint32_t vaddr_start, uint32_
 {
     /**
      * Disable Cache, after this function, involved code and data should be placed in internal RAM.
-     *
-     * @note we call this for now, but this will be refactored to move out of `spi_flash`
      */
-    spi_flash_disable_interrupts_caches_and_other_cpu();
+    s_stop_cache();
 
     s_unmapping_operation(vaddr_start, size);
 
     //enable Cache, after this function, internal RAM access is no longer mandatory
-    spi_flash_enable_interrupts_caches_and_other_cpu();
+    s_start_cache();
 }
 
 esp_err_t esp_mmu_unmap(void *ptr)
@@ -759,8 +776,11 @@ esp_err_t IRAM_ATTR esp_mmu_map_dump_mapped_blocks_private(void)
 static bool NOINLINE_ATTR IRAM_ATTR s_vaddr_to_paddr(uint32_t vaddr, esp_paddr_t *out_paddr, mmu_target_t *out_target)
 {
     uint32_t mmu_id = 0;
-    //we call this for now, but this will be refactored to move out of `spi_flash`
-    spi_flash_disable_interrupts_caches_and_other_cpu();
+    /**
+     * Disable Cache, after this function, involved code and data should be placed in internal RAM.
+     */
+    s_stop_cache();
+
 #if SOC_MMU_PER_EXT_MEM_TARGET
     mmu_id = mmu_hal_get_id_from_vaddr(vaddr);
 #endif
@@ -770,7 +790,9 @@ static bool NOINLINE_ATTR IRAM_ATTR s_vaddr_to_paddr(uint32_t vaddr, esp_paddr_t
         is_mapped = mmu_hal_vaddr_to_paddr(1, vaddr, out_paddr, out_target);
     }
 #endif
-    spi_flash_enable_interrupts_caches_and_other_cpu();
+
+    //enable Cache, after this function, internal RAM access is no longer mandatory
+    s_start_cache();
 
     return is_mapped;
 }
@@ -794,14 +816,19 @@ esp_err_t esp_mmu_vaddr_to_paddr(void *vaddr, esp_paddr_t *out_paddr, mmu_target
 
 static bool NOINLINE_ATTR IRAM_ATTR s_paddr_to_vaddr(esp_paddr_t paddr, mmu_target_t target, mmu_vaddr_t type, uint32_t *out_vaddr)
 {
-    //we call this for now, but this will be refactored to move out of `spi_flash`
-    spi_flash_disable_interrupts_caches_and_other_cpu();
+    /**
+     * Disable Cache, after this function, involved code and data should be placed in internal RAM.
+     */
+    s_stop_cache();
+
     uint32_t mmu_id = 0;
 #if SOC_MMU_PER_EXT_MEM_TARGET
     mmu_id = mmu_hal_get_id_from_target(target);
 #endif
     bool found = mmu_hal_paddr_to_vaddr(mmu_id, paddr, target, type, out_vaddr);
-    spi_flash_enable_interrupts_caches_and_other_cpu();
+
+    //enable Cache, after this function, internal RAM access is no longer mandatory
+    s_start_cache();
 
     return found;
 }

components/esp_mm/include/esp_private/esp_cache_private.h

@@ -53,6 +53,18 @@ void esp_cache_freeze_ext_mem_cache(void);
  * @note This API must be called after a `esp_cache_freeze_ext_mem_cache`
  */
 void esp_cache_unfreeze_ext_mem_cache(void);
+
+/**
+ * @brief Freeze external memory cache and disable non-iram interrupts
+ *
+ * @note This API will enter a critical section, you will need to call `esp_cache_unfreeze_caches_enable_interrupts` to exit it.
+ */
+void esp_cache_freeze_caches_disable_interrupts(void);
+
+/**
+ * @brief Unfreeze external memory cache and re-enable non-iram interrupts
+ */
+void esp_cache_unfreeze_caches_enable_interrupts(void);
 #endif
 
 /**

components/esp_mm/linker.lf

@@ -3,7 +3,8 @@ archive: libesp_mm.a
 entries:
 
     if APP_BUILD_TYPE_PURE_RAM_APP = n:
-        esp_cache (noflash)
+        esp_cache_msync (noflash)
+        esp_cache_utils (noflash)
 
         if IDF_TARGET_ESP32 = y:
             cache_esp32 (noflash)

components/esp_mm/test_apps/mm/main/CMakeLists.txt

@@ -6,9 +6,12 @@ if(CONFIG_SOC_CACHE_WRITEBACK_SUPPORTED)
     list(APPEND srcs "test_cache_msync.c")
 endif()
 
+if(CONFIG_SOC_CACHE_FREEZE_SUPPORTED)
+    list(APPEND srcs "test_cache_utils.c")
+endif()
 
 # In order for the cases defined by `TEST_CASE` to be linked into the final elf,
 # the component can be registered as WHOLE_ARCHIVE
 idf_component_register(SRCS ${srcs}
-                       PRIV_REQUIRES unity esp_partition spi_flash esp_mm driver esp_timer test_mm_utils
+                       PRIV_REQUIRES unity esp_partition spi_flash esp_mm driver esp_timer test_mm_utils test_utils
                        WHOLE_ARCHIVE)

components/esp_mm/test_apps/mm/main/idf_component.yml

@@ -0,0 +1,3 @@
+dependencies:
+  test_utils:
+    path: ${IDF_PATH}/tools/unit-test-app/components/test_utils

components/esp_mm/test_apps/mm/main/test_cache_utils.c

@@ -0,0 +1,67 @@
+/*
+ * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
+*
+* SPDX-License-Identifier: Apache-2.0
+*/
+
+#include <sys/param.h>
+#include <string.h>
+#include "sdkconfig.h"
+#include "inttypes.h"
+#include "unity.h"
+#include "esp_log.h"
+#include "esp_attr.h"
+#include "esp_cpu.h"
+#include "esp_private/esp_cache_private.h"
+#include "esp_private/cache_utils.h"
+#include "test_utils.h"
+
+#if !CONFIG_FREERTOS_UNICORE
+
+#define RECORD_TIME_PREPARE()   uint32_t __t1, __t2
+#define RECORD_TIME_START()     do {__t1 = esp_cpu_get_cycle_count();} while(0)
+#define RECORD_TIME_END(p_time) do{__t2 = esp_cpu_get_cycle_count(); p_time = (__t2 - __t1);} while(0)
+#define GET_US_BY_CCOUNT(t)     ((double)(t)/CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ)
+
+static void IRAM_ATTR NOINLINE_ATTR s_test_func_freeze_unfreeze(void)
+{
+    esp_cache_freeze_caches_disable_interrupts();
+    for (int i = 0; i < 100; i++) {
+        asm volatile("nop");
+    }
+    esp_cache_unfreeze_caches_enable_interrupts();
+}
+
+static void IRAM_ATTR NOINLINE_ATTR s_test_func_disable_enable(void)
+{
+    spi_flash_disable_interrupts_caches_and_other_cpu();
+    for (int i = 0; i < 100; i++) {
+        asm volatile("nop");
+    }
+    spi_flash_enable_interrupts_caches_and_other_cpu();
+}
+
+TEST_CASE("test esp_cache_freeze_caches_disable_interrupts speed", "[cache]")
+{
+    uint32_t cache_disable_enable_time = 0;
+    uint32_t cache_freeze_unfreeze_time = 0;
+
+    s_test_func_freeze_unfreeze();
+    s_test_func_disable_enable();
+
+    RECORD_TIME_PREPARE();
+    RECORD_TIME_START();
+    s_test_func_freeze_unfreeze();
+    RECORD_TIME_END(cache_freeze_unfreeze_time);
+    uint32_t cache_freeze_unfreeze_time_us = GET_US_BY_CCOUNT(cache_freeze_unfreeze_time);
+    IDF_LOG_PERFORMANCE("Cache freeze time", "cache_freeze_unfreeze_time: %"PRId32" cpu cycles, %"PRId32" us", cache_freeze_unfreeze_time, cache_freeze_unfreeze_time_us);
+
+    RECORD_TIME_START();
+    s_test_func_disable_enable();
+    RECORD_TIME_END(cache_disable_enable_time);
+    uint32_t cache_disable_enable_time_us = GET_US_BY_CCOUNT(cache_disable_enable_time);
+    IDF_LOG_PERFORMANCE("Cache disable time", "cache_disable_enable_time: %"PRId32", cpu cycles, %"PRId32" us", cache_disable_enable_time, cache_disable_enable_time_us);
+
+    TEST_ASSERT(cache_freeze_unfreeze_time < cache_disable_enable_time);
+}
+#endif  // #if !CONFIG_FREERTOS_UNICORE