diff --git a/components/esp_mm/CMakeLists.txt b/components/esp_mm/CMakeLists.txt index 923ab0aae6..bf81e65185 100644 --- a/components/esp_mm/CMakeLists.txt +++ b/components/esp_mm/CMakeLists.txt @@ -14,14 +14,15 @@ set(srcs) if(NOT CONFIG_APP_BUILD_TYPE_PURE_RAM_APP) set(srcs "esp_mmu_map.c" "port/${target}/ext_mem_layout.c" - "esp_cache.c") + "esp_cache_msync.c" + "esp_cache_utils.c") if(CONFIG_IDF_TARGET_ESP32) list(APPEND srcs "cache_esp32.c") endif() else() if(CONFIG_SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) - list(APPEND srcs "esp_cache.c") + list(APPEND srcs "esp_cache_msync.c") endif() endif() diff --git a/components/esp_mm/esp_cache.c b/components/esp_mm/esp_cache_msync.c similarity index 88% rename from components/esp_mm/esp_cache.c rename to components/esp_mm/esp_cache_msync.c index d193ace39e..e8af782c26 100644 --- a/components/esp_mm/esp_cache.c +++ b/components/esp_mm/esp_cache_msync.c @@ -147,42 +147,6 @@ esp_err_t esp_cache_msync(void *addr, size_t size, int flags) return ESP_OK; } -void esp_cache_suspend_ext_mem_cache(void) -{ -#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) - /** - * before suspending the external mem cache, writeback internal mem cache content back to external mem cache - * to avoid stuck issue caused by internal mem cache auto-writeback - */ - cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL); -#endif - cache_hal_suspend(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); -} - -void esp_cache_resume_ext_mem_cache(void) -{ - cache_hal_resume(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); -} - -#if SOC_CACHE_FREEZE_SUPPORTED -void esp_cache_freeze_ext_mem_cache(void) -{ -#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) - /** - * before freezing the external mem cache, writeback internal mem cache content back to external mem cache - * to avoid stuck issue caused by internal mem cache auto-writeback - */ - cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL); -#endif - cache_hal_freeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); -} - -void esp_cache_unfreeze_ext_mem_cache(void) -{ - cache_hal_unfreeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); -} -#endif //#if SOC_CACHE_FREEZE_SUPPORTED - //The esp_cache_aligned_malloc function is marked deprecated but also called by other //(also deprecated) functions in this file. In order to work around that generating warnings, it's //split into a non-deprecated internal function and the stubbed external deprecated function. diff --git a/components/esp_mm/esp_cache_utils.c b/components/esp_mm/esp_cache_utils.c new file mode 100644 index 0000000000..f986c3b4ff --- /dev/null +++ b/components/esp_mm/esp_cache_utils.c @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +* +* SPDX-License-Identifier: Apache-2.0 +*/ + +#include +#include +#include +#include "sys/lock.h" +#include "sdkconfig.h" +#include "esp_check.h" +#include "esp_log.h" +#include "freertos/FreeRTOS.h" +#include "esp_memory_utils.h" +#include "soc/soc_caps.h" +#include "hal/cache_hal.h" +#include "hal/cache_ll.h" +#include "esp_private/esp_cache_private.h" +#include "esp_private/critical_section.h" +#if __riscv +#include "riscv/rv_utils.h" +#endif + +#define ALIGN_UP_BY(num, align) (((num) + ((align) - 1)) & ~((align) - 1)) + +void esp_cache_suspend_ext_mem_cache(void) +{ +#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) + /** + * before suspending the external mem cache, writeback internal mem cache content back to external mem cache + * to avoid stuck issue caused by internal mem cache auto-writeback + */ + cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL); +#endif + cache_hal_suspend(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); +} + +void esp_cache_resume_ext_mem_cache(void) +{ + cache_hal_resume(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); +} + +/*----------------------------------------------------------------------------- + * Cache Freeze Related + *----------------------------------------------------------------------------*/ +#if SOC_CACHE_FREEZE_SUPPORTED +DEFINE_CRIT_SECTION_LOCK_STATIC(s_spinlock); + +void esp_cache_freeze_ext_mem_cache(void) +{ +#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) +#if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE + int cpuid = xPortGetCoreID(); + uint32_t other_cpuid = (cpuid == 0) ? 1 : 0; + esp_cpu_stall(other_cpuid); +#else + //single core mode, don't need to stall other core +#endif + /** + * before freezing the external mem cache, writeback internal mem cache content back to external mem cache + * to avoid stuck issue caused by internal mem cache auto-writeback + */ + cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL); +#endif + + cache_hal_freeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); + +#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) +#if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE + esp_cpu_unstall(other_cpuid); +#else + //single core mode, don't need to unstall other core +#endif +#endif +} + +void esp_cache_unfreeze_ext_mem_cache(void) +{ + cache_hal_unfreeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); +} + +static inline bool s_task_stack_is_sane_when_cache_frozen(void) +{ + const void *sp = (const void *)esp_cpu_get_sp(); + + return esp_ptr_in_dram(sp) +#if CONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP + || esp_ptr_in_rtc_dram_fast(sp) +#endif + ; +} + +void esp_cache_freeze_caches_disable_interrupts(void) +{ + assert(s_task_stack_is_sane_when_cache_frozen()); + esp_os_enter_critical_safe(&s_spinlock); + + /** + * - disable non-iram interrupt on current core + * - current core call cache freeze + * - external access from other cores will hang on cache + */ + esp_intr_noniram_disable(); + esp_cache_freeze_ext_mem_cache(); +} + +void esp_cache_unfreeze_caches_enable_interrupts(void) +{ + esp_cache_unfreeze_ext_mem_cache(); + esp_intr_noniram_enable(); + esp_os_exit_critical_safe(&s_spinlock); +} +#endif diff --git a/components/esp_mm/esp_mmu_map.c b/components/esp_mm/esp_mmu_map.c index 32dadf7a82..52b719cb54 100644 --- a/components/esp_mm/esp_mmu_map.c +++ b/components/esp_mm/esp_mmu_map.c @@ -27,6 +27,7 @@ #include "esp_private/cache_utils.h" #include "esp_private/esp_cache_esp32_private.h" +#include "esp_private/esp_cache_private.h" #include "esp_private/esp_mmu_map_private.h" #include "ext_mem_layout.h" #include "esp_mmu_map.h" @@ -373,6 +374,26 @@ IRAM_ATTR esp_err_t esp_mmu_paddr_find_caps(const esp_paddr_t paddr, mmu_mem_cap return ESP_OK; } +static void IRAM_ATTR NOINLINE_ATTR s_stop_cache(void) +{ +#if SOC_CACHE_FREEZE_SUPPORTED && !CONFIG_IDF_TARGET_ESP32P4 + // On P4, due to limitations on stalling another core, we temporarily use cache disable/enable + esp_cache_freeze_caches_disable_interrupts(); +#else + spi_flash_disable_interrupts_caches_and_other_cpu(); +#endif +} + +static void IRAM_ATTR NOINLINE_ATTR s_start_cache(void) +{ +#if SOC_CACHE_FREEZE_SUPPORTED && !CONFIG_IDF_TARGET_ESP32P4 + // On P4, due to limitations on stalling another core, we temporarily use cache disable/enable + esp_cache_unfreeze_caches_enable_interrupts(); +#else + spi_flash_enable_interrupts_caches_and_other_cpu(); +#endif +} + static void IRAM_ATTR NOINLINE_ATTR s_do_cache_invalidate(uint32_t vaddr_start, uint32_t size) { #if CONFIG_IDF_TARGET_ESP32 @@ -416,10 +437,8 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_mapping(mmu_target_t target, uint32_t v { /** * Disable Cache, after this function, involved code and data should be placed in internal RAM. - * - * @note we call this for now, but this will be refactored to move out of `spi_flash` */ - spi_flash_disable_interrupts_caches_and_other_cpu(); + s_stop_cache(); uint32_t actual_mapped_len = s_mapping_operation(target, vaddr_start, paddr_start, size); @@ -433,7 +452,7 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_mapping(mmu_target_t target, uint32_t v s_do_cache_invalidate(vaddr_start, size); //enable Cache, after this function, internal RAM access is no longer mandatory - spi_flash_enable_interrupts_caches_and_other_cpu(); + s_start_cache(); ESP_EARLY_LOGV(TAG, "actual_mapped_len is 0x%"PRIx32, actual_mapped_len); } @@ -613,15 +632,13 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_unmapping(uint32_t vaddr_start, uint32_ { /** * Disable Cache, after this function, involved code and data should be placed in internal RAM. - * - * @note we call this for now, but this will be refactored to move out of `spi_flash` */ - spi_flash_disable_interrupts_caches_and_other_cpu(); + s_stop_cache(); s_unmapping_operation(vaddr_start, size); //enable Cache, after this function, internal RAM access is no longer mandatory - spi_flash_enable_interrupts_caches_and_other_cpu(); + s_start_cache(); } esp_err_t esp_mmu_unmap(void *ptr) @@ -759,8 +776,11 @@ esp_err_t IRAM_ATTR esp_mmu_map_dump_mapped_blocks_private(void) static bool NOINLINE_ATTR IRAM_ATTR s_vaddr_to_paddr(uint32_t vaddr, esp_paddr_t *out_paddr, mmu_target_t *out_target) { uint32_t mmu_id = 0; - //we call this for now, but this will be refactored to move out of `spi_flash` - spi_flash_disable_interrupts_caches_and_other_cpu(); + /** + * Disable Cache, after this function, involved code and data should be placed in internal RAM. + */ + s_stop_cache(); + #if SOC_MMU_PER_EXT_MEM_TARGET mmu_id = mmu_hal_get_id_from_vaddr(vaddr); #endif @@ -770,7 +790,9 @@ static bool NOINLINE_ATTR IRAM_ATTR s_vaddr_to_paddr(uint32_t vaddr, esp_paddr_t is_mapped = mmu_hal_vaddr_to_paddr(1, vaddr, out_paddr, out_target); } #endif - spi_flash_enable_interrupts_caches_and_other_cpu(); + + //enable Cache, after this function, internal RAM access is no longer mandatory + s_start_cache(); return is_mapped; } @@ -794,14 +816,19 @@ esp_err_t esp_mmu_vaddr_to_paddr(void *vaddr, esp_paddr_t *out_paddr, mmu_target static bool NOINLINE_ATTR IRAM_ATTR s_paddr_to_vaddr(esp_paddr_t paddr, mmu_target_t target, mmu_vaddr_t type, uint32_t *out_vaddr) { - //we call this for now, but this will be refactored to move out of `spi_flash` - spi_flash_disable_interrupts_caches_and_other_cpu(); + /** + * Disable Cache, after this function, involved code and data should be placed in internal RAM. + */ + s_stop_cache(); + uint32_t mmu_id = 0; #if SOC_MMU_PER_EXT_MEM_TARGET mmu_id = mmu_hal_get_id_from_target(target); #endif bool found = mmu_hal_paddr_to_vaddr(mmu_id, paddr, target, type, out_vaddr); - spi_flash_enable_interrupts_caches_and_other_cpu(); + + //enable Cache, after this function, internal RAM access is no longer mandatory + s_start_cache(); return found; } diff --git a/components/esp_mm/include/esp_private/esp_cache_private.h b/components/esp_mm/include/esp_private/esp_cache_private.h index 7380fddfa4..9768765b73 100644 --- a/components/esp_mm/include/esp_private/esp_cache_private.h +++ b/components/esp_mm/include/esp_private/esp_cache_private.h @@ -53,6 +53,18 @@ void esp_cache_freeze_ext_mem_cache(void); * @note This API must be called after a `esp_cache_freeze_ext_mem_cache` */ void esp_cache_unfreeze_ext_mem_cache(void); + +/** + * @brief Freeze external memory cache and disable non-iram interrupts + * + * @note This API will enter a critical section, you will need to call `esp_cache_unfreeze_caches_enable_interrupts` to exit it. + */ +void esp_cache_freeze_caches_disable_interrupts(void); + +/** + * @brief Unfreeze external memory cache and re-enable non-iram interrupts + */ +void esp_cache_unfreeze_caches_enable_interrupts(void); #endif /** diff --git a/components/esp_mm/linker.lf b/components/esp_mm/linker.lf index 1af07775dd..bf3f3820c3 100644 --- a/components/esp_mm/linker.lf +++ b/components/esp_mm/linker.lf @@ -3,7 +3,8 @@ archive: libesp_mm.a entries: if APP_BUILD_TYPE_PURE_RAM_APP = n: - esp_cache (noflash) + esp_cache_msync (noflash) + esp_cache_utils (noflash) if IDF_TARGET_ESP32 = y: cache_esp32 (noflash) diff --git a/components/esp_mm/test_apps/mm/main/CMakeLists.txt b/components/esp_mm/test_apps/mm/main/CMakeLists.txt index dc8881aef7..6b99509dbe 100644 --- a/components/esp_mm/test_apps/mm/main/CMakeLists.txt +++ b/components/esp_mm/test_apps/mm/main/CMakeLists.txt @@ -6,9 +6,12 @@ if(CONFIG_SOC_CACHE_WRITEBACK_SUPPORTED) list(APPEND srcs "test_cache_msync.c") endif() +if(CONFIG_SOC_CACHE_FREEZE_SUPPORTED) + list(APPEND srcs "test_cache_utils.c") +endif() # In order for the cases defined by `TEST_CASE` to be linked into the final elf, # the component can be registered as WHOLE_ARCHIVE idf_component_register(SRCS ${srcs} - PRIV_REQUIRES unity esp_partition spi_flash esp_mm driver esp_timer test_mm_utils + PRIV_REQUIRES unity esp_partition spi_flash esp_mm driver esp_timer test_mm_utils test_utils WHOLE_ARCHIVE) diff --git a/components/esp_mm/test_apps/mm/main/idf_component.yml b/components/esp_mm/test_apps/mm/main/idf_component.yml new file mode 100644 index 0000000000..f5001494e2 --- /dev/null +++ b/components/esp_mm/test_apps/mm/main/idf_component.yml @@ -0,0 +1,3 @@ +dependencies: + test_utils: + path: ${IDF_PATH}/tools/unit-test-app/components/test_utils diff --git a/components/esp_mm/test_apps/mm/main/test_cache_utils.c b/components/esp_mm/test_apps/mm/main/test_cache_utils.c new file mode 100644 index 0000000000..70f2529ec3 --- /dev/null +++ b/components/esp_mm/test_apps/mm/main/test_cache_utils.c @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +* +* SPDX-License-Identifier: Apache-2.0 +*/ + +#include +#include +#include "sdkconfig.h" +#include "inttypes.h" +#include "unity.h" +#include "esp_log.h" +#include "esp_attr.h" +#include "esp_cpu.h" +#include "esp_private/esp_cache_private.h" +#include "esp_private/cache_utils.h" +#include "test_utils.h" + +#if !CONFIG_FREERTOS_UNICORE + +#define RECORD_TIME_PREPARE() uint32_t __t1, __t2 +#define RECORD_TIME_START() do {__t1 = esp_cpu_get_cycle_count();} while(0) +#define RECORD_TIME_END(p_time) do{__t2 = esp_cpu_get_cycle_count(); p_time = (__t2 - __t1);} while(0) +#define GET_US_BY_CCOUNT(t) ((double)(t)/CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ) + +static void IRAM_ATTR NOINLINE_ATTR s_test_func_freeze_unfreeze(void) +{ + esp_cache_freeze_caches_disable_interrupts(); + for (int i = 0; i < 100; i++) { + asm volatile("nop"); + } + esp_cache_unfreeze_caches_enable_interrupts(); +} + +static void IRAM_ATTR NOINLINE_ATTR s_test_func_disable_enable(void) +{ + spi_flash_disable_interrupts_caches_and_other_cpu(); + for (int i = 0; i < 100; i++) { + asm volatile("nop"); + } + spi_flash_enable_interrupts_caches_and_other_cpu(); +} + +TEST_CASE("test esp_cache_freeze_caches_disable_interrupts speed", "[cache]") +{ + uint32_t cache_disable_enable_time = 0; + uint32_t cache_freeze_unfreeze_time = 0; + + s_test_func_freeze_unfreeze(); + s_test_func_disable_enable(); + + RECORD_TIME_PREPARE(); + RECORD_TIME_START(); + s_test_func_freeze_unfreeze(); + RECORD_TIME_END(cache_freeze_unfreeze_time); + uint32_t cache_freeze_unfreeze_time_us = GET_US_BY_CCOUNT(cache_freeze_unfreeze_time); + IDF_LOG_PERFORMANCE("Cache freeze time", "cache_freeze_unfreeze_time: %"PRId32" cpu cycles, %"PRId32" us", cache_freeze_unfreeze_time, cache_freeze_unfreeze_time_us); + + RECORD_TIME_START(); + s_test_func_disable_enable(); + RECORD_TIME_END(cache_disable_enable_time); + uint32_t cache_disable_enable_time_us = GET_US_BY_CCOUNT(cache_disable_enable_time); + IDF_LOG_PERFORMANCE("Cache disable time", "cache_disable_enable_time: %"PRId32", cpu cycles, %"PRId32" us", cache_disable_enable_time, cache_disable_enable_time_us); + + TEST_ASSERT(cache_freeze_unfreeze_time < cache_disable_enable_time); +} +#endif // #if !CONFIG_FREERTOS_UNICORE