diff --git a/components/esp_mm/CMakeLists.txt b/components/esp_mm/CMakeLists.txt index 9878840bab..3fb323dce2 100644 --- a/components/esp_mm/CMakeLists.txt +++ b/components/esp_mm/CMakeLists.txt @@ -14,11 +14,16 @@ set(srcs) if(NOT CONFIG_APP_BUILD_TYPE_PURE_RAM_APP) set(srcs "esp_mmu_map.c" "port/${target}/ext_mem_layout.c" - "esp_cache.c") + "esp_cache_msync.c" + "esp_cache_utils.c") if(CONFIG_IDF_TARGET_ESP32) list(APPEND srcs "cache_esp32.c") endif() +else() + if(CONFIG_SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) + list(APPEND srcs "esp_cache_msync.c") + endif() endif() idf_component_register(SRCS ${srcs} diff --git a/components/esp_mm/esp_cache.c b/components/esp_mm/esp_cache_msync.c similarity index 100% rename from components/esp_mm/esp_cache.c rename to components/esp_mm/esp_cache_msync.c diff --git a/components/esp_mm/esp_cache_utils.c b/components/esp_mm/esp_cache_utils.c new file mode 100644 index 0000000000..392f3a7173 --- /dev/null +++ b/components/esp_mm/esp_cache_utils.c @@ -0,0 +1,97 @@ +/* + * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +* +* SPDX-License-Identifier: Apache-2.0 +*/ + +#include +#include +#include +#include "sys/lock.h" +#include "sdkconfig.h" +#include "esp_check.h" +#include "esp_log.h" +#include "freertos/FreeRTOS.h" +#include "esp_memory_utils.h" +#include "soc/soc_caps.h" +#include "hal/cache_hal.h" +#include "hal/cache_ll.h" +#include "esp_private/esp_cache_private.h" +#include "esp_private/critical_section.h" +#if __riscv +#include "riscv/rv_utils.h" +#endif + +#define ALIGN_UP_BY(num, align) (((num) + ((align) - 1)) & ~((align) - 1)) + +/*----------------------------------------------------------------------------- + * Cache Freeze Related + *----------------------------------------------------------------------------*/ +#if SOC_CACHE_FREEZE_SUPPORTED +DEFINE_CRIT_SECTION_LOCK_STATIC(s_spinlock); + +void esp_cache_freeze_ext_mem_cache(void) +{ +#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) +#if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE + int cpuid = xPortGetCoreID(); + uint32_t other_cpuid = (cpuid == 0) ? 1 : 0; + esp_cpu_stall(other_cpuid); +#else + //single core mode, don't need to stall other core +#endif + /** + * before freezing the external mem cache, writeback internal mem cache content back to external mem cache + * to avoid stuck issue caused by internal mem cache auto-writeback + */ + cache_ll_writeback_all(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA, CACHE_LL_ID_ALL); +#endif + + cache_hal_freeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); + +#if (CONFIG_SPIRAM && SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE) +#if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE + esp_cpu_unstall(other_cpuid); +#else + //single core mode, don't need to unstall other core +#endif +#endif +} + +void esp_cache_unfreeze_ext_mem_cache(void) +{ + cache_hal_unfreeze(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_ALL); +} + +static inline bool s_task_stack_is_sane_when_cache_frozen(void) +{ + const void *sp = (const void *)esp_cpu_get_sp(); + + return esp_ptr_in_dram(sp) +#if CONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP + || esp_ptr_in_rtc_dram_fast(sp) +#endif + ; +} + +void esp_cache_freeze_caches_disable_interrupts(void) +{ + assert(s_task_stack_is_sane_when_cache_frozen()); + esp_os_enter_critical_safe(&s_spinlock); + + /** + * - disable non-iram interrupt on current core + * - current core call cache freeze + * - external access from other cores will hang on cache + */ + esp_intr_noniram_disable(); + esp_cache_freeze_ext_mem_cache(); +} + +void esp_cache_unfreeze_caches_enable_interrupts(void) +{ + esp_cache_unfreeze_ext_mem_cache(); + esp_intr_noniram_enable(); + esp_os_exit_critical_safe(&s_spinlock); +} +#endif diff --git a/components/esp_mm/esp_mmu_map.c b/components/esp_mm/esp_mmu_map.c index d3dec35b55..94c0177991 100644 --- a/components/esp_mm/esp_mmu_map.c +++ b/components/esp_mm/esp_mmu_map.c @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -26,6 +26,7 @@ #include "esp_private/cache_utils.h" #include "esp_private/esp_cache_esp32_private.h" +#include "esp_private/esp_cache_private.h" #include "esp_private/esp_mmu_map_private.h" #include "ext_mem_layout.h" #include "esp_mmu_map.h" @@ -372,6 +373,26 @@ IRAM_ATTR esp_err_t esp_mmu_paddr_find_caps(const esp_paddr_t paddr, mmu_mem_cap return ESP_OK; } +static void IRAM_ATTR NOINLINE_ATTR s_stop_cache(void) +{ +#if SOC_CACHE_FREEZE_SUPPORTED && !CONFIG_IDF_TARGET_ESP32P4 + // On P4, due to limitations on stalling another core, we temporarily use cache disable/enable + esp_cache_freeze_caches_disable_interrupts(); +#else + spi_flash_disable_interrupts_caches_and_other_cpu(); +#endif +} + +static void IRAM_ATTR NOINLINE_ATTR s_start_cache(void) +{ +#if SOC_CACHE_FREEZE_SUPPORTED && !CONFIG_IDF_TARGET_ESP32P4 + // On P4, due to limitations on stalling another core, we temporarily use cache disable/enable + esp_cache_unfreeze_caches_enable_interrupts(); +#else + spi_flash_enable_interrupts_caches_and_other_cpu(); +#endif +} + static void IRAM_ATTR NOINLINE_ATTR s_do_cache_invalidate(uint32_t vaddr_start, uint32_t size) { #if CONFIG_IDF_TARGET_ESP32 @@ -420,10 +441,8 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_mapping(mmu_target_t target, uint32_t v { /** * Disable Cache, after this function, involved code and data should be placed in internal RAM. - * - * @note we call this for now, but this will be refactored to move out of `spi_flash` */ - spi_flash_disable_interrupts_caches_and_other_cpu(); + s_stop_cache(); uint32_t actual_mapped_len = s_mapping_operation(target, vaddr_start, paddr_start, size); @@ -437,7 +456,7 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_mapping(mmu_target_t target, uint32_t v s_do_cache_invalidate(vaddr_start, size); //enable Cache, after this function, internal RAM access is no longer mandatory - spi_flash_enable_interrupts_caches_and_other_cpu(); + s_start_cache(); ESP_EARLY_LOGV(TAG, "actual_mapped_len is 0x%"PRIx32, actual_mapped_len); } @@ -623,15 +642,13 @@ static void IRAM_ATTR NOINLINE_ATTR s_do_unmapping(uint32_t vaddr_start, uint32_ { /** * Disable Cache, after this function, involved code and data should be placed in internal RAM. - * - * @note we call this for now, but this will be refactored to move out of `spi_flash` */ - spi_flash_disable_interrupts_caches_and_other_cpu(); + s_stop_cache(); s_unmapping_operation(vaddr_start, size); //enable Cache, after this function, internal RAM access is no longer mandatory - spi_flash_enable_interrupts_caches_and_other_cpu(); + s_start_cache(); } esp_err_t esp_mmu_unmap(void *ptr) @@ -766,11 +783,16 @@ esp_err_t IRAM_ATTR esp_mmu_map_dump_mapped_blocks_private(void) ---------------------------------------------------------------*/ static bool NOINLINE_ATTR IRAM_ATTR s_vaddr_to_paddr(uint32_t vaddr, esp_paddr_t *out_paddr, mmu_target_t *out_target) { - //we call this for now, but this will be refactored to move out of `spi_flash` - spi_flash_disable_interrupts_caches_and_other_cpu(); + /** + * Disable Cache, after this function, involved code and data should be placed in internal RAM. + */ + s_stop_cache(); + //On ESP32, core 1 settings should be the same as the core 0 bool is_mapped = mmu_hal_vaddr_to_paddr(0, vaddr, out_paddr, out_target); - spi_flash_enable_interrupts_caches_and_other_cpu(); + + //enable Cache, after this function, internal RAM access is no longer mandatory + s_start_cache(); return is_mapped; } @@ -794,11 +816,16 @@ esp_err_t esp_mmu_vaddr_to_paddr(void *vaddr, esp_paddr_t *out_paddr, mmu_target static bool NOINLINE_ATTR IRAM_ATTR s_paddr_to_vaddr(esp_paddr_t paddr, mmu_target_t target, mmu_vaddr_t type, uint32_t *out_vaddr) { - //we call this for now, but this will be refactored to move out of `spi_flash` - spi_flash_disable_interrupts_caches_and_other_cpu(); + /** + * Disable Cache, after this function, involved code and data should be placed in internal RAM. + */ + s_stop_cache(); + //On ESP32, core 1 settings should be the same as the core 0 bool found = mmu_hal_paddr_to_vaddr(0, paddr, target, type, out_vaddr); - spi_flash_enable_interrupts_caches_and_other_cpu(); + + //enable Cache, after this function, internal RAM access is no longer mandatory + s_start_cache(); return found; } diff --git a/components/esp_mm/include/esp_private/esp_cache_private.h b/components/esp_mm/include/esp_private/esp_cache_private.h index cbad00163f..83cd8a2514 100644 --- a/components/esp_mm/include/esp_private/esp_cache_private.h +++ b/components/esp_mm/include/esp_private/esp_cache_private.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include "esp_err.h" #include "esp_bit_defs.h" +#include "soc/soc_caps.h" #ifdef __cplusplus extern "C" { @@ -27,6 +28,20 @@ extern "C" { */ #define ESP_CACHE_MALLOC_FLAG_DMA BIT(1) +#if SOC_CACHE_FREEZE_SUPPORTED +/** + * @brief Freeze external memory cache and disable non-iram interrupts + * + * @note This API will enter a critical section, you will need to call `esp_cache_unfreeze_caches_enable_interrupts` to exit it. + */ +void esp_cache_freeze_caches_disable_interrupts(void); + +/** + * @brief Unfreeze external memory cache and re-enable non-iram interrupts + */ +void esp_cache_unfreeze_caches_enable_interrupts(void); +#endif + /** * @brief Helper function for malloc a cache aligned data memory buffer * diff --git a/components/esp_mm/linker.lf b/components/esp_mm/linker.lf index a48fe34cbf..0ddc988feb 100644 --- a/components/esp_mm/linker.lf +++ b/components/esp_mm/linker.lf @@ -3,7 +3,8 @@ archive: libesp_mm.a entries: if APP_BUILD_TYPE_PURE_RAM_APP = n: - esp_cache (noflash) + esp_cache_msync (noflash) + esp_cache_utils (noflash) if IDF_TARGET_ESP32 = y: cache_esp32 (noflash) diff --git a/components/esp_mm/test_apps/mm/main/CMakeLists.txt b/components/esp_mm/test_apps/mm/main/CMakeLists.txt index cb7a1de73d..abf19b3ad3 100644 --- a/components/esp_mm/test_apps/mm/main/CMakeLists.txt +++ b/components/esp_mm/test_apps/mm/main/CMakeLists.txt @@ -6,9 +6,12 @@ if(CONFIG_SOC_CACHE_WRITEBACK_SUPPORTED) list(APPEND srcs "test_cache_msync.c") endif() +if(CONFIG_SOC_CACHE_FREEZE_SUPPORTED) + list(APPEND srcs "test_cache_utils.c") +endif() # In order for the cases defined by `TEST_CASE` to be linked into the final elf, # the component can be registered as WHOLE_ARCHIVE idf_component_register(SRCS ${srcs} - PRIV_REQUIRES unity esp_partition spi_flash esp_mm driver esp_timer test_mm_utils + PRIV_REQUIRES unity esp_partition spi_flash esp_mm driver esp_timer test_mm_utils test_utils WHOLE_ARCHIVE) diff --git a/components/esp_mm/test_apps/mm/main/idf_component.yml b/components/esp_mm/test_apps/mm/main/idf_component.yml new file mode 100644 index 0000000000..f5001494e2 --- /dev/null +++ b/components/esp_mm/test_apps/mm/main/idf_component.yml @@ -0,0 +1,3 @@ +dependencies: + test_utils: + path: ${IDF_PATH}/tools/unit-test-app/components/test_utils diff --git a/components/esp_mm/test_apps/mm/main/test_cache_utils.c b/components/esp_mm/test_apps/mm/main/test_cache_utils.c new file mode 100644 index 0000000000..d1d46e2675 --- /dev/null +++ b/components/esp_mm/test_apps/mm/main/test_cache_utils.c @@ -0,0 +1,69 @@ +/* + * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +* +* SPDX-License-Identifier: Apache-2.0 +*/ + +#include +#include +#include "sdkconfig.h" +#include "inttypes.h" +#include "unity.h" +#include "esp_log.h" +#include "esp_attr.h" +#include "esp_cpu.h" +#include "esp_private/esp_cache_private.h" +#include "esp_private/cache_utils.h" +#include "test_utils.h" + +#if !CONFIG_FREERTOS_UNICORE +#if !CONFIG_IDF_TARGET_ESP32P4 + +#define RECORD_TIME_PREPARE() uint32_t __t1, __t2 +#define RECORD_TIME_START() do {__t1 = esp_cpu_get_cycle_count();} while(0) +#define RECORD_TIME_END(p_time) do{__t2 = esp_cpu_get_cycle_count(); p_time = (__t2 - __t1);} while(0) +#define GET_US_BY_CCOUNT(t) ((double)(t)/CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ) + +static void IRAM_ATTR NOINLINE_ATTR s_test_func_freeze_unfreeze(void) +{ + esp_cache_freeze_caches_disable_interrupts(); + for (int i = 0; i < 100; i++) { + asm volatile("nop"); + } + esp_cache_unfreeze_caches_enable_interrupts(); +} + +static void IRAM_ATTR NOINLINE_ATTR s_test_func_disable_enable(void) +{ + spi_flash_disable_interrupts_caches_and_other_cpu(); + for (int i = 0; i < 100; i++) { + asm volatile("nop"); + } + spi_flash_enable_interrupts_caches_and_other_cpu(); +} + +TEST_CASE("test esp_cache_freeze_caches_disable_interrupts speed", "[cache]") +{ + uint32_t cache_disable_enable_time = 0; + uint32_t cache_freeze_unfreeze_time = 0; + + s_test_func_freeze_unfreeze(); + s_test_func_disable_enable(); + + RECORD_TIME_PREPARE(); + RECORD_TIME_START(); + s_test_func_freeze_unfreeze(); + RECORD_TIME_END(cache_freeze_unfreeze_time); + uint32_t cache_freeze_unfreeze_time_us = GET_US_BY_CCOUNT(cache_freeze_unfreeze_time); + IDF_LOG_PERFORMANCE("Cache freeze time", "cache_freeze_unfreeze_time: %"PRId32" cpu cycles, %"PRId32" us", cache_freeze_unfreeze_time, cache_freeze_unfreeze_time_us); + + RECORD_TIME_START(); + s_test_func_disable_enable(); + RECORD_TIME_END(cache_disable_enable_time); + uint32_t cache_disable_enable_time_us = GET_US_BY_CCOUNT(cache_disable_enable_time); + IDF_LOG_PERFORMANCE("Cache disable time", "cache_disable_enable_time: %"PRId32", cpu cycles, %"PRId32" us", cache_disable_enable_time, cache_disable_enable_time_us); + + TEST_ASSERT(cache_freeze_unfreeze_time < cache_disable_enable_time); +} +#endif // #if !CONFIG_IDF_TARGET_ESP32P4 +#endif // #if !CONFIG_FREERTOS_UNICORE