From 640d3917834eb03b698aa6affa70ff44a5a1a8cc Mon Sep 17 00:00:00 2001 From: gaoxiaojie Date: Mon, 17 Jul 2023 14:30:26 +0800 Subject: [PATCH] fix(rom_cache): use assembly to implement api cache_writeback_items_freeze ensure cache freeze -> Writeback enable -> wait done -> cache unfreeze routine never trigger a window overflow --- components/esp_rom/CMakeLists.txt | 4 + components/esp_rom/linker.lf | 2 + .../patches/esp_rom_cache_esp32s2_esp32s3.c | 27 +--- .../patches/esp_rom_cache_writeback_esp32s3.S | 132 ++++++++++++++++++ 4 files changed, 142 insertions(+), 23 deletions(-) create mode 100644 components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S diff --git a/components/esp_rom/CMakeLists.txt b/components/esp_rom/CMakeLists.txt index da42cce443..b73745b4f3 100644 --- a/components/esp_rom/CMakeLists.txt +++ b/components/esp_rom/CMakeLists.txt @@ -50,6 +50,10 @@ if(CONFIG_ESP_ROM_HAS_FLASH_COUNT_PAGES_BUG OR CONFIG_ESP_ROM_HAS_CACHE_WRITEBAC list(APPEND sources "patches/esp_rom_cache_esp32s2_esp32s3.c") endif() +if(CONFIG_ESP_ROM_HAS_CACHE_WRITEBACK_BUG) + list(APPEND sources "patches/esp_rom_cache_writeback_esp32s3.S") +endif() + idf_component_register(SRCS ${sources} INCLUDE_DIRS ${include_dirs} PRIV_REQUIRES ${private_required_comp} diff --git a/components/esp_rom/linker.lf b/components/esp_rom/linker.lf index e9cb68a742..99ddf6bdd4 100644 --- a/components/esp_rom/linker.lf +++ b/components/esp_rom/linker.lf @@ -4,6 +4,8 @@ entries: esp_rom_spiflash (noflash) if ESP_ROM_HAS_FLASH_COUNT_PAGES_BUG = y: esp_rom_cache_esp32s2_esp32s3 (noflash) + if ESP_ROM_HAS_CACHE_WRITEBACK_BUG = y: + esp_rom_cache_writeback_esp32s3 (noflash) if HEAP_TLSF_USE_ROM_IMPL = y && ESP_ROM_TLSF_CHECK_PATCH = y: esp_rom_tlsf (noflash) if SOC_SYSTIMER_SUPPORTED = y: diff --git a/components/esp_rom/patches/esp_rom_cache_esp32s2_esp32s3.c b/components/esp_rom/patches/esp_rom_cache_esp32s2_esp32s3.c index 9404397379..9866d652a8 100644 --- a/components/esp_rom/patches/esp_rom_cache_esp32s2_esp32s3.c +++ b/components/esp_rom/patches/esp_rom_cache_esp32s2_esp32s3.c @@ -90,27 +90,8 @@ extern void Cache_Freeze_DCache_Enable(cache_freeze_mode_t mode); #endif //#if ESP_ROM_HAS_CACHE_SUSPEND_WAITI_BUG #if ESP_ROM_HAS_CACHE_WRITEBACK_BUG -static void __attribute__((optimize("-O2"))) Cache_WriteBack_Items_Freeze(uint32_t addr, uint32_t items) -{ - /* Please do not modify this function, it must strictly follow the current execution sequence, - * otherwise it may cause unexpected errors. - */ - REG_WRITE(EXTMEM_DCACHE_SYNC_ADDR_REG, addr); - REG_SET_FIELD(EXTMEM_DCACHE_SYNC_SIZE_REG, EXTMEM_DCACHE_SYNC_SIZE, items); - - /*enable dcache freeze, mode = CACHE_FREEZE_ACK_BUSY*/ - REG_CLR_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_MODE); - REG_SET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_ENA); - while (!REG_GET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_DONE)); - - REG_SET_BIT(EXTMEM_DCACHE_SYNC_CTRL_REG, EXTMEM_DCACHE_WRITEBACK_ENA); - while(!REG_GET_BIT(EXTMEM_DCACHE_SYNC_CTRL_REG, EXTMEM_DCACHE_SYNC_DONE)); - - /*disable dcache freeze*/ - REG_CLR_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_ENA); - while (REG_GET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_DONE)); -} - +/* Defined in esp_rom_cache_writeback_esp32s3.S */ +extern void cache_writeback_items_freeze(uint32_t addr, uint32_t items); // renamed for patch extern int rom_Cache_WriteBack_Addr(uint32_t addr, uint32_t size); int Cache_WriteBack_Addr(uint32_t addr, uint32_t size) @@ -141,7 +122,7 @@ int Cache_WriteBack_Addr(uint32_t addr, uint32_t size) /*writeback start unaligned mem, one cacheline*/ irq_status = XTOS_SET_INTLEVEL(XCHAL_NMILEVEL);//mask all interrupts - Cache_WriteBack_Items_Freeze(start, 1); + cache_writeback_items_freeze(start, 1); XTOS_RESTORE_INTLEVEL(irq_status); if (size == 0) { @@ -157,7 +138,7 @@ int Cache_WriteBack_Addr(uint32_t addr, uint32_t size) /*writeback end unaligned mem, one cacheline*/ irq_status = XTOS_SET_INTLEVEL(XCHAL_NMILEVEL);//mask all interrupts - Cache_WriteBack_Items_Freeze(end, 1); + cache_writeback_items_freeze(end, 1); XTOS_RESTORE_INTLEVEL(irq_status); if (size == 0) { diff --git a/components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S b/components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S new file mode 100644 index 0000000000..9a742fe596 --- /dev/null +++ b/components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S @@ -0,0 +1,132 @@ +/* + * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include "soc/extmem_reg.h" + +/** + * @brief Write back the cache items of DCache, enable cache freeze during writeback. + * Operation will be done CACHE_LINE_SIZE aligned. + * If the region is not in DCache addr room, nothing will be done. + * Please do not call this function in your SDK application. + * @param uint32_t addr: start address to write back + * @param uint32_t items: cache lines to invalidate, items * cache_line_size should + * not exceed the bus address size(4MB) + * + * void cache_writeback_items_freeze(uint32_t addr, uint32_t items) +*/ + +/******************************************************************************* + +This function is a cache write-back function that works around the following +hardware errata on the ESP32-S3: + +- Core X manually triggers (via the EXTMEM_DCACHE_SYNC_CTRL_REG register) the +write-back of one or more cache lines. +- While the write-back is in progress, there are two scenarios that may cause +cache hit error. + - Core X enters the interrupt handler and access the same cache line + being written back. + - Core Y access the same cache line being written back. + +To workaround this errata, the following steps must be taken when manually +triggering a cache write-back: + +- Core X must disable interrupts so that it cannot be preempted +- Core X must freeze the cache (via the EXTMEM_DCACHE_FREEZE_REG register) to +prevent Core Y from accessing the same cache lines that are about to be written +back. +- Core X now triggers the cache write-back. During the write-back... + - If Core Y attempts the access any address in the cache region, Core Y will + busy wait until the cache is unfrozen. + - Core X must ensure that it does not access any address in the cache region, + otherwise Core X will busy wait thus causing a deadlock. +- After the write-back is complete, Core X unfreezes the cache, and reenables +interrupts. + +Notes: + +- Please do not modify this function, it must strictly follow the current execution +sequence, otherwise it may cause unexpected errors. +- This function is written in assmebly to ensure that the function itself never +accesses any cache address while the cache is frozen. Unexpected cache access +could occur if... + - the function triggers an window overflow onto a stack placed in PSRAM. + Thus, we only use two window panes (a0 to a8) in this function and trigger + all window overflows before freezing the cache. + - the function accesses literals/read-only variables placed in Flash. + +*******************************************************************************/ + + .align 4 + /* + Create dedicated literal pool for this function. Mostly used to store out + of range movi transformations. + */ + .literal_position + .global cache_writeback_items_freeze + .type cache_writeback_items_freeze, @function +cache_writeback_items_freeze: + entry sp, 32 + + /* REG_WRITE(EXTMEM_DCACHE_SYNC_ADDR_REG, addr); */ + movi a4, EXTMEM_DCACHE_SYNC_ADDR_REG + s32i a2, a4, 0 + /* REG_WRITE(EXTMEM_DCACHE_SYNC_SIZE_REG, items); */ + movi a4, EXTMEM_DCACHE_SYNC_SIZE_REG + s32i a3, a4, 0 + memw /* About to freeze the cache. Ensure all previous memory R/W are completed */ + + movi a2, EXTMEM_DCACHE_FREEZE_REG + movi a3, EXTMEM_DCACHE_SYNC_CTRL_REG + + /* + REG_CLR_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_MODE); + REG_SET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_ENA); + */ + l32i a4, a2, 0 /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */ + movi a5, ~(EXTMEM_DCACHE_FREEZE_MODE_M) + and a4, a4, a5 + movi a5, EXTMEM_DCACHE_FREEZE_ENA_M + or a4, a4, a5 + s32i a4, a2, 0 /* *(EXTMEM_DCACHE_FREEZE_REG) = a4 */ + + /* while (!REG_GET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_DONE)); */ + movi a5, EXTMEM_DCACHE_FREEZE_DONE_M +_wait_freeze_done: + l32i a4, a2, 0 /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */ + memw + bnone a4, a5, _wait_freeze_done + + /* REG_SET_BIT(EXTMEM_DCACHE_SYNC_CTRL_REG, EXTMEM_DCACHE_WRITEBACK_ENA); */ + l32i a4, a3, 0 /* a4 = *(EXTMEM_DCACHE_SYNC_CTRL_REG) */ + movi a5, EXTMEM_DCACHE_WRITEBACK_ENA_M + or a4, a4, a5 + s32i a4, a3, 0 /* *(EXTMEM_DCACHE_SYNC_CTRL_REG) = a4 */ + + /* while(!REG_GET_BIT(EXTMEM_DCACHE_SYNC_CTRL_REG, EXTMEM_DCACHE_SYNC_DONE)); */ + movi a5, EXTMEM_DCACHE_SYNC_DONE_M +_wait_writeback_done: + l32i a4, a3, 0 /* a4 = *(EXTMEM_DCACHE_SYNC_CTRL_REG) */ + memw + bnone a4, a5, _wait_writeback_done + + /* REG_CLR_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_ENA); */ + l32i a4, a2, 0 /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */ + movi a5, ~(EXTMEM_DCACHE_FREEZE_ENA_M) + and a4, a4, a5 + s32i a4, a2, 0 /* *(EXTMEM_DCACHE_FREEZE_REG) = a4 */ + + /* while (REG_GET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_DONE)); */ + movi a5, EXTMEM_DCACHE_FREEZE_DONE_M +_wait_unfreeze_done: + l32i a4, a2, 0 /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */ + memw + bany a4, a5, _wait_unfreeze_done + + retw + .size cache_writeback_items_freeze, . - cache_writeback_items_freeze