From d1db2df31670994a5ca60b5d2227f10bb77d9b16 Mon Sep 17 00:00:00 2001 From: baohongde Date: Wed, 1 Sep 2021 21:55:50 +0800 Subject: [PATCH] components/bt: High level interrupt in bluetooth components/os: Move ETS_T1_WDT_INUM, ETS_CACHEERR_INUM and ETS_DPORT_INUM to l5 interrupt components/os: high level interrupt(5) components/os: hli_api: meta queue: fix out of bounds access, check for overflow components/os: hli: don't spill registers, instead save them to a separate region Level 4 interrupt has a chance of preempting a window overflow or underflow exception. Therefore it is not possible to use standard context save functions, as the SP on entry to Level 4 interrupt may be invalid (e.g. in WindowUnderflow4). Instead, mask window overflows and save the entire general purpose register file, plus some of the special registers. Then clear WindowStart, allowing the C handler to execute without spilling the old windows. On exit from the interrupt handler, do everything in reverse. components/bt: using high level interrupt in lc components/os: Add DRAM_ATTR to avoid feature `Allow .bss segment placed in external memory` components/bt: optimize code structure components/os: Modify the BT assert process to adapt to coredump and HLI components/os: Disable exception mode after saving special registers To store some registers first, avoid stuck due to live lock after disabling exception mode components/os: using dport instead of AHB in BT to fix live lock components/bt: Fix hli queue send error components/bt: Fix CI fail # Conflicts: # components/bt/CMakeLists.txt # components/bt/component.mk # components/bt/controller/bt.c # components/bt/controller/lib # components/esp_common/src/int_wdt.c # components/esp_system/port/soc/esp32/dport_panic_highint_hdl.S # components/soc/esp32/include/soc/soc.h --- components/bt/CMakeLists.txt | 6 +- components/bt/component.mk | 1 + components/bt/controller/esp32/bt.c | 449 +++++++----------- components/bt/controller/esp32/hli_api.c | 294 ++++++++++++ components/bt/controller/esp32/hli_api.h | 160 +++++++ components/bt/controller/esp32/hli_vectors.S | 225 +++++++++ components/bt/controller/lib_esp32 | 2 +- .../src/esp_ipc_isr/esp_ipc_isr_handler.S | 42 +- components/esp_system/int_wdt.c | 10 +- .../port/arch/xtensa/panic_handler_asm.S | 2 +- .../esp_system/port/soc/esp32/highint_hdl.S | 390 ++++++++------- .../freertos/port/xtensa/xtensa_vectors.S | 6 +- components/soc/esp32/include/soc/soc.h | 21 +- 13 files changed, 1109 insertions(+), 499 deletions(-) create mode 100644 components/bt/controller/esp32/hli_api.c create mode 100644 components/bt/controller/esp32/hli_api.h create mode 100644 components/bt/controller/esp32/hli_vectors.S diff --git a/components/bt/CMakeLists.txt b/components/bt/CMakeLists.txt index 68e668e9d9..efacc806a1 100644 --- a/components/bt/CMakeLists.txt +++ b/components/bt/CMakeLists.txt @@ -1,6 +1,8 @@ if(CONFIG_BT_ENABLED) if(CONFIG_IDF_TARGET_ESP32) - set(srcs "controller/esp32/bt.c") + set(srcs "controller/esp32/bt.c" + "controller/esp32/hli_api.c" + "controller/esp32/hli_vectors.S") elseif(CONFIG_IDF_TARGET_ESP32C3) set(srcs "controller/esp32c3/bt.c") elseif(CONFIG_IDF_TARGET_ESP32S3) @@ -608,6 +610,8 @@ if(CONFIG_BT_ENABLED) if(CONFIG_IDF_TARGET_ESP32) target_link_libraries(${COMPONENT_LIB} INTERFACE "-L${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32/esp32") target_link_libraries(${COMPONENT_LIB} PUBLIC btdm_app) + + target_link_libraries(${COMPONENT_LIB} INTERFACE "-u ld_include_hli_vectors_bt") elseif(CONFIG_IDF_TARGET_ESP32C3) target_link_libraries(${COMPONENT_LIB} INTERFACE "-L${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32c3_family/esp32c3") diff --git a/components/bt/component.mk b/components/bt/component.mk index 1ba960e148..bd8d52371a 100644 --- a/components/bt/component.mk +++ b/components/bt/component.mk @@ -10,6 +10,7 @@ COMPONENT_ADD_INCLUDEDIRS := include LIBS := btdm_app COMPONENT_ADD_LDFLAGS := -lbt -L $(COMPONENT_PATH)/controller/lib_esp32/esp32 \ + -u ld_include_hli_vectors_bt \ $(addprefix -l,$(LIBS)) # re-link program if BT binary libs change diff --git a/components/bt/controller/esp32/bt.c b/components/bt/controller/esp32/bt.c index 8808618c81..995d9bf2ca 100644 --- a/components/bt/controller/esp32/bt.c +++ b/components/bt/controller/esp32/bt.c @@ -40,6 +40,7 @@ #include "driver/periph_ctrl.h" #include "soc/rtc.h" #include "soc/soc_memory_layout.h" +#include "soc/dport_reg.h" #include "esp32/clk.h" #include "esp_coexist_internal.h" #if !CONFIG_FREERTOS_UNICORE @@ -47,6 +48,7 @@ #endif #include "esp_rom_sys.h" +#include "hli_api.h" #if CONFIG_BT_ENABLED @@ -54,6 +56,7 @@ ************************************************************************ */ +#define UNUSED(x) (void)(x) #define BTDM_LOG_TAG "BTDM_INIT" #define BTDM_INIT_PERIOD (5000) /* ms */ @@ -92,14 +95,9 @@ do{\ } while(0) #define OSI_FUNCS_TIME_BLOCKING 0xffffffff -#define OSI_VERSION 0x00010002 +#define OSI_VERSION 0x00010003 #define OSI_MAGIC_VALUE 0xFADEBEAD -/* SPIRAM Configuration */ -#if CONFIG_SPIRAM_USE_MALLOC -#define BTDM_MAX_QUEUE_NUM (5) -#endif - /* Types definition ************************************************************************ */ @@ -117,15 +115,6 @@ typedef struct { intptr_t end; } btdm_dram_available_region_t; -/* PSRAM configuration */ -#if CONFIG_SPIRAM_USE_MALLOC -typedef struct { - QueueHandle_t handle; - void *storage; - void *buffer; -} btdm_queue_item_t; -#endif - /* OSI function */ struct osi_funcs_t { uint32_t _version; @@ -184,6 +173,10 @@ struct osi_funcs_t { void *(* _coex_schm_curr_phase_get)(void); int (* _coex_wifi_channel_get)(uint8_t *primary, uint8_t *secondary); int (* _coex_register_wifi_channel_change_callback)(void *cb); + xt_handler (*_set_isr_l3)(int n, xt_handler f, void *arg); + void (*_interrupt_l3_disable)(void); + void (*_interrupt_l3_restore)(void); + void *(* _customer_queue_create)(uint32_t queue_len, uint32_t item_size); uint32_t _magic; }; @@ -264,12 +257,10 @@ extern uint32_t _btdm_data_end; /* Local Function Declare ********************************************************************* */ -#if CONFIG_SPIRAM_USE_MALLOC -static bool btdm_queue_generic_register(const btdm_queue_item_t *queue); -static bool btdm_queue_generic_deregister(btdm_queue_item_t *queue); -#endif /* CONFIG_SPIRAM_USE_MALLOC */ -static void IRAM_ATTR interrupt_disable(void); -static void IRAM_ATTR interrupt_restore(void); +static xt_handler set_isr_hlevel_wrapper(int n, xt_handler f, void *arg); +static void IRAM_ATTR interrupt_hlevel_disable(void); +static void IRAM_ATTR interrupt_hlevel_restore(void); +static void IRAM_ATTR task_yield(void); static void IRAM_ATTR task_yield_from_isr(void); static void *semphr_create_wrapper(uint32_t max, uint32_t init); static void semphr_delete_wrapper(void *semphr); @@ -281,12 +272,12 @@ static void *mutex_create_wrapper(void); static void mutex_delete_wrapper(void *mutex); static int32_t mutex_lock_wrapper(void *mutex); static int32_t mutex_unlock_wrapper(void *mutex); -static void *queue_create_wrapper(uint32_t queue_len, uint32_t item_size); -static void queue_delete_wrapper(void *queue); -static int32_t queue_send_wrapper(void *queue, void *item, uint32_t block_time_ms); -static int32_t IRAM_ATTR queue_send_from_isr_wrapper(void *queue, void *item, void *hptw); -static int32_t queue_recv_wrapper(void *queue, void *item, uint32_t block_time_ms); -static int32_t IRAM_ATTR queue_recv_from_isr_wrapper(void *queue, void *item, void *hptw); +static void *queue_create_hlevel_wrapper(uint32_t queue_len, uint32_t item_size); +static void queue_delete_hlevel_wrapper(void *queue); +static int32_t IRAM_ATTR queue_send_hlevel_wrapper(void *queue, void *item, uint32_t block_time_ms); +static int32_t IRAM_ATTR queue_send_from_isr_hlevel_wrapper(void *queue, void *item, void *hptw); +static int32_t IRAM_ATTR queue_recv_hlevel_wrapper(void *queue, void *item, uint32_t block_time_ms); +static int32_t IRAM_ATTR queue_recv_from_isr_hlevel_wrapper(void *queue, void *item, void *hptw); static int32_t task_create_wrapper(void *task_func, const char *name, uint32_t stack_depth, void *param, uint32_t prio, void *task_handle, uint32_t core_id); static void task_delete_wrapper(void *task_handle); static bool IRAM_ATTR is_in_isr_wrapper(void); @@ -317,17 +308,21 @@ static uint8_t coex_schm_curr_period_get_wrapper(void); static void * coex_schm_curr_phase_get_wrapper(void); static int coex_wifi_channel_get_wrapper(uint8_t *primary, uint8_t *secondary); static int coex_register_wifi_channel_change_callback_wrapper(void *cb); +static void *customer_queue_create_hlevel_wrapper(uint32_t queue_len, uint32_t item_size); +static void IRAM_ATTR interrupt_l3_disable(void); +static void IRAM_ATTR interrupt_l3_restore(void); + /* Local variable definition *************************************************************************** */ /* OSI funcs */ static const struct osi_funcs_t osi_funcs_ro = { ._version = OSI_VERSION, - ._set_isr = xt_set_interrupt_handler, + ._set_isr = set_isr_hlevel_wrapper, ._ints_on = xt_ints_on, - ._interrupt_disable = interrupt_disable, - ._interrupt_restore = interrupt_restore, - ._task_yield = vPortYield, + ._interrupt_disable = interrupt_hlevel_disable, + ._interrupt_restore = interrupt_hlevel_restore, + ._task_yield = task_yield, ._task_yield_from_isr = task_yield_from_isr, ._semphr_create = semphr_create_wrapper, ._semphr_delete = semphr_delete_wrapper, @@ -339,12 +334,12 @@ static const struct osi_funcs_t osi_funcs_ro = { ._mutex_delete = mutex_delete_wrapper, ._mutex_lock = mutex_lock_wrapper, ._mutex_unlock = mutex_unlock_wrapper, - ._queue_create = queue_create_wrapper, - ._queue_delete = queue_delete_wrapper, - ._queue_send = queue_send_wrapper, - ._queue_send_from_isr = queue_send_from_isr_wrapper, - ._queue_recv = queue_recv_wrapper, - ._queue_recv_from_isr = queue_recv_from_isr_wrapper, + ._queue_create = queue_create_hlevel_wrapper, + ._queue_delete = queue_delete_hlevel_wrapper, + ._queue_send = queue_send_hlevel_wrapper, + ._queue_send_from_isr = queue_send_from_isr_hlevel_wrapper, + ._queue_recv = queue_recv_hlevel_wrapper, + ._queue_recv_from_isr = queue_recv_from_isr_hlevel_wrapper, ._task_create = task_create_wrapper, ._task_delete = task_delete_wrapper, ._is_in_isr = is_in_isr_wrapper, @@ -378,6 +373,10 @@ static const struct osi_funcs_t osi_funcs_ro = { ._coex_schm_curr_phase_get = coex_schm_curr_phase_get_wrapper, ._coex_wifi_channel_get = coex_wifi_channel_get_wrapper, ._coex_register_wifi_channel_change_callback = coex_register_wifi_channel_change_callback_wrapper, + ._set_isr_l3 = xt_set_interrupt_handler, + ._interrupt_l3_disable = interrupt_l3_disable, + ._interrupt_l3_restore = interrupt_l3_restore, + ._customer_queue_create = customer_queue_create_hlevel_wrapper, ._magic = OSI_MAGIC_VALUE, }; @@ -404,11 +403,6 @@ SOC_RESERVE_MEMORY_REGION(SOC_MEM_BT_DATA_START, SOC_MEM_BT_DATA_END, static DRAM_ATTR struct osi_funcs_t *osi_funcs_p; -#if CONFIG_SPIRAM_USE_MALLOC -static DRAM_ATTR btdm_queue_item_t btdm_queue_table[BTDM_MAX_QUEUE_NUM]; -static DRAM_ATTR SemaphoreHandle_t btdm_queue_table_mux = NULL; -#endif /* #if CONFIG_SPIRAM_USE_MALLOC */ - /* Static variable declare */ // timestamp when PHY/RF was switched on static DRAM_ATTR int64_t s_time_phy_rf_just_enabled = 0; @@ -448,53 +442,45 @@ static inline void btdm_check_and_init_bb(void) } } -#if CONFIG_SPIRAM_USE_MALLOC -static bool btdm_queue_generic_register(const btdm_queue_item_t *queue) -{ - if (!btdm_queue_table_mux || !queue) { - return NULL; - } +struct interrupt_hlevel_cb{ + uint32_t status; + uint8_t nested; +}; - bool ret = false; - btdm_queue_item_t *item; - xSemaphoreTake(btdm_queue_table_mux, portMAX_DELAY); - for (int i = 0; i < BTDM_MAX_QUEUE_NUM; ++i) { - item = &btdm_queue_table[i]; - if (item->handle == NULL) { - memcpy(item, queue, sizeof(btdm_queue_item_t)); - ret = true; - break; - } +static DRAM_ATTR struct interrupt_hlevel_cb hli_cb = { + .status = 0, + .nested = 0, +}; + +static xt_handler set_isr_hlevel_wrapper(int mask, xt_handler f, void *arg) +{ + esp_err_t err = hli_intr_register((intr_handler_t) f, arg, DPORT_PRO_INTR_STATUS_0_REG, mask); + if (err == ESP_OK) { + return f; + } else { + return 0; + } + } + +static void IRAM_ATTR interrupt_hlevel_disable(void) +{ + assert(xPortGetCoreID() == CONFIG_BTDM_CTRL_PINNED_TO_CORE); + uint32_t status = hli_intr_disable(); + if (hli_cb.nested++ == 0) { + hli_cb.status = status; } - xSemaphoreGive(btdm_queue_table_mux); - return ret; } -static bool btdm_queue_generic_deregister(btdm_queue_item_t *queue) +static void IRAM_ATTR interrupt_hlevel_restore(void) { - if (!btdm_queue_table_mux || !queue) { - return false; + assert(xPortGetCoreID() == CONFIG_BTDM_CTRL_PINNED_TO_CORE); + assert(hli_cb.nested > 0); + if (--hli_cb.nested == 0) { + hli_intr_restore(hli_cb.status); } - - bool ret = false; - btdm_queue_item_t *item; - xSemaphoreTake(btdm_queue_table_mux, portMAX_DELAY); - for (int i = 0; i < BTDM_MAX_QUEUE_NUM; ++i) { - item = &btdm_queue_table[i]; - if (item->handle == queue->handle) { - memcpy(queue, item, sizeof(btdm_queue_item_t)); - memset(item, 0, sizeof(btdm_queue_item_t)); - ret = true; - break; - } - } - xSemaphoreGive(btdm_queue_table_mux); - return ret; } -#endif /* CONFIG_SPIRAM_USE_MALLOC */ - -static void IRAM_ATTR interrupt_disable(void) +static void IRAM_ATTR interrupt_l3_disable(void) { if (xPortInIsrContext()) { portENTER_CRITICAL_ISR(&global_int_mux); @@ -503,7 +489,7 @@ static void IRAM_ATTR interrupt_disable(void) } } -static void IRAM_ATTR interrupt_restore(void) +static void IRAM_ATTR interrupt_l3_restore(void) { if (xPortInIsrContext()) { portEXIT_CRITICAL_ISR(&global_int_mux); @@ -512,6 +498,12 @@ static void IRAM_ATTR interrupt_restore(void) } } +static void IRAM_ATTR task_yield(void) +{ + vPortYield(); +} + + static void IRAM_ATTR task_yield_from_isr(void) { portYIELD_FROM_ISR(); @@ -519,148 +511,58 @@ static void IRAM_ATTR task_yield_from_isr(void) static void *semphr_create_wrapper(uint32_t max, uint32_t init) { -#if !CONFIG_SPIRAM_USE_MALLOC - return (void *)xSemaphoreCreateCounting(max, init); -#else - StaticQueue_t *queue_buffer = NULL; - QueueHandle_t handle = NULL; - - queue_buffer = heap_caps_malloc(sizeof(StaticQueue_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT); - if (!queue_buffer) { - goto error; - } - - handle = xSemaphoreCreateCountingStatic(max, init, queue_buffer); - if (!handle) { - goto error; - } - - btdm_queue_item_t item = { - .handle = handle, - .storage = NULL, - .buffer = queue_buffer, - }; - - if (!btdm_queue_generic_register(&item)) { - goto error; - } - return handle; - - error: - if (handle) { - vSemaphoreDelete(handle); - } - if (queue_buffer) { - free(queue_buffer); - } - - return NULL; -#endif + SemaphoreHandle_t downstream_semaphore = xSemaphoreCreateCounting(max, init); + assert(downstream_semaphore); + hli_queue_handle_t s_semaphore = hli_semaphore_create(max, downstream_semaphore); + assert(downstream_semaphore); + return s_semaphore; } static void semphr_delete_wrapper(void *semphr) { -#if !CONFIG_SPIRAM_USE_MALLOC - vSemaphoreDelete(semphr); -#else - btdm_queue_item_t item = { - .handle = semphr, - .storage = NULL, - .buffer = NULL, - }; + if (((hli_queue_handle_t)semphr)->downstream != NULL) { + vSemaphoreDelete(((hli_queue_handle_t)semphr)->downstream); + } - if (btdm_queue_generic_deregister(&item)) { - vSemaphoreDelete(item.handle); - free(item.buffer); - } - - return; -#endif + hli_queue_delete(semphr); } static int32_t IRAM_ATTR semphr_take_from_isr_wrapper(void *semphr, void *hptw) { - return (int32_t)xSemaphoreTakeFromISR(semphr, hptw); + return (int32_t)xSemaphoreTakeFromISR(((hli_queue_handle_t)semphr)->downstream, hptw); } static int32_t IRAM_ATTR semphr_give_from_isr_wrapper(void *semphr, void *hptw) { - return (int32_t)xSemaphoreGiveFromISR(semphr, hptw); + UNUSED(hptw); + assert(xPortGetCoreID() == CONFIG_BTDM_CTRL_PINNED_TO_CORE); + return hli_semaphore_give(semphr); } static int32_t semphr_take_wrapper(void *semphr, uint32_t block_time_ms) { + bool ret; if (block_time_ms == OSI_FUNCS_TIME_BLOCKING) { - return (int32_t)xSemaphoreTake(semphr, portMAX_DELAY); + ret = xSemaphoreTake(((hli_queue_handle_t)semphr)->downstream, portMAX_DELAY); } else { - return (int32_t)xSemaphoreTake(semphr, block_time_ms / portTICK_PERIOD_MS); + ret = xSemaphoreTake(((hli_queue_handle_t)semphr)->downstream, block_time_ms / portTICK_PERIOD_MS); } + return (int32_t)ret; } static int32_t semphr_give_wrapper(void *semphr) { - return (int32_t)xSemaphoreGive(semphr); + return (int32_t)xSemaphoreGive(((hli_queue_handle_t)semphr)->downstream); } static void *mutex_create_wrapper(void) { -#if CONFIG_SPIRAM_USE_MALLOC - StaticQueue_t *queue_buffer = NULL; - QueueHandle_t handle = NULL; - - queue_buffer = heap_caps_malloc(sizeof(StaticQueue_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT); - if (!queue_buffer) { - goto error; - } - - handle = xSemaphoreCreateMutexStatic(queue_buffer); - if (!handle) { - goto error; - } - - btdm_queue_item_t item = { - .handle = handle, - .storage = NULL, - .buffer = queue_buffer, - }; - - if (!btdm_queue_generic_register(&item)) { - goto error; - } - return handle; - - error: - if (handle) { - vSemaphoreDelete(handle); - } - if (queue_buffer) { - free(queue_buffer); - } - - return NULL; -#else return (void *)xSemaphoreCreateMutex(); -#endif } static void mutex_delete_wrapper(void *mutex) { -#if !CONFIG_SPIRAM_USE_MALLOC vSemaphoreDelete(mutex); -#else - btdm_queue_item_t item = { - .handle = mutex, - .storage = NULL, - .buffer = NULL, - }; - - if (btdm_queue_generic_deregister(&item)) { - vSemaphoreDelete(item.handle); - free(item.buffer); - } - - return; -#endif } static int32_t mutex_lock_wrapper(void *mutex) @@ -673,104 +575,74 @@ static int32_t mutex_unlock_wrapper(void *mutex) return (int32_t)xSemaphoreGive(mutex); } -static void *queue_create_wrapper(uint32_t queue_len, uint32_t item_size) +static void *queue_create_hlevel_wrapper(uint32_t queue_len, uint32_t item_size) { -#if CONFIG_SPIRAM_USE_MALLOC - StaticQueue_t *queue_buffer = NULL; - uint8_t *queue_storage = NULL; - QueueHandle_t handle = NULL; - - queue_buffer = heap_caps_malloc(sizeof(StaticQueue_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT); - if (!queue_buffer) { - goto error; - } - - queue_storage = heap_caps_malloc((queue_len*item_size), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT); - if (!queue_storage ) { - goto error; - } - - handle = xQueueCreateStatic(queue_len, item_size, queue_storage, queue_buffer); - if (!handle) { - goto error; - } - - btdm_queue_item_t item = { - .handle = handle, - .storage = queue_storage, - .buffer = queue_buffer, - }; - - if (!btdm_queue_generic_register(&item)) { - goto error; - } - - return handle; - - error: - if (handle) { - vQueueDelete(handle); - } - if (queue_storage) { - free(queue_storage); - } - if (queue_buffer) { - free(queue_buffer); - } - - return NULL; -#else - return (void *)xQueueCreate(queue_len, item_size); -#endif + QueueHandle_t downstream_queue = xQueueCreate(queue_len, item_size); + assert(downstream_queue); + hli_queue_handle_t queue = hli_queue_create(queue_len, item_size, downstream_queue); + assert(queue); + return queue; } -static void queue_delete_wrapper(void *queue) +static void *customer_queue_create_hlevel_wrapper(uint32_t queue_len, uint32_t item_size) { -#if !CONFIG_SPIRAM_USE_MALLOC - vQueueDelete(queue); -#else - btdm_queue_item_t item = { - .handle = queue, - .storage = NULL, - .buffer = NULL, - }; - - if (btdm_queue_generic_deregister(&item)) { - vQueueDelete(item.handle); - free(item.storage); - free(item.buffer); - } - - return; -#endif + QueueHandle_t downstream_queue = xQueueCreate(queue_len, item_size); + assert(downstream_queue); + hli_queue_handle_t queue = hli_customer_queue_create(queue_len, item_size, downstream_queue); + assert(queue); + return queue; } -static int32_t queue_send_wrapper(void *queue, void *item, uint32_t block_time_ms) +static void queue_delete_hlevel_wrapper(void *queue) +{ + if (((hli_queue_handle_t)queue)->downstream != NULL) { + vQueueDelete(((hli_queue_handle_t)queue)->downstream); + } + hli_queue_delete(queue); +} + +static int32_t queue_send_hlevel_wrapper(void *queue, void *item, uint32_t block_time_ms) { if (block_time_ms == OSI_FUNCS_TIME_BLOCKING) { - return (int32_t)xQueueSend(queue, item, portMAX_DELAY); + return (int32_t)xQueueSend(((hli_queue_handle_t)queue)->downstream, item, portMAX_DELAY); } else { - return (int32_t)xQueueSend(queue, item, block_time_ms / portTICK_PERIOD_MS); + return (int32_t)xQueueSend(((hli_queue_handle_t)queue)->downstream, item, block_time_ms / portTICK_PERIOD_MS); } } -static int32_t IRAM_ATTR queue_send_from_isr_wrapper(void *queue, void *item, void *hptw) +/** + * Queue send from isr + * @param queue The queue which will send to + * @param item The message which will be send + * @param hptw need do task yield or not + * @return send success or not + * There is an issue here: When the queue is full, it may reture true but it send fail to the queue, sometimes. + * But in Bluetooth controller's isr, We don't care about the return value. + * It only required tp send success when the queue is empty all the time. + * So, this function meets the requirement. + */ +static int32_t IRAM_ATTR queue_send_from_isr_hlevel_wrapper(void *queue, void *item, void *hptw) { - return (int32_t)xQueueSendFromISR(queue, item, hptw); + UNUSED(hptw); + assert(xPortGetCoreID() == CONFIG_BTDM_CTRL_PINNED_TO_CORE); + return hli_queue_put(queue, item); } -static int32_t queue_recv_wrapper(void *queue, void *item, uint32_t block_time_ms) +static int32_t queue_recv_hlevel_wrapper(void *queue, void *item, uint32_t block_time_ms) { + bool ret; if (block_time_ms == OSI_FUNCS_TIME_BLOCKING) { - return (int32_t)xQueueReceive(queue, item, portMAX_DELAY); + ret = (int32_t)xQueueReceive(((hli_queue_handle_t)queue)->downstream, item, portMAX_DELAY); } else { - return (int32_t)xQueueReceive(queue, item, block_time_ms / portTICK_PERIOD_MS); + ret =(int32_t)xQueueReceive(((hli_queue_handle_t)queue)->downstream, item, block_time_ms / portTICK_PERIOD_MS); } + + return ret; } -static int32_t IRAM_ATTR queue_recv_from_isr_wrapper(void *queue, void *item, void *hptw) +static int32_t IRAM_ATTR queue_recv_from_isr_hlevel_wrapper(void *queue, void *item, void *hptw) { - return (int32_t)xQueueReceiveFromISR(queue, item, hptw); + return (int32_t)xQueueReceiveFromISR(((hli_queue_handle_t)queue)->downstream, item, hptw); } static int32_t task_create_wrapper(void *task_func, const char *name, uint32_t stack_depth, void *param, uint32_t prio, void *task_handle, uint32_t core_id) @@ -1317,11 +1189,31 @@ esp_err_t esp_bt_mem_release(esp_bt_mode_t mode) return ESP_OK; } +static void hli_queue_setup_cb(void* arg) +{ + hli_queue_setup(); +} + +static void hli_queue_setup_pinned_to_core(int core_id) +{ +#if CONFIG_FREERTOS_UNICORE + hli_queue_setup_cb(NULL); +#else /* CONFIG_FREERTOS_UNICORE */ + if (xPortGetCoreID() == core_id) { + hli_queue_setup_cb(NULL); + } else { + esp_ipc_call(core_id, hli_queue_setup_cb, NULL); + } +#endif /* !CONFIG_FREERTOS_UNICORE */ +} + esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg) { esp_err_t err; uint32_t btdm_cfg_mask = 0; + hli_queue_setup_pinned_to_core(CONFIG_BTDM_CTRL_PINNED_TO_CORE); + //if all the bt available memory was already released, cannot initialize bluetooth controller if (btdm_dram_available_region[0].mode == ESP_BT_MODE_IDLE) { return ESP_ERR_INVALID_STATE; @@ -1362,14 +1254,6 @@ esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg) ESP_LOGI(BTDM_LOG_TAG, "BT controller compile version [%s]", btdm_controller_get_compile_version()); -#if CONFIG_SPIRAM_USE_MALLOC - btdm_queue_table_mux = xSemaphoreCreateMutex(); - if (btdm_queue_table_mux == NULL) { - return ESP_ERR_NO_MEM; - } - memset(btdm_queue_table, 0, sizeof(btdm_queue_item_t) * BTDM_MAX_QUEUE_NUM); -#endif - s_wakeup_req_sem = semphr_create_wrapper(1, 0); if (s_wakeup_req_sem == NULL) { err = ESP_ERR_NO_MEM; @@ -1515,12 +1399,6 @@ esp_err_t esp_bt_controller_deinit(void) semphr_delete_wrapper(s_wakeup_req_sem); s_wakeup_req_sem = NULL; -#if CONFIG_SPIRAM_USE_MALLOC - vSemaphoreDelete(btdm_queue_table_mux); - btdm_queue_table_mux = NULL; - memset(btdm_queue_table, 0, sizeof(btdm_queue_item_t) * BTDM_MAX_QUEUE_NUM); -#endif - free(osi_funcs_p); osi_funcs_p = NULL; @@ -1736,4 +1614,15 @@ esp_err_t esp_ble_scan_dupilcate_list_flush(void) return ESP_OK; } +/** + * This function re-write controller's function, + * As coredump can not show paramerters in function which is in a .a file. + * + * After coredump fixing this issue, just delete this function. + */ +void IRAM_ATTR r_assert(const char *condition, int param0, int param1, const char *file, int line) +{ + __asm__ __volatile__("ill\n"); +} + #endif /* CONFIG_BT_ENABLED */ diff --git a/components/bt/controller/esp32/hli_api.c b/components/bt/controller/esp32/hli_api.c new file mode 100644 index 0000000000..b4574495f7 --- /dev/null +++ b/components/bt/controller/esp32/hli_api.c @@ -0,0 +1,294 @@ +// Copyright 2020 Espressif Systems (Shanghai) PTE LTD +// All rights reserved. + +#include +#include "esp_log.h" +#include "esp_heap_caps.h" +#include "xtensa/core-macros.h" +#include "soc/dport_reg.h" +#include "hli_api.h" +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" + + +#define HLI_MAX_HANDLERS 4 + +typedef struct { + intr_handler_t handler; + void* arg; + uint32_t intr_reg; + uint32_t intr_mask; +} hli_handler_info_t; + +typedef struct { +#define CUSTOMER_TYPE_REQUEST (0) +#define CUSTOMER_TYPE_RELEASE (1) + struct { + uint32_t cb_type; + union { + int (* request)(uint32_t, uint32_t, uint32_t); + int (* release)(uint32_t); + } cb; + } customer_cb; + uint32_t arg0, arg1, arg2; +} customer_swisr_t; + +static void IRAM_ATTR customer_swisr_handle(customer_swisr_t *cus_swisr) +{ + if (cus_swisr->customer_cb.cb_type == CUSTOMER_TYPE_REQUEST) { + if (cus_swisr->customer_cb.cb.request != NULL) { + cus_swisr->customer_cb.cb.request(cus_swisr->arg0, cus_swisr->arg1, cus_swisr->arg2); + } + } else if(cus_swisr->customer_cb.cb_type == CUSTOMER_TYPE_RELEASE) { + if (cus_swisr->customer_cb.cb.release != NULL) { + cus_swisr->customer_cb.cb.release(cus_swisr->arg0); + } + } +} + +static DRAM_ATTR hli_handler_info_t s_hli_handlers[HLI_MAX_HANDLERS]; +// static const char* TAG = "hli_queue"; + +esp_err_t hli_intr_register(intr_handler_t handler, void* arg, uint32_t intr_reg, uint32_t intr_mask) +{ + for (hli_handler_info_t* hip = s_hli_handlers; + hip < s_hli_handlers + HLI_MAX_HANDLERS; + ++hip) { + if (hip->handler == NULL) { + hip->arg = arg; + hip->intr_reg = intr_reg; + hip->intr_mask = intr_mask; + hip->handler = handler; /* set last, indicates the entry as valid */ + return ESP_OK; + } + } + return ESP_ERR_NO_MEM; +} + +void IRAM_ATTR hli_c_handler(void) +{ + bool handled = false; + /* Iterate over registered interrupt handlers, + * and check if the expected mask is present in the interrupt status register. + */ + for (hli_handler_info_t* hip = s_hli_handlers; + hip < s_hli_handlers + HLI_MAX_HANDLERS; + ++hip) { + if (hip->handler == NULL) { + continue; + } + uint32_t reg = hip->intr_reg; + uint32_t val; + if (reg == 0) { /* special case for CPU internal interrupts */ + val = XTHAL_GET_INTERRUPT(); + } else { + /* "reg" might not be in DPORT, but this will work in any case */ + val = DPORT_REG_READ(reg); + } + if ((val & hip->intr_mask) != 0) { + handled = true; + (*hip->handler)(hip->arg); + } + } + if (!handled) { + // esp_rom_printf(DRAM_STR("hli_c_handler: no handler found!\n")); + // abort(); + } +} + +uint32_t IRAM_ATTR hli_intr_disable(void) +{ + // disable level 4 and below + return XTOS_SET_INTLEVEL(XCHAL_DEBUGLEVEL - 2); +} + +void IRAM_ATTR hli_intr_restore(uint32_t state) +{ + XTOS_RESTORE_JUST_INTLEVEL(state); +} + +#define HLI_META_QUEUE_SIZE 16 +#define HLI_QUEUE_MAX_ELEM_SIZE 32 +#define HLI_QUEUE_SW_INT_NUM 29 + +#define HLI_QUEUE_FLAG_SEMAPHORE BIT(0) +#define HLI_QUEUE_FLAG_CUSTOMER BIT(1) + +static DRAM_ATTR struct hli_queue_t *s_meta_queue_ptr = NULL; +intr_handle_t ret_handle; + +static inline char* IRAM_ATTR wrap_ptr(hli_queue_handle_t queue, char *ptr) +{ + return (ptr == queue->bufend) ? queue->buf : ptr; +} + +static inline bool IRAM_ATTR queue_empty(hli_queue_handle_t queue) +{ + return queue->begin == queue->end; +} + +static inline bool IRAM_ATTR queue_full(hli_queue_handle_t queue) +{ + return wrap_ptr(queue, queue->end + queue->elem_size) == queue->begin; +} + +static void IRAM_ATTR queue_isr_handler(void* arg) +{ + int do_yield = pdFALSE; + XTHAL_SET_INTCLEAR(BIT(HLI_QUEUE_SW_INT_NUM)); + hli_queue_handle_t queue; + + while (hli_queue_get(s_meta_queue_ptr, &queue)) { + static DRAM_ATTR char scratch[HLI_QUEUE_MAX_ELEM_SIZE]; + while (hli_queue_get(queue, scratch)) { + int res = pdPASS; + if ((queue->flags & HLI_QUEUE_FLAG_CUSTOMER) != 0) { + customer_swisr_handle((customer_swisr_t *)scratch); + } else if ((queue->flags & HLI_QUEUE_FLAG_SEMAPHORE) != 0) { + res = xSemaphoreGiveFromISR((SemaphoreHandle_t) queue->downstream, &do_yield); + } else { + res = xQueueSendFromISR(queue->downstream, scratch, &do_yield); + } + if (res == pdFAIL) { + // ESP_EARLY_LOGE(TAG, "Failed to send to %s %p", (queue->flags & HLI_QUEUE_FLAG_SEMAPHORE) == 0 ? "queue" : "semaphore", queue->downstream); + } + } + } + if (do_yield) { + portYIELD_FROM_ISR(); + } +} + +/* Notify the level 3 handler that an element is added to the given hli queue. + * Do this by placing the queue handle onto s_meta_queue, and raising a SW interrupt. + * + * This function must be called with HL interrupts disabled! + */ +static void IRAM_ATTR queue_signal(hli_queue_handle_t queue) +{ + /* See if the queue is already in s_meta_queue, before adding */ + bool found = false; + const hli_queue_handle_t *end = (hli_queue_handle_t*) s_meta_queue_ptr->end; + hli_queue_handle_t *item = (hli_queue_handle_t*) s_meta_queue_ptr->begin; + for (;item != end; item = (hli_queue_handle_t*) wrap_ptr(s_meta_queue_ptr, (char*) (item + 1))) { + if (*item == queue) { + found = true; + break; + } + } + if (!found) { + bool res = hli_queue_put(s_meta_queue_ptr, &queue); + if (!res) { + esp_rom_printf(DRAM_STR("Fatal error in queue_signal: s_meta_queue full\n")); + abort(); + } + XTHAL_SET_INTSET(BIT(HLI_QUEUE_SW_INT_NUM)); + } +} + +static void queue_init(hli_queue_handle_t queue, size_t buf_size, size_t elem_size, QueueHandle_t downstream) +{ + queue->elem_size = elem_size; + queue->begin = queue->buf; + queue->end = queue->buf; + queue->bufend = queue->buf + buf_size; + queue->downstream = downstream; + queue->flags = 0; +} + +void hli_queue_setup(void) +{ + if (s_meta_queue_ptr == NULL) { + s_meta_queue_ptr = hli_queue_create(HLI_META_QUEUE_SIZE, sizeof(void*), NULL); + ESP_ERROR_CHECK(esp_intr_alloc(ETS_INTERNAL_SW1_INTR_SOURCE, ESP_INTR_FLAG_IRAM, queue_isr_handler, NULL, &ret_handle)); + xt_ints_on(BIT(HLI_QUEUE_SW_INT_NUM)); + } +} + +void hli_queue_shutdown(void) +{ + if (s_meta_queue_ptr != NULL) { + hli_queue_delete(s_meta_queue_ptr); + s_meta_queue_ptr = NULL; + esp_intr_free(ret_handle); + xt_ints_off(BIT(HLI_QUEUE_SW_INT_NUM)); + } +} + +hli_queue_handle_t hli_queue_create(size_t nelem, size_t elem_size, QueueHandle_t downstream) +{ + const size_t buf_elem = nelem + 1; + if (elem_size > HLI_QUEUE_MAX_ELEM_SIZE) { + return NULL; + } + size_t buf_size = buf_elem * elem_size; + hli_queue_handle_t res = (hli_queue_handle_t) heap_caps_malloc(sizeof(*res) + buf_size, + MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT); + if (res == NULL) { + return NULL; + } + queue_init(res, buf_size, elem_size, downstream); + return res; +} + +hli_queue_handle_t hli_customer_queue_create(size_t nelem, size_t elem_size, QueueHandle_t downstream) +{ + hli_queue_handle_t res = hli_queue_create(nelem, elem_size, (QueueHandle_t) downstream); + if (res == NULL) { + return NULL; + } + res->flags |= HLI_QUEUE_FLAG_CUSTOMER; + return res; +} + +hli_queue_handle_t hli_semaphore_create(size_t max_count, SemaphoreHandle_t downstream) +{ + const size_t elem_size = 1; + hli_queue_handle_t res = hli_queue_create(max_count, elem_size, (QueueHandle_t) downstream); + if (res == NULL) { + return NULL; + } + res->flags |= HLI_QUEUE_FLAG_SEMAPHORE; + return res; +} + +void hli_queue_delete(hli_queue_handle_t queue) +{ + free(queue); +} + +bool IRAM_ATTR hli_queue_get(hli_queue_handle_t queue, void* out) +{ + uint32_t int_state = hli_intr_disable(); + bool res = false; + if (!queue_empty(queue)) { + memcpy(out, queue->begin, queue->elem_size); + queue->begin = wrap_ptr(queue, queue->begin + queue->elem_size); + res = true; + } + hli_intr_restore(int_state); + return res; +} + +bool IRAM_ATTR hli_queue_put(hli_queue_handle_t queue, const void* data) +{ + uint32_t int_state = hli_intr_disable(); + bool res = false; + bool was_empty = queue_empty(queue); + if (!queue_full(queue)) { + memcpy(queue->end, data, queue->elem_size); + queue->end = wrap_ptr(queue, queue->end + queue->elem_size); + if (was_empty && queue != s_meta_queue_ptr) { + queue_signal(queue); + } + res = true; + } + hli_intr_restore(int_state); + return res; +} + +bool IRAM_ATTR hli_semaphore_give(hli_queue_handle_t queue) +{ + uint8_t data = 0; + return hli_queue_put(queue, &data); +} diff --git a/components/bt/controller/esp32/hli_api.h b/components/bt/controller/esp32/hli_api.h new file mode 100644 index 0000000000..35af59bbcf --- /dev/null +++ b/components/bt/controller/esp32/hli_api.h @@ -0,0 +1,160 @@ +// Copyright 2020 Espressif Systems (Shanghai) PTE LTD +// All rights reserved. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "esp_err.h" +#include "esp_intr_alloc.h" +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/semphr.h" + +/*** Queues ***/ + +struct hli_queue_t +{ + size_t elem_size; + char* begin; + char* end; + const char* bufend; + QueueHandle_t downstream; + int flags; + char buf[0]; +}; + +/** + * @brief Register a high level interrupt function + * + * @param handler interrupt handler function + * @param arg argument to pass to the interrupt handler + * @param intr_reg address of the peripheral register containing the interrupt status, + * or value 0 to get the status from CPU INTERRUPT register + * @param intr_mask mask of the interrupt, in the interrupt status register + * @return + * - ESP_OK on success + * - ESP_ERR_NO_MEM if too many handlers are registered + */ +esp_err_t hli_intr_register(intr_handler_t handler, void* arg, uint32_t intr_reg, uint32_t intr_mask); + +/** + * @brief Mask all interrupts (including high level ones) on the current CPU + * + * @return uint32_t interrupt status, pass it to hli_intr_restore + */ +uint32_t hli_intr_disable(void); + +/** + * @brief Re-enable interrupts + * + * @param state value returned by hli_intr_disable + */ +void hli_intr_restore(uint32_t state); + +/** + * @brief Type of a hli queue + */ +typedef struct hli_queue_t* hli_queue_handle_t; + +/** + * @brief Initialize hli_queue module. Must be called once before using hli queue APIs. + */ +void hli_queue_setup(void); + +/** + * @brief Shutdown hli_queue module. + */ +void hli_queue_shutdown(void); + +/** + * @brief Create a hli queue, wrapping a FreeRTOS queue + * + * This queue can be used from high level interrupts, + * but **ONLY ON THE CPU WHERE hli_queue_setup WAS CALLED**. Values sent to this + * queue are automatically forwarded to "downstream" FreeRTOS queue using a level 3 + * software interrupt. + * + * @param nelem number of elements in the queue + * @param elem_size size of one element; must match element size of a downstream queue + * @param downstream FreeRTOS queue to send the values to + * @return hli_queue_handle_t handle of the created queue, or NULL on failure + */ +hli_queue_handle_t hli_queue_create(size_t nelem, size_t elem_size, QueueHandle_t downstream); + +/** + * @brief Create a customer hli queue, wrapping a FreeRTOS queue + * + * This queue can be used from high level interrupts, + * but **ONLY ON THE CPU WHERE hli_queue_setup WAS CALLED**. Values sent to this + * queue are automatically forwarded to "downstream" FreeRTOS queue using a level 3 + * software interrupt. + * + * @param nelem number of elements in the queue + * @param elem_size size of one element; must match element size of a downstream queue + * @param downstream FreeRTOS queue to send the values to + * @return hli_queue_handle_t handle of the created queue, or NULL on failure + */ +hli_queue_handle_t hli_customer_queue_create(size_t nelem, size_t elem_size, QueueHandle_t downstream); + +/** + * @brief Create a hli queue, wrapping a FreeRTOS semaphore + * + * See notes on hli_queue_create. + * + * @param max_count maximum semaphore count + * @param downstream FreeRTOS semaphore to forward the calls to + * @return hli_queue_handle_t handle of the created queue, or NULL on failure + */ +hli_queue_handle_t hli_semaphore_create(size_t max_count, SemaphoreHandle_t downstream); + +/** + * @brief Delete a hli queue + * + * Make sure noone is using the queue before deleting it. + * + * @param queue handle returned by hli_queue_create or hli_semaphore_create + */ +void hli_queue_delete(hli_queue_handle_t queue); + +/** + * @brief Get one element from a hli queue + * + * Usually not used, values get sent to a downstream FreeRTOS queue automatically. + * However if downstream queue is NULL, this API can be used to get values from a hli queue. + * + * @param queue handle of a queue + * @param out pointer where to store the element + * @return true if the element was successfully read from the queue + */ +bool hli_queue_get(hli_queue_handle_t queue, void* out); + +/** + * @brief Put one element into a hli queue + * + * This puts copies an element into the queue and raises a software interrupt (level 3). + * In the interrupt, the value is copied to a FreeRTOS "downstream" queue. + * + * Note that if the value does not fit into a downstream queue, no error is returned, + * and the value is lost. + * + * @param queue handle of a queue + * @param data pointer to the element to be sent + * @return true if data was placed into the hli queue successfully + */ +bool hli_queue_put(hli_queue_handle_t queue, const void* data); + +/** + * @brief "Give" a semaphore wrapped by a hli queue + * + * @param queue handle returned by hli_semaphore_create + * @return true if the event was sent to a hli queue successfully + */ +bool hli_semaphore_give(hli_queue_handle_t queue); + +#ifdef __cplusplus +} +#endif diff --git a/components/bt/controller/esp32/hli_vectors.S b/components/bt/controller/esp32/hli_vectors.S new file mode 100644 index 0000000000..fabf836a12 --- /dev/null +++ b/components/bt/controller/esp32/hli_vectors.S @@ -0,0 +1,225 @@ +// Copyright 2020 Espressif Systems (Shanghai) PTE LTD +// All rights reserved. + +#include +#include +#include +#include "freertos/xtensa_context.h" +#include "sdkconfig.h" +#include "soc/soc.h" + +/* Interrupt stack size, for C code. + * TODO: reduce and make configurable. + */ +#define L4_INTR_STACK_SIZE 4096 + +/* Save area for the CPU state: + * - 64 words for the general purpose registers + * - 7 words for some of the special registers: + * - WINDOWBASE, WINDOWSTART — only WINDOWSTART is truly needed + * - SAR, LBEG, LEND, LCOUNT — since the C code might use these + * - EPC1 — since the C code might cause window overflow exceptions + * This is not laid out a standard exception frame structure + * for simplicity of the save/restore code. + */ +#define REG_FILE_SIZE (64 * 4) +#define SPECREG_OFFSET REG_FILE_SIZE +#define SPECREG_SIZE (7 * 4) +#define REG_SAVE_AREA_SIZE (SPECREG_OFFSET * SPECREG_SIZE) + + .data +_l4_intr_stack: + .space L4_INTR_STACK_SIZE +_l4_save_ctx: + .space REG_SAVE_AREA_SIZE + + .section .iram1,"ax" + .global xt_highint4 + .type xt_highint4,@function + .align 4 + +xt_highint4: + movi a0, _l4_save_ctx + /* save 4 lower registers */ + s32i a1, a0, 4 + s32i a2, a0, 8 + s32i a3, a0, 12 + rsr a2, EXCSAVE_4 /* holds the value of a0 */ + s32i a2, a0, 0 + + /* Save special registers */ + addi a0, a0, SPECREG_OFFSET + rsr a2, WINDOWBASE + s32i a2, a0, 0 + rsr a2, WINDOWSTART + s32i a2, a0, 4 + rsr a2, SAR + s32i a2, a0, 8 + rsr a2, LBEG + s32i a2, a0, 12 + rsr a2, LEND + s32i a2, a0, 16 + rsr a2, LCOUNT + s32i a2, a0, 20 + rsr a2, EPC1 + s32i a2, a0, 24 + + /* disable exception mode, window overflow */ + movi a0, PS_INTLEVEL(5) | PS_EXCM /*TOCHECK*/ + wsr a0, PS + rsync + + /* Save the remaining physical registers. + * 4 registers are already saved, which leaves 60 registers to save. + * (FIXME: consider the case when the CPU is configured with physical 32 registers) + * These 60 registers are saved in 5 iterations, 12 registers at a time. + */ + movi a1, 5 + movi a3, _l4_save_ctx + 4 * 4 + + /* This is repeated 5 times, each time the window is shifted by 12 registers. + * We come here with a1 = downcounter, a3 = save pointer, a2 and a0 unused. + */ +1: + s32i a4, a3, 0 + s32i a5, a3, 4 + s32i a6, a3, 8 + s32i a7, a3, 12 + s32i a8, a3, 16 + s32i a9, a3, 20 + s32i a10, a3, 24 + s32i a11, a3, 28 + s32i a12, a3, 32 + s32i a13, a3, 36 + s32i a14, a3, 40 + s32i a15, a3, 44 + + /* We are about to rotate the window, so that a12-a15 will become the new a0-a3. + * Copy a0-a3 to a12-15 to still have access to these values. + * At the same time we can decrement the counter and adjust the save area pointer + */ + + /* a0 is constant (_l4_save_ctx), no need to copy */ + addi a13, a1, -1 /* copy and decrement the downcounter */ + /* a2 is scratch so no need to copy */ + addi a15, a3, 48 /* copy and adjust the save area pointer */ + beqz a13, 2f /* have saved all registers ? */ + rotw 3 /* rotate the window and go back */ + j 1b + + /* the loop is complete */ +2: + rotw 4 /* this brings us back to the original window */ + /* a0 still points to _l4_save_ctx */ + + /* Can clear WINDOWSTART now, all registers are saved */ + rsr a2, WINDOWBASE + /* WINDOWSTART = (1 << WINDOWBASE) */ + movi a3, 1 + ssl a2 + sll a3, a3 + wsr a3, WINDOWSTART + +_highint4_stack_switch: + movi a0, 0 + movi sp, _l4_intr_stack + L4_INTR_STACK_SIZE - 16 + s32e a0, sp, -12 /* For GDB: set null SP */ + s32e a0, sp, -16 /* For GDB: set null PC */ + movi a0, _highint4_stack_switch /* For GDB: cosmetics, for the frame where stack switch happened */ + + /* Set up PS for C, disable all interrupts except NMI and debug, and clear EXCM. */ + movi a6, PS_INTLEVEL(4) | PS_UM | PS_WOE + wsr a6, PS + rsync + + /* Call C handler */ + mov a6, sp + call4 hli_c_handler + + l32e sp, sp, -12 /* switch back to the original stack */ + + /* Done with C handler; re-enable exception mode, disabling window overflow */ + movi a2, PS_INTLEVEL(5) | PS_EXCM /* TOCHECK */ + wsr a2, PS + rsync + + /* Restore the special registers. + * WINDOWSTART will be restored near the end. + */ + movi a0, _l4_save_ctx + SPECREG_OFFSET + l32i a2, a0, 8 + wsr a2, SAR + l32i a2, a0, 12 + wsr a2, LBEG + l32i a2, a0, 16 + wsr a2, LEND + l32i a2, a0, 20 + wsr a2, LCOUNT + l32i a2, a0, 24 + wsr a2, EPC1 + + /* Restoring the physical registers. + * This is the reverse to the saving process above. + */ + + /* Rotate back to the final window, then start loading 12 registers at a time, + * in 5 iterations. + * Again, a1 is the downcounter and a3 is the save area pointer. + * After each rotation, a1 and a3 are copied from a13 and a15. + * To simplify the loop, we put the initial values into a13 and a15. + */ + rotw -4 + movi a15, _l4_save_ctx + 64 * 4 /* point to the end of the save area */ + movi a13, 5 + +1: + /* Copy a1 and a3 from their previous location, + * at the same time decrementing and adjusting the save area pointer. + */ + addi a1, a13, -1 + addi a3, a15, -48 + + /* Load 12 registers */ + l32i a4, a3, 0 + l32i a5, a3, 4 + l32i a6, a3, 8 + l32i a7, a3, 12 + l32i a8, a3, 16 + l32i a9, a3, 20 + l32i a10, a3, 24 + l32i a11, a3, 28 /* ensure PS and EPC written */ + l32i a12, a3, 32 + l32i a13, a3, 36 + l32i a14, a3, 40 + l32i a15, a3, 44 + + /* Done with the loop? */ + beqz a1, 2f + /* If no, rotate the window and repeat */ + rotw -3 + j 1b + +2: + /* Done with the loop. Only 4 registers (a0-a3 in the original window) remain + * to be restored. Also need to restore WINDOWSTART, since all the general + * registers are now in place. + */ + movi a0, _l4_save_ctx + + l32i a2, a0, SPECREG_OFFSET + 4 + wsr a2, WINDOWSTART + + l32i a1, a0, 4 + l32i a2, a0, 8 + l32i a3, a0, 12 + rsr a0, EXCSAVE_4 /* holds the value of a0 before the interrupt handler */ + + /* Return from the interrupt, restoring PS from EPS_4 */ + rfi 4 + +/* The linker has no reason to link in this file; all symbols it exports are already defined + (weakly!) in the default int handler. Define a symbol here so we can use it to have the + linker inspect this anyway. */ + + .global ld_include_hli_vectors_bt +ld_include_hli_vectors_bt: diff --git a/components/bt/controller/lib_esp32 b/components/bt/controller/lib_esp32 index fb49791b7c..c9748123de 160000 --- a/components/bt/controller/lib_esp32 +++ b/components/bt/controller/lib_esp32 @@ -1 +1 @@ -Subproject commit fb49791b7c1a8a35f06e68124c90022667b4cff1 +Subproject commit c9748123def947a24964c3882d55af75b7762460 diff --git a/components/esp_ipc/src/esp_ipc_isr/esp_ipc_isr_handler.S b/components/esp_ipc/src/esp_ipc_isr/esp_ipc_isr_handler.S index c58ee7567e..292bbc50dd 100644 --- a/components/esp_ipc/src/esp_ipc_isr/esp_ipc_isr_handler.S +++ b/components/esp_ipc/src/esp_ipc_isr/esp_ipc_isr_handler.S @@ -15,14 +15,14 @@ /* High-priority interrupt - IPC_ISR handler */ -#define L4_INTR_STACK_SIZE 16 -#define L4_INTR_A0_OFFSET 0 -#define L4_INTR_A2_OFFSET 4 -#define L4_INTR_A3_OFFSET 8 -#define L4_INTR_A4_OFFSET 12 +#define L5_INTR_STACK_SIZE 16 +#define L5_INTR_A0_OFFSET 0 +#define L5_INTR_A2_OFFSET 4 +#define L5_INTR_A3_OFFSET 8 +#define L5_INTR_A4_OFFSET 12 .data -_l4_intr_stack: - .space L4_INTR_STACK_SIZE +_l5_intr_stack: + .space L5_INTR_STACK_SIZE .section .iram1,"ax" .global esp_ipc_isr_handler .type esp_ipc_isr_handler,@function @@ -32,15 +32,15 @@ esp_ipc_isr_handler: /* Because the interrupt cause code has protection that only allows one cpu to enter in the IPC_ISR section of the L4 interrupt at one time, there's no need to have two - _l4_intr_stack for each cpu */ + _l5_intr_stack for each cpu */ /* Save A0, A2, A3, A4 so we can use those registers further*/ - movi a0, _l4_intr_stack - s32i a2, a0, L4_INTR_A2_OFFSET - s32i a3, a0, L4_INTR_A3_OFFSET - s32i a4, a0, L4_INTR_A4_OFFSET - rsr a2, EXCSAVE_4 - s32i a2, a0, L4_INTR_A0_OFFSET + movi a0, _l5_intr_stack + s32i a2, a0, L5_INTR_A2_OFFSET + s32i a3, a0, L5_INTR_A3_OFFSET + s32i a4, a0, L5_INTR_A4_OFFSET + rsr a2, EXCSAVE_5 + s32i a2, a0, L5_INTR_A0_OFFSET /* disable nested iterrupts */ /* PS.EXCM is changed from 1 to 0 . It allows using usually exception handler instead of the Double exception handler. */ @@ -85,16 +85,16 @@ esp_ipc_isr_handler: callx0 a0 /* Done. Restore registers and return. */ - movi a0, _l4_intr_stack - l32i a2, a0, L4_INTR_A2_OFFSET - l32i a3, a0, L4_INTR_A3_OFFSET - l32i a4, a0, L4_INTR_A4_OFFSET + movi a0, _l5_intr_stack + l32i a2, a0, L5_INTR_A2_OFFSET + l32i a3, a0, L5_INTR_A3_OFFSET + l32i a4, a0, L5_INTR_A4_OFFSET /* set the end flag */ movi a0, esp_ipc_isr_end_fl s32i a0, a0, 0 /* restore a0 */ - rsr a0, EXCSAVE_4 - /* restores PS from EPS[4] and jumps to the address in EPC[4] */ - rfi 4 + rsr a0, EXCSAVE_5 + /* restores PS from EPS[5] and jumps to the address in EPC[5] */ + rfi 5 diff --git a/components/esp_system/int_wdt.c b/components/esp_system/int_wdt.c index da2f880cc1..bda5761bd7 100644 --- a/components/esp_system/int_wdt.c +++ b/components/esp_system/int_wdt.c @@ -51,7 +51,7 @@ static wdt_hal_context_t iwdt_context; */ #define IWDT_LIVELOCK_TIMEOUT_MS (20) -extern uint32_t _l4_intr_livelock_counter, _l4_intr_livelock_max; +extern uint32_t _l5_intr_livelock_counter, _l5_intr_livelock_max; #endif //Take care: the tick hook can also be called before esp_int_wdt_init() is called. @@ -70,9 +70,9 @@ static void IRAM_ATTR tick_hook(void) wdt_hal_write_protect_disable(&iwdt_context); //Reconfigure stage timeouts #if CONFIG_ESP32_ECO3_CACHE_LOCK_FIX - _l4_intr_livelock_counter = 0; + _l5_intr_livelock_counter = 0; wdt_hal_config_stage(&iwdt_context, WDT_STAGE0, - CONFIG_ESP_INT_WDT_TIMEOUT_MS * 1000 / IWDT_TICKS_PER_US / (_l4_intr_livelock_max + 1), WDT_STAGE_ACTION_INT); //Set timeout before interrupt + CONFIG_ESP_INT_WDT_TIMEOUT_MS * 1000 / IWDT_TICKS_PER_US / (_l5_intr_livelock_max + 1), WDT_STAGE_ACTION_INT); //Set timeout before interrupt #else wdt_hal_config_stage(&iwdt_context, WDT_STAGE0, CONFIG_ESP_INT_WDT_TIMEOUT_MS * 1000 / IWDT_TICKS_PER_US, WDT_STAGE_ACTION_INT); //Set timeout before interrupt #endif @@ -136,11 +136,11 @@ void esp_int_wdt_cpu_init(void) * This is a workaround for issue 3.15 in "ESP32 ECO and workarounds for * Bugs" document. */ - _l4_intr_livelock_counter = 0; + _l5_intr_livelock_counter = 0; if (soc_has_cache_lock_bug()) { assert((portTICK_PERIOD_MS << 1) <= IWDT_LIVELOCK_TIMEOUT_MS); assert(CONFIG_ESP_INT_WDT_TIMEOUT_MS >= (IWDT_LIVELOCK_TIMEOUT_MS * 3)); - _l4_intr_livelock_max = CONFIG_ESP_INT_WDT_TIMEOUT_MS / IWDT_LIVELOCK_TIMEOUT_MS - 1; + _l5_intr_livelock_max = CONFIG_ESP_INT_WDT_TIMEOUT_MS / IWDT_LIVELOCK_TIMEOUT_MS - 1; } #endif diff --git a/components/esp_system/port/arch/xtensa/panic_handler_asm.S b/components/esp_system/port/arch/xtensa/panic_handler_asm.S index b0a5754271..3328f55537 100644 --- a/components/esp_system/port/arch/xtensa/panic_handler_asm.S +++ b/components/esp_system/port/arch/xtensa/panic_handler_asm.S @@ -54,7 +54,7 @@ _xt_panic: s32i a0, sp, XT_STK_A0 /* Set up PS for C, disable all interrupts except NMI and debug, and clear EXCM. */ - movi a0, PS_INTLEVEL(5) | PS_UM | PS_WOE + movi a0, PS_INTLEVEL(XCHAL_DEBUGLEVEL - 2) | PS_UM | PS_WOE wsr a0, PS //Call panic handler diff --git a/components/esp_system/port/soc/esp32/highint_hdl.S b/components/esp_system/port/soc/esp32/highint_hdl.S index 02316f612d..e9b59afd9f 100644 --- a/components/esp_system/port/soc/esp32/highint_hdl.S +++ b/components/esp_system/port/soc/esp32/highint_hdl.S @@ -33,156 +33,10 @@ Interrupt , a high-priority interrupt, is used for several things: */ -#define L4_INTR_STACK_SIZE 12 -#define L4_INTR_A2_OFFSET 0 -#define L4_INTR_A3_OFFSET 4 -#define L4_INTR_A4_OFFSET 8 - .data -_l4_intr_stack: - .space L4_INTR_STACK_SIZE*portNUM_PROCESSORS /* This allocates stacks for each individual CPU. */ - -#if CONFIG_ESP32_ECO3_CACHE_LOCK_FIX && CONFIG_ESP_INT_WDT - .global _l4_intr_livelock_counter - .global _l4_intr_livelock_max - .align 16 -_l4_intr_livelock_counter: - .word 0 -_l4_intr_livelock_max: - .word 0 -_l4_intr_livelock_sync: - .word 0, 0 -_l4_intr_livelock_app: - .word 0 -_l4_intr_livelock_pro: - .word 0 -#endif - - .section .iram1,"ax" - .global xt_highint4 - .type xt_highint4,@function - .align 4 -xt_highint4: - -#ifndef CONFIG_FREERTOS_UNICORE - /* See if we're here for the IPC_ISR interrupt */ - rsr a0, INTERRUPT - extui a0, a0, ETS_IPC_ISR_INUM, 1 - bnez a0, esp_ipc_isr_handler -#endif // not CONFIG_FREERTOS_UNICORE - -#if CONFIG_ESP32_ECO3_CACHE_LOCK_FIX && CONFIG_ESP_INT_WDT - /* See if we're here for the tg1 watchdog interrupt */ - rsr a0, INTERRUPT - extui a0, a0, ETS_T1_WDT_INUM, 1 - beqz a0, 1f - - wsr a5, depc /* use DEPC as temp storage */ - movi a0, _l4_intr_livelock_counter - l32i a0, a0, 0 - movi a5, _l4_intr_livelock_max - l32i a5, a5, 0 - bltu a0, a5, .handle_livelock_int /* _l4_intr_livelock_counter < _l4_intr_livelock_max */ - - rsr a5, depc /* restore a5 */ -#endif - - /* Allocate exception frame and save minimal context. */ -1: mov a0, sp - addi sp, sp, -XT_STK_FRMSZ - s32i a0, sp, XT_STK_A1 - #if XCHAL_HAVE_WINDOWED - s32e a0, sp, -12 /* for debug backtrace */ - #endif - rsr a0, PS /* save interruptee's PS */ - s32i a0, sp, XT_STK_PS - rsr a0, EPC_4 /* save interruptee's PC */ - s32i a0, sp, XT_STK_PC - #if XCHAL_HAVE_WINDOWED - s32e a0, sp, -16 /* for debug backtrace */ - #endif - s32i a12, sp, XT_STK_A12 /* _xt_context_save requires A12- */ - s32i a13, sp, XT_STK_A13 /* A13 to have already been saved */ - call0 _xt_context_save - - /* Save vaddr into exception frame */ - rsr a0, EXCVADDR - s32i a0, sp, XT_STK_EXCVADDR - - /* Figure out reason, save into EXCCAUSE reg */ - - rsr a0, INTERRUPT - extui a0, a0, ETS_MEMACCESS_ERR_INUM, 1 /* get cacheerr int bit */ - beqz a0, 1f - /* Kill this interrupt; we cannot reset it. */ - rsr a0, INTENABLE - movi a4, ~(1<