From d4f2e03e4aa58a395c7479aa7f3b39eceddccf9f Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Thu, 6 May 2021 19:26:21 +0200 Subject: [PATCH] newlib: stdatomic: emulate 64-bit atomics on 32-bit SMP SoCs Closes https://github.com/espressif/esp-idf/issues/3163 --- components/newlib/stdatomic.c | 149 +++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 55 deletions(-) diff --git a/components/newlib/stdatomic.c b/components/newlib/stdatomic.c index 771bfd030c..993fcd1ac2 100644 --- a/components/newlib/stdatomic.c +++ b/components/newlib/stdatomic.c @@ -1,4 +1,4 @@ -// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD +// Copyright 2015-2021 Espressif Systems (Shanghai) PTE LTD // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,61 +17,78 @@ #include "sdkconfig.h" #include #include +#include "soc/soc_caps.h" +#include "freertos/FreeRTOS.h" #ifdef __XTENSA__ - #include "xtensa/config/core-isa.h" -#include "xtensa/xtruntime.h" - -// This allows nested interrupts disabling and restoring via local registers or stack. -// They can be called from interrupts too. -// WARNING: Only applies to current CPU. -#define _ATOMIC_ENTER_CRITICAL(void) ({ \ - unsigned state = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); \ - atomic_benchmark_intr_disable(); \ - state; \ -}) - -#define _ATOMIC_EXIT_CRITICAL(state) do { \ - atomic_benchmark_intr_restore(state); \ - XTOS_RESTORE_JUST_INTLEVEL(state); \ - } while (0) #ifndef XCHAL_HAVE_S32C1I #error "XCHAL_HAVE_S32C1I not defined, include correct header!" #endif -#define NO_ATOMICS_SUPPORT (XCHAL_HAVE_S32C1I == 0) +#define HAS_ATOMICS_32 (XCHAL_HAVE_S32C1I == 1) +// no 64-bit atomics on Xtensa +#define HAS_ATOMICS_64 0 + #else // RISCV -#include "freertos/portmacro.h" - -// This allows nested interrupts disabling and restoring via local registers or stack. -// They can be called from interrupts too. -// WARNING: Only applies to current CPU. -#define _ATOMIC_ENTER_CRITICAL(void) ({ \ - unsigned state = portENTER_CRITICAL_NESTED(); \ - atomic_benchmark_intr_disable(); \ - state; \ -}) - -#define _ATOMIC_EXIT_CRITICAL(state) do { \ - atomic_benchmark_intr_restore(state); \ - portEXIT_CRITICAL_NESTED(state); \ - } while (0) - -#ifndef __riscv_atomic // GCC toolchain will define this pre-processor if "A" extension is supported +// GCC toolchain will define this pre-processor if "A" extension is supported +#ifndef __riscv_atomic #define __riscv_atomic 0 #endif -#define NO_ATOMICS_SUPPORT (__riscv_atomic == 0) +#define HAS_ATOMICS_32 (__riscv_atomic == 1) +#define HAS_ATOMICS_64 ((__riscv_atomic == 1) && (__riscv_xlen == 64)) +#endif // (__XTENSA__, __riscv) -#endif +#if SOC_CPU_CORES_NUM == 1 +// Single core SoC: atomics can be implemented using portENTER_CRITICAL_NESTED +// and portEXIT_CRITICAL_NESTED, which disable and enable interrupts. +#define _ATOMIC_ENTER_CRITICAL() ({ \ + unsigned state = portENTER_CRITICAL_NESTED(); \ + state; \ +}) -//reserved to measure atomic operation time -#define atomic_benchmark_intr_disable() -#define atomic_benchmark_intr_restore(STATE) +#define _ATOMIC_EXIT_CRITICAL(state) do { \ + portEXIT_CRITICAL_NESTED(state); \ + } while (0) + +#else // SOC_CPU_CORES_NUM + +_Static_assert(HAS_ATOMICS_32, "32-bit atomics should be supported if SOC_CPU_CORES_NUM > 1"); +// Only need to implement 64-bit atomics here. Use a single global portMUX_TYPE spinlock +// to emulate the atomics. +static portMUX_TYPE s_atomic_lock = portMUX_INITIALIZER_UNLOCKED; + +// Return value is not used but kept for compatibility with the single-core version above. +#define _ATOMIC_ENTER_CRITICAL() ({ \ + portENTER_CRITICAL_SAFE(&s_atomic_lock); \ + 0; \ +}) + +#define _ATOMIC_EXIT_CRITICAL(state) do { \ + (void) (state); \ + portEXIT_CRITICAL_SAFE(&s_atomic_lock); \ +} while(0) + +#endif // SOC_CPU_CORES_NUM + +#define ATOMIC_LOAD(n, type) type __atomic_load_ ## n (const type* mem, int memorder) \ +{ \ + unsigned state = _ATOMIC_ENTER_CRITICAL(); \ + type ret = *mem; \ + _ATOMIC_EXIT_CRITICAL(state); \ + return ret; \ +} + +#define ATOMIC_STORE(n, type) void __atomic_store_ ## n (type* mem, type val, int memorder) \ +{ \ + unsigned state = _ATOMIC_ENTER_CRITICAL(); \ + *mem = val; \ + _ATOMIC_EXIT_CRITICAL(state); \ +} #define ATOMIC_EXCHANGE(n, type) type __atomic_exchange_ ## n (type* mem, type val, int memorder) \ { \ @@ -169,76 +186,98 @@ return ret; \ } -#if NO_ATOMICS_SUPPORT +#if !HAS_ATOMICS_32 ATOMIC_EXCHANGE(1, uint8_t) ATOMIC_EXCHANGE(2, uint16_t) ATOMIC_EXCHANGE(4, uint32_t) -ATOMIC_EXCHANGE(8, uint64_t) CMP_EXCHANGE(1, uint8_t) CMP_EXCHANGE(2, uint16_t) CMP_EXCHANGE(4, uint32_t) -CMP_EXCHANGE(8, uint64_t) FETCH_ADD(1, uint8_t) FETCH_ADD(2, uint16_t) FETCH_ADD(4, uint32_t) -FETCH_ADD(8, uint64_t) FETCH_SUB(1, uint8_t) FETCH_SUB(2, uint16_t) FETCH_SUB(4, uint32_t) -FETCH_SUB(8, uint64_t) FETCH_AND(1, uint8_t) FETCH_AND(2, uint16_t) FETCH_AND(4, uint32_t) -FETCH_AND(8, uint64_t) FETCH_OR(1, uint8_t) FETCH_OR(2, uint16_t) FETCH_OR(4, uint32_t) -FETCH_OR(8, uint64_t) FETCH_XOR(1, uint8_t) FETCH_XOR(2, uint16_t) FETCH_XOR(4, uint32_t) -FETCH_XOR(8, uint64_t) SYNC_FETCH_OP(add, 1, uint8_t) SYNC_FETCH_OP(add, 2, uint16_t) SYNC_FETCH_OP(add, 4, uint32_t) -SYNC_FETCH_OP(add, 8, uint64_t) SYNC_FETCH_OP(sub, 1, uint8_t) SYNC_FETCH_OP(sub, 2, uint16_t) SYNC_FETCH_OP(sub, 4, uint32_t) -SYNC_FETCH_OP(sub, 8, uint64_t) SYNC_FETCH_OP(and, 1, uint8_t) SYNC_FETCH_OP(and, 2, uint16_t) SYNC_FETCH_OP(and, 4, uint32_t) -SYNC_FETCH_OP(and, 8, uint64_t) SYNC_FETCH_OP(or, 1, uint8_t) SYNC_FETCH_OP(or, 2, uint16_t) SYNC_FETCH_OP(or, 4, uint32_t) -SYNC_FETCH_OP(or, 8, uint64_t) SYNC_FETCH_OP(xor, 1, uint8_t) SYNC_FETCH_OP(xor, 2, uint16_t) SYNC_FETCH_OP(xor, 4, uint32_t) -SYNC_FETCH_OP(xor, 8, uint64_t) SYNC_BOOL_CMP_EXCHANGE(1, uint8_t) SYNC_BOOL_CMP_EXCHANGE(2, uint16_t) SYNC_BOOL_CMP_EXCHANGE(4, uint32_t) -SYNC_BOOL_CMP_EXCHANGE(8, uint64_t) SYNC_VAL_CMP_EXCHANGE(1, uint8_t) SYNC_VAL_CMP_EXCHANGE(2, uint16_t) SYNC_VAL_CMP_EXCHANGE(4, uint32_t) + +#endif // !HAS_ATOMICS_32 + +#if !HAS_ATOMICS_64 + +ATOMIC_LOAD(8, uint64_t) + +ATOMIC_STORE(8, uint64_t) + +ATOMIC_EXCHANGE(8, uint64_t) + +CMP_EXCHANGE(8, uint64_t) + +FETCH_ADD(8, uint64_t) + +FETCH_SUB(8, uint64_t) + +FETCH_AND(8, uint64_t) + +FETCH_OR(8, uint64_t) + +FETCH_XOR(8, uint64_t) + +SYNC_FETCH_OP(add, 8, uint64_t) + +SYNC_FETCH_OP(sub, 8, uint64_t) + +SYNC_FETCH_OP(and, 8, uint64_t) + +SYNC_FETCH_OP(or, 8, uint64_t) + +SYNC_FETCH_OP(xor, 8, uint64_t) + +SYNC_BOOL_CMP_EXCHANGE(8, uint64_t) + SYNC_VAL_CMP_EXCHANGE(8, uint64_t) -#endif +#endif // !HAS_ATOMICS_64