From ada6c5f95b4825e03456d08f715577f71cb1f0cd Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Thu, 4 Jun 2026 18:57:03 +1000 Subject: [PATCH] SP gen: FP_ECC init mutex improvement F-1379 Better handling of the lazy mutex initialization to use atomics where available. Improved atomic code when no system support: - add types - used types in functions Add --no-ec to unit.test to not run wolfCrypt tests. --- tests/unit.c | 6 +- wolfcrypt/src/sp_arm32.c | 384 +++++++++++++++++++++++++++--------- wolfcrypt/src/sp_arm64.c | 336 +++++++++++++++++++++++-------- wolfcrypt/src/sp_armthumb.c | 384 +++++++++++++++++++++++++++--------- wolfcrypt/src/sp_c32.c | 192 +++++++++++++----- wolfcrypt/src/sp_c64.c | 192 +++++++++++++----- wolfcrypt/src/sp_cortexm.c | 384 +++++++++++++++++++++++++++--------- wolfcrypt/src/sp_x86_64.c | 368 +++++++++++++++++++++++++--------- wolfssl/wolfcrypt/wc_port.h | 26 +-- 9 files changed, 1704 insertions(+), 568 deletions(-) diff --git a/tests/unit.c b/tests/unit.c index 1734f7abbe..5f788fe051 100644 --- a/tests/unit.c +++ b/tests/unit.c @@ -43,6 +43,7 @@ int allTesting = 1; int apiTesting = 1; +int wolfCryptTesting = 1; int myoptind = 0; char* myoptarg = NULL; int unit_test(int argc, char** argv); @@ -202,6 +203,9 @@ int unit_test(int argc, char** argv) ApiTest_PrintTestCases(); goto exit; } + else if (XSTRCMP(argv[1], "--no-wc") == 0) { + wolfCryptTesting = 0; + } else if (XSTRCMP(argv[1], "--api") == 0) { allTesting = 0; } @@ -257,7 +261,7 @@ int unit_test(int argc, char** argv) #ifndef NO_CRYPT_TEST /* wc_ test */ - if (allTesting) { + if (allTesting && wolfCryptTesting) { func_args wc_args; printf("\nwolfCrypt unit test:\n"); diff --git a/wolfcrypt/src/sp_arm32.c b/wolfcrypt/src/sp_arm32.c index dd10cc64ad..d2bad638b8 100644 --- a/wolfcrypt/src/sp_arm32.c +++ b/wolfcrypt/src/sp_arm32.c @@ -75775,9 +75775,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -75844,6 +75844,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -75866,19 +75867,42 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 8 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -75892,9 +75916,9 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_8(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -76156,9 +76180,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -76225,6 +76249,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -76247,19 +76272,42 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 8 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -76273,9 +76321,9 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_8(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -93808,9 +93856,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -93877,6 +93925,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -93899,19 +93948,42 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 12 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -93925,9 +93997,9 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_12(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -94205,9 +94277,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -94274,6 +94346,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -94296,19 +94369,42 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 12 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -94322,9 +94418,9 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_12(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -120967,9 +121063,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -121036,6 +121132,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -121058,19 +121155,42 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 17 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -121084,9 +121204,9 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_17(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -121384,9 +121504,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -121453,6 +121573,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -121475,19 +121596,42 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 17 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -121501,9 +121645,9 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_17(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -150732,9 +150876,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -150801,6 +150945,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -150823,19 +150968,42 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 32 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -150849,9 +151017,9 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_32(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -151046,9 +151214,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -151115,6 +151283,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -151137,19 +151306,42 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 32 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -151163,9 +151355,9 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_32(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); diff --git a/wolfcrypt/src/sp_arm64.c b/wolfcrypt/src/sp_arm64.c index 59751ebe0c..01bce22285 100644 --- a/wolfcrypt/src/sp_arm64.c +++ b/wolfcrypt/src/sp_arm64.c @@ -24586,9 +24586,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -24655,6 +24655,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -24677,19 +24678,42 @@ static int sp_256_ecc_mulmod_4(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 4 * 5, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -24703,9 +24727,9 @@ static int sp_256_ecc_mulmod_4(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_4(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -24974,9 +24998,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -25043,6 +25067,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -25065,19 +25090,42 @@ static int sp_256_ecc_mulmod_4(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 4 * 5, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -25091,9 +25139,9 @@ static int sp_256_ecc_mulmod_4(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_4(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -45088,9 +45136,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -45157,6 +45205,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -45179,19 +45228,42 @@ static int sp_384_ecc_mulmod_6(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 6 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -45205,9 +45277,9 @@ static int sp_384_ecc_mulmod_6(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_6(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -45476,9 +45548,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -45545,6 +45617,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -45567,19 +45640,42 @@ static int sp_384_ecc_mulmod_6(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 6 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -45593,9 +45689,9 @@ static int sp_384_ecc_mulmod_6(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_6(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -73076,9 +73172,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -73145,6 +73241,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -73167,19 +73264,42 @@ static int sp_521_ecc_mulmod_9(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 9 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -73193,9 +73313,9 @@ static int sp_521_ecc_mulmod_9(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_9(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -73482,9 +73602,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -73551,6 +73671,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -73573,19 +73694,42 @@ static int sp_521_ecc_mulmod_9(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 9 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -73599,9 +73743,9 @@ static int sp_521_ecc_mulmod_9(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_9(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -116561,9 +116705,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -116630,6 +116774,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -116652,19 +116797,42 @@ static int sp_1024_ecc_mulmod_16(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 16 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -116678,9 +116846,9 @@ static int sp_1024_ecc_mulmod_16(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_16(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); diff --git a/wolfcrypt/src/sp_armthumb.c b/wolfcrypt/src/sp_armthumb.c index f56e1d9091..264fd2376a 100644 --- a/wolfcrypt/src/sp_armthumb.c +++ b/wolfcrypt/src/sp_armthumb.c @@ -101395,9 +101395,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -101464,6 +101464,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -101486,19 +101487,42 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 8 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -101512,9 +101536,9 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_8(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -101776,9 +101800,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -101845,6 +101869,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -101867,19 +101892,42 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 8 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -101893,9 +101941,9 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_8(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -111863,9 +111911,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -111932,6 +111980,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -111954,19 +112003,42 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 12 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -111980,9 +112052,9 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_12(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -112260,9 +112332,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -112329,6 +112401,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -112351,19 +112424,42 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 12 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -112377,9 +112473,9 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_12(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -125126,9 +125222,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -125195,6 +125291,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -125217,19 +125314,42 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 17 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -125243,9 +125363,9 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_17(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -125543,9 +125663,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -125612,6 +125732,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -125634,19 +125755,42 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 17 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -125660,9 +125804,9 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_17(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -209386,9 +209530,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -209455,6 +209599,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -209477,19 +209622,42 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 32 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -209503,9 +209671,9 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_32(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -209700,9 +209868,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -209769,6 +209937,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -209791,19 +209960,42 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 32 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -209817,9 +210009,9 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_32(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); diff --git a/wolfcrypt/src/sp_c32.c b/wolfcrypt/src/sp_c32.c index 8262fb2f73..25fb6fc576 100644 --- a/wolfcrypt/src/sp_c32.c +++ b/wolfcrypt/src/sp_c32.c @@ -23319,9 +23319,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -23388,6 +23388,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -23410,19 +23411,42 @@ static int sp_256_ecc_mulmod_9(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 9 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -23436,9 +23460,9 @@ static int sp_256_ecc_mulmod_9(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_9(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -30437,9 +30461,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -30506,6 +30530,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -30528,19 +30553,42 @@ static int sp_384_ecc_mulmod_15(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 15 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -30554,9 +30602,9 @@ static int sp_384_ecc_mulmod_15(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_15(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -37617,9 +37665,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -37686,6 +37734,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -37708,19 +37757,42 @@ static int sp_521_ecc_mulmod_21(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 21 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -37734,9 +37806,9 @@ static int sp_521_ecc_mulmod_21(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_21(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -45757,9 +45829,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -45826,6 +45898,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -45848,19 +45921,42 @@ static int sp_1024_ecc_mulmod_42(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 42 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -45874,9 +45970,9 @@ static int sp_1024_ecc_mulmod_42(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_42(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); diff --git a/wolfcrypt/src/sp_c64.c b/wolfcrypt/src/sp_c64.c index b0dd93cbd9..2c9875fd3a 100644 --- a/wolfcrypt/src/sp_c64.c +++ b/wolfcrypt/src/sp_c64.c @@ -23947,9 +23947,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -24016,6 +24016,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -24038,19 +24039,42 @@ static int sp_256_ecc_mulmod_5(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 5 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -24064,9 +24088,9 @@ static int sp_256_ecc_mulmod_5(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_5(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -30492,9 +30516,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -30561,6 +30585,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -30583,19 +30608,42 @@ static int sp_384_ecc_mulmod_7(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 7 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -30609,9 +30657,9 @@ static int sp_384_ecc_mulmod_7(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_7(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -37510,9 +37558,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -37579,6 +37627,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -37601,19 +37650,42 @@ static int sp_521_ecc_mulmod_9(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 9 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -37627,9 +37699,9 @@ static int sp_521_ecc_mulmod_9(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_9(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -44832,9 +44904,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -44901,6 +44973,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -44923,19 +44996,42 @@ static int sp_1024_ecc_mulmod_18(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 18 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -44949,9 +45045,9 @@ static int sp_1024_ecc_mulmod_18(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_18(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); diff --git a/wolfcrypt/src/sp_cortexm.c b/wolfcrypt/src/sp_cortexm.c index f9d22092d9..cd9361516d 100644 --- a/wolfcrypt/src/sp_cortexm.c +++ b/wolfcrypt/src/sp_cortexm.c @@ -37881,9 +37881,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -37950,6 +37950,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -37972,19 +37973,42 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 8 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -37998,9 +38022,9 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_8(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -38262,9 +38286,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -38331,6 +38355,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -38353,19 +38378,42 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 8 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -38379,9 +38427,9 @@ static int sp_256_ecc_mulmod_8(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_8(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -47989,9 +48037,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -48058,6 +48106,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -48080,19 +48129,42 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 12 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -48106,9 +48178,9 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_12(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -48386,9 +48458,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -48455,6 +48527,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -48477,19 +48550,42 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 12 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -48503,9 +48599,9 @@ static int sp_384_ecc_mulmod_12(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_12(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -59986,9 +60082,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -60055,6 +60151,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -60077,19 +60174,42 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 17 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -60103,9 +60223,9 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_17(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -60403,9 +60523,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -60472,6 +60592,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -60494,19 +60615,42 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 17 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -60520,9 +60664,9 @@ static int sp_521_ecc_mulmod_17(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_17(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -74469,9 +74613,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -74538,6 +74682,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -74560,19 +74705,42 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 32 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -74586,9 +74754,9 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_32(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -74783,9 +74951,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -74852,6 +75020,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -74874,19 +75043,42 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 32 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -74900,9 +75092,9 @@ static int sp_1024_ecc_mulmod_32(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_32(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); diff --git a/wolfcrypt/src/sp_x86_64.c b/wolfcrypt/src/sp_x86_64.c index 7ce9b9ab6a..a720f91843 100644 --- a/wolfcrypt/src/sp_x86_64.c +++ b/wolfcrypt/src/sp_x86_64.c @@ -10441,9 +10441,9 @@ static THREAD_LS_T int sp_cache_256_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_256_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_256 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_256 = 0; #endif static wolfSSL_Mutex sp_cache_256_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_256_lock); #endif @@ -10510,6 +10510,7 @@ static void sp_ecc_get_cache_256(const sp_point_256* g, sp_cache_256_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -10532,19 +10533,42 @@ static int sp_256_ecc_mulmod_4(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 4 * 5, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -10558,9 +10582,9 @@ static int sp_256_ecc_mulmod_4(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_4(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -10848,6 +10872,7 @@ static int sp_256_ecc_mulmod_stripe_avx2_4(sp_point_256* r, const sp_point_256* } #endif /* FP_ECC | WOLFSSL_SP_SMALL */ + /* Multiply the base point of P256 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -10870,19 +10895,42 @@ static int sp_256_ecc_mulmod_avx2_4(sp_point_256* r, const sp_point_256* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 4 * 5, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_256 == 0) { - wc_InitMutex(&sp_cache_256_lock); - initCacheMutex_256 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_256) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_256, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_256_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_256, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_256_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_256_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_256(g, &cache); @@ -10896,9 +10944,9 @@ static int sp_256_ecc_mulmod_avx2_4(sp_point_256* r, const sp_point_256* g, err = sp_256_ecc_mulmod_stripe_avx2_4(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_256_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -29217,9 +29265,9 @@ static THREAD_LS_T int sp_cache_384_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_384_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_384 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_384 = 0; #endif static wolfSSL_Mutex sp_cache_384_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_384_lock); #endif @@ -29286,6 +29334,7 @@ static void sp_ecc_get_cache_384(const sp_point_384* g, sp_cache_384_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -29308,19 +29357,42 @@ static int sp_384_ecc_mulmod_6(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 6 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -29334,9 +29406,9 @@ static int sp_384_ecc_mulmod_6(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_6(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -29627,6 +29699,7 @@ static int sp_384_ecc_mulmod_stripe_avx2_6(sp_point_384* r, const sp_point_384* } #endif /* FP_ECC | WOLFSSL_SP_SMALL */ + /* Multiply the base point of P384 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -29649,19 +29722,42 @@ static int sp_384_ecc_mulmod_avx2_6(sp_point_384* r, const sp_point_384* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 6 * 7, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_384 == 0) { - wc_InitMutex(&sp_cache_384_lock); - initCacheMutex_384 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_384) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_384, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_384_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_384, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_384_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_384_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_384(g, &cache); @@ -29675,9 +29771,9 @@ static int sp_384_ecc_mulmod_avx2_6(sp_point_384* r, const sp_point_384* g, err = sp_384_ecc_mulmod_stripe_avx2_6(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_384_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -53703,9 +53799,9 @@ static THREAD_LS_T int sp_cache_521_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_521_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_521 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_521 = 0; #endif static wolfSSL_Mutex sp_cache_521_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_521_lock); #endif @@ -53772,6 +53868,7 @@ static void sp_ecc_get_cache_521(const sp_point_521* g, sp_cache_521_t** cache) } #endif /* FP_ECC */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -53794,19 +53891,42 @@ static int sp_521_ecc_mulmod_9(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 9 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -53820,9 +53940,9 @@ static int sp_521_ecc_mulmod_9(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_9(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -54113,6 +54233,7 @@ static int sp_521_ecc_mulmod_stripe_avx2_9(sp_point_521* r, const sp_point_521* } #endif /* FP_ECC | WOLFSSL_SP_SMALL */ + /* Multiply the base point of P521 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -54135,19 +54256,42 @@ static int sp_521_ecc_mulmod_avx2_9(sp_point_521* r, const sp_point_521* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 9 * 6, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_521 == 0) { - wc_InitMutex(&sp_cache_521_lock); - initCacheMutex_521 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_521) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_521, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_521_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_521, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_521_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_521_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_521(g, &cache); @@ -54161,9 +54305,9 @@ static int sp_521_ecc_mulmod_avx2_9(sp_point_521* r, const sp_point_521* g, err = sp_521_ecc_mulmod_stripe_avx2_9(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_521_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -94505,9 +94649,9 @@ static THREAD_LS_T int sp_cache_1024_last = -1; /* Cache has been initialized. */ static THREAD_LS_T int sp_cache_1024_inited = 0; -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) #ifndef WOLFSSL_MUTEX_INITIALIZER - static volatile int initCacheMutex_1024 = 0; + static wolfSSL_Atomic_Uint initCacheMutex_1024 = 0; #endif static wolfSSL_Mutex sp_cache_1024_lock WOLFSSL_MUTEX_INITIALIZER_CLAUSE(sp_cache_1024_lock); #endif @@ -94574,6 +94718,7 @@ static void sp_ecc_get_cache_1024(const sp_point_1024* g, sp_cache_1024_t** cach } #endif /* FP_ECC */ + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -94596,19 +94741,42 @@ static int sp_1024_ecc_mulmod_16(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 16 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -94622,9 +94790,9 @@ static int sp_1024_ecc_mulmod_16(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_16(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); @@ -94898,6 +95066,7 @@ static int sp_1024_ecc_mulmod_stripe_avx2_16(sp_point_1024* r, const sp_point_10 return err; } + /* Multiply the base point of P1024 by the scalar and return the result. * If map is true then convert result to affine coordinates. * @@ -94920,19 +95089,42 @@ static int sp_1024_ecc_mulmod_avx2_16(sp_point_1024* r, const sp_point_1024* g, int err = MP_OKAY; SP_ALLOC_VAR(sp_digit, tmp, 2 * 16 * 38, heap, DYNAMIC_TYPE_ECC); -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) if (err == MP_OKAY) { - #ifndef WOLFSSL_MUTEX_INITIALIZER - if (initCacheMutex_1024 == 0) { - wc_InitMutex(&sp_cache_1024_lock); - initCacheMutex_1024 = 1; + #ifndef WOLFSSL_MUTEX_INITIALIZER + /* Lazy initialization of mutex - one atomic with three states: + * 0 = uninitialized, 1 = initialization in progress, + * 2 = initialized. + */ + if (WOLFSSL_ATOMIC_LOAD(initCacheMutex_1024) != 2) { + unsigned int expected_then_actual; + + for (;;) { + expected_then_actual = 0; + if (wolfSSL_Atomic_Uint_CompareExchange( + &initCacheMutex_1024, &expected_then_actual, + 1) == 1) { + /* Won race - initialize mutex. On failure, reset state + * to 0 so that a later call retries. */ + err = wc_InitMutex(&sp_cache_1024_lock); + WOLFSSL_ATOMIC_STORE(initCacheMutex_1024, + (err == 0) ? 2U : 0U); + break; + } + if (expected_then_actual == 2) { + /* Another thread completed initialization. */ + break; + } + /* Initialization in progress in another thread. */ + WC_RELAX_LONG_LOOP(); + } } - #endif - if (wc_LockMutex(&sp_cache_1024_lock) != 0) { + #endif + if ((err == MP_OKAY) && (wc_LockMutex(&sp_cache_1024_lock) != 0)) { err = BAD_MUTEX_E; } } -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ if (err == MP_OKAY) { sp_ecc_get_cache_1024(g, &cache); @@ -94946,9 +95138,9 @@ static int sp_1024_ecc_mulmod_avx2_16(sp_point_1024* r, const sp_point_1024* g, err = sp_1024_ecc_mulmod_stripe_avx2_16(r, g, cache->table, k, map, ct, heap); } -#ifndef HAVE_THREAD_LS +#if !defined(SINGLE_THREADED) && !defined(HAVE_THREAD_LS) wc_UnLockMutex(&sp_cache_1024_lock); -#endif /* HAVE_THREAD_LS */ +#endif /* !SINGLE_THREADED && !HAVE_THREAD_LS */ } SP_FREE_VAR(tmp, heap, DYNAMIC_TYPE_ECC); diff --git a/wolfssl/wolfcrypt/wc_port.h b/wolfssl/wolfcrypt/wc_port.h index 042551bfc0..f1e88c9b4d 100644 --- a/wolfssl/wolfcrypt/wc_port.h +++ b/wolfssl/wolfcrypt/wc_port.h @@ -581,6 +581,8 @@ #endif /* !WOLFSSL_NO_ATOMICS */ #ifdef WOLFSSL_NO_ATOMICS + typedef volatile int wolfSSL_Atomic_Int; + typedef volatile unsigned int wolfSSL_Atomic_Uint; #define WOLFSSL_ATOMIC_INITIALIZER(x) (x) #define WOLFSSL_ATOMIC_LOAD(x) (x) #define WOLFSSL_ATOMIC_STORE(x, val) (x) = (val) @@ -639,31 +641,33 @@ */ #define wolfSSL_Atomic_Int_Init(c, i) (*(c) = (i)) #define wolfSSL_Atomic_Uint_Init(c, i) (*(c) = (i)) - static WC_INLINE int wolfSSL_Atomic_Int_FetchAdd(int *c, int i) { + static WC_INLINE int wolfSSL_Atomic_Int_FetchAdd(wolfSSL_Atomic_Int *c, + int i) + { int ret = *c; *c += i; return ret; } - static WC_INLINE int wolfSSL_Atomic_Int_FetchSub(int *c, int i) { + static WC_INLINE int wolfSSL_Atomic_Int_FetchSub(wolfSSL_Atomic_Int *c, int i) { int ret = *c; *c -= i; return ret; } - static WC_INLINE int wolfSSL_Atomic_Int_AddFetch(int *c, int i) { + static WC_INLINE int wolfSSL_Atomic_Int_AddFetch(wolfSSL_Atomic_Int *c, int i) { return (*c += i); } - static WC_INLINE int wolfSSL_Atomic_Int_SubFetch(int *c, int i) { + static WC_INLINE int wolfSSL_Atomic_Int_SubFetch(wolfSSL_Atomic_Int *c, int i) { return (*c -= i); } static WC_INLINE int wolfSSL_Atomic_Int_Exchange( - int *c, int new_i) + wolfSSL_Atomic_Int *c, int new_i) { int ret = *c; *c = new_i; return ret; } static WC_INLINE int wolfSSL_Atomic_Int_CompareExchange( - int *c, int *expected_i, int new_i) + wolfSSL_Atomic_Int *c, int *expected_i, int new_i) { if (*c == *expected_i) { *c = new_i; @@ -687,31 +691,31 @@ } } static WC_INLINE unsigned int wolfSSL_Atomic_Uint_FetchAdd( - unsigned int *c, unsigned int i) + wolfSSL_Atomic_Uint *c, unsigned int i) { unsigned int ret = *c; *c += i; return ret; } static WC_INLINE unsigned int wolfSSL_Atomic_Uint_FetchSub( - unsigned int *c, unsigned int i) + wolfSSL_Atomic_Uint *c, unsigned int i) { unsigned int ret = *c; *c -= i; return ret; } static WC_INLINE unsigned int wolfSSL_Atomic_Uint_AddFetch( - unsigned int *c, unsigned int i) + wolfSSL_Atomic_Uint *c, unsigned int i) { return (*c += i); } static WC_INLINE unsigned int wolfSSL_Atomic_Uint_SubFetch( - unsigned int *c, unsigned int i) + wolfSSL_Atomic_Uint *c, unsigned int i) { return (*c -= i); } static WC_INLINE int wolfSSL_Atomic_Uint_CompareExchange( - unsigned int *c, unsigned int *expected_i, unsigned int new_i) + wolfSSL_Atomic_Uint *c, unsigned int *expected_i, unsigned int new_i) { if (*c == *expected_i) { *c = new_i;