wolfSSL/wolfssl
1
0
Fork 1
mirror of https://github.com/wolfSSL/wolfssl.git synced 2026-01-26 18:32:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

linuxkm/linuxkm_wc_port.h and linuxkm/x86_vector_register_glue.c: refactor

Browse Source 
wc_save_vector_registers_x86() and wc_restore_vector_registers_x86() to allow
  recursive WC_SVR_FLAG_INHIBIT while already in a vector save context;

linuxkm/lkcapi_sha_glue.c: in get_drbg() and put_drbg(),
  DISABLE_VECTOR_REGISTERS()...REENABLE_VECTOR_REGISTERS() if tfm ==
  crypto_default_rng.
This commit is contained in:
Daniel Pouzzner
2025-08-28 11:02:45 -05:00
parent 0224ef3d2e
commit f7c7ac275a
4 changed files with 50 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.wolfssl_known_macro_extras
View File

@@ -661,7 +661,6 @@ WOLFSSL_CAAM_BLACK_KEY_SM
WOLFSSL_CAAM_NO_BLACK_KEY
WOLFSSL_CALLBACKS
WOLFSSL_CHECK_DESKEY
WOLFSSL_CHECK_MEM_ZERO
WOLFSSL_CHIBIOS
WOLFSSL_CLANG_TIDY
WOLFSSL_CLIENT_EXAMPLE

10
linuxkm/linuxkm_wc_port.h
View File

@@ -469,7 +469,7 @@
extern void free_wolfcrypt_linuxkm_fpu_states(void);
WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
WOLFSSL_API void wc_restore_vector_registers_x86(void);
WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#include <asm/i387.h>
@@ -505,14 +505,14 @@
#endif
#endif
#ifndef RESTORE_VECTOR_REGISTERS
#define RESTORE_VECTOR_REGISTERS() wc_restore_vector_registers_x86()
#define RESTORE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_NONE)
#endif
#ifndef DISABLE_VECTOR_REGISTERS
#define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
#endif
#ifndef REENABLE_VECTOR_REGISTERS
#define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86()
#define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
#endif
#elif defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64))
@@ -1217,12 +1217,12 @@
#if defined(CONFIG_X86)
WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
WOLFSSL_API void wc_restore_vector_registers_x86(void);
WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);
#ifndef DISABLE_VECTOR_REGISTERS
#define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
#endif
#ifndef REENABLE_VECTOR_REGISTERS
#define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86()
#define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
#endif
#else /* !CONFIG_X86 */
#error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.

30
linuxkm/lkcapi_sha_glue.c
View File

@@ -968,6 +968,7 @@ struct wc_linuxkm_drbg_ctx {
struct wc_rng_inst {
wolfSSL_Atomic_Int lock;
WC_RNG rng;
int disabled_vec_ops;
} *rngs; /* one per CPU ID */
};
@@ -1089,8 +1090,14 @@ static inline struct wc_rng_inst *get_drbg(struct crypto_rng *tfm) {
for (;;) {
int expected = 0;
if (likely(__atomic_compare_exchange_n(&ctx->rngs[n].lock, &expected, new_lock_value, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE)))
return &ctx->rngs[n];
if (likely(__atomic_compare_exchange_n(&ctx->rngs[n].lock, &expected, new_lock_value, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE))) {
struct wc_rng_inst *drbg = &ctx->rngs[n];
if (tfm == crypto_default_rng)
drbg->disabled_vec_ops = (DISABLE_VECTOR_REGISTERS() == 0);
else
drbg->disabled_vec_ops = 0;
return drbg;
}
++n;
if (n >= (int)ctx->n_rngs)
n = 0;
@@ -1108,8 +1115,11 @@ static inline struct wc_rng_inst *get_drbg_n(struct wc_linuxkm_drbg_ctx *ctx, in
for (;;) {
int expected = 0;
if (likely(__atomic_compare_exchange_n(&ctx->rngs[n].lock, &expected, 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE)))
return &ctx->rngs[n];
if (likely(__atomic_compare_exchange_n(&ctx->rngs[n].lock, &expected, 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE))) {
struct wc_rng_inst *drbg = &ctx->rngs[n];
drbg->disabled_vec_ops = 0;
return drbg;
}
if (can_sleep) {
if (signal_pending(current))
return NULL;
@@ -1127,6 +1137,10 @@ static inline void put_drbg(struct wc_rng_inst *drbg) {
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
int migration_disabled = (drbg->lock == 2);
#endif
if (drbg->disabled_vec_ops) {
REENABLE_VECTOR_REGISTERS();
drbg->disabled_vec_ops = 0;
}
__atomic_store_n(&(drbg->lock),0,__ATOMIC_RELEASE);
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
@@ -1140,7 +1154,6 @@ static int wc_linuxkm_drbg_generate(struct crypto_rng *tfm,
u8 *dst, unsigned int dlen)
{
int ret, retried = 0;
int need_fpu_restore;
struct wc_rng_inst *drbg = get_drbg(tfm);
if (! drbg) {
@@ -1148,11 +1161,6 @@ static int wc_linuxkm_drbg_generate(struct crypto_rng *tfm,
return -EFAULT;
}
/* for the default RNG, make sure we don't cache an underlying SHA256
* method that uses vector insns (forbidden from irq handlers).
*/
need_fpu_restore = (tfm == crypto_default_rng) ? (DISABLE_VECTOR_REGISTERS() == 0) : 0;
retry:
if (slen > 0) {
@@ -1186,8 +1194,6 @@ retry:
out:
if (need_fpu_restore)
REENABLE_VECTOR_REGISTERS();
put_drbg(drbg);
return ret;

38
linuxkm/x86_vector_register_glue.c
View File

@@ -346,7 +346,14 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags)
/* allow for nested calls */
if (pstate && (pstate->fpu_state != 0U)) {
if (unlikely(pstate->fpu_state & WC_FPU_INHIBITED_FLAG)) {
if (unlikely((pstate->fpu_state & WC_FPU_COUNT_MASK)
== WC_FPU_COUNT_MASK))
{
pr_err("ERROR: wc_save_vector_registers_x86 recursion register overflow for "
"pid %d on CPU %d.\n", pstate->pid, raw_smp_processor_id());
return BAD_STATE_E;
}
if (pstate->fpu_state & WC_FPU_INHIBITED_FLAG) {
if (flags & WC_SVR_FLAG_INHIBIT) {
/* allow recursive inhibit calls as long as the whole stack of
* them is inhibiting.
@@ -357,15 +364,12 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags)
else
return WC_ACCEL_INHIBIT_E;
}
if (unlikely(flags & WC_SVR_FLAG_INHIBIT))
return BAD_STATE_E;
if (unlikely((pstate->fpu_state & WC_FPU_COUNT_MASK)
== WC_FPU_COUNT_MASK))
{
pr_err("ERROR: wc_save_vector_registers_x86 recursion register overflow for "
"pid %d on CPU %d.\n", pstate->pid, raw_smp_processor_id());
return BAD_STATE_E;
} else {
if (flags & WC_SVR_FLAG_INHIBIT) {
++pstate->fpu_state;
pstate->fpu_state |= WC_FPU_INHIBITED_FLAG;
return 0;
}
else {
++pstate->fpu_state;
return 0;
}
@@ -475,7 +479,7 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags)
__builtin_unreachable();
}
void wc_restore_vector_registers_x86(void)
void wc_restore_vector_registers_x86(enum wc_svr_flags flags)
{
struct wc_thread_fpu_count_ent *pstate;
@@ -494,6 +498,14 @@ void wc_restore_vector_registers_x86(void)
}
if ((--pstate->fpu_state & WC_FPU_COUNT_MASK) > 0U) {
if (flags & WC_SVR_FLAG_INHIBIT) {
if (pstate->fpu_state & WC_FPU_INHIBITED_FLAG)
pstate->fpu_state &= ~WC_FPU_INHIBITED_FLAG;
else
VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called by pid %d on CPU %d "
"with _INHIBIT flag but saved state isn't _INHIBITED_.\n", task_pid_nr(current),
raw_smp_processor_id());
}
return;
}
@@ -505,6 +517,10 @@ void wc_restore_vector_registers_x86(void)
#endif
local_bh_enable();
} else if (unlikely(pstate->fpu_state & WC_FPU_INHIBITED_FLAG)) {
if (unlikely(! (flags & WC_SVR_FLAG_INHIBIT)))
VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called by pid %d on CPU %d "
"without _INHIBIT flag but saved state is _INHIBITED_.\n", task_pid_nr(current),
raw_smp_processor_id());
pstate->fpu_state = 0U;
wc_linuxkm_fpu_state_release(pstate);
local_bh_enable();