From f7c7ac275a6364a8598f192a4b52a02aca4e3ec4 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Thu, 28 Aug 2025 11:02:45 -0500 Subject: [PATCH] linuxkm/linuxkm_wc_port.h and linuxkm/x86_vector_register_glue.c: refactor wc_save_vector_registers_x86() and wc_restore_vector_registers_x86() to allow recursive WC_SVR_FLAG_INHIBIT while already in a vector save context; linuxkm/lkcapi_sha_glue.c: in get_drbg() and put_drbg(), DISABLE_VECTOR_REGISTERS()...REENABLE_VECTOR_REGISTERS() if tfm == crypto_default_rng. --- .wolfssl_known_macro_extras | 1 - linuxkm/linuxkm_wc_port.h | 10 ++++---- linuxkm/lkcapi_sha_glue.c | 30 +++++++++++++---------- linuxkm/x86_vector_register_glue.c | 38 +++++++++++++++++++++--------- 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 6a8f6ab7c..2c68d0b00 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -661,7 +661,6 @@ WOLFSSL_CAAM_BLACK_KEY_SM WOLFSSL_CAAM_NO_BLACK_KEY WOLFSSL_CALLBACKS WOLFSSL_CHECK_DESKEY -WOLFSSL_CHECK_MEM_ZERO WOLFSSL_CHIBIOS WOLFSSL_CLANG_TIDY WOLFSSL_CLIENT_EXAMPLE diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 197b9176a..14f8ae693 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -469,7 +469,7 @@ extern void free_wolfcrypt_linuxkm_fpu_states(void); WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void); WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags); - WOLFSSL_API void wc_restore_vector_registers_x86(void); + WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags); #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) #include @@ -505,14 +505,14 @@ #endif #endif #ifndef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() wc_restore_vector_registers_x86() + #define RESTORE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_NONE) #endif #ifndef DISABLE_VECTOR_REGISTERS #define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT) #endif #ifndef REENABLE_VECTOR_REGISTERS - #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86() + #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT) #endif #elif defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64)) @@ -1217,12 +1217,12 @@ #if defined(CONFIG_X86) WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void); WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags); - WOLFSSL_API void wc_restore_vector_registers_x86(void); + WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags); #ifndef DISABLE_VECTOR_REGISTERS #define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT) #endif #ifndef REENABLE_VECTOR_REGISTERS - #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86() + #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT) #endif #else /* !CONFIG_X86 */ #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture. diff --git a/linuxkm/lkcapi_sha_glue.c b/linuxkm/lkcapi_sha_glue.c index 0e73dd981..584ebc3f3 100644 --- a/linuxkm/lkcapi_sha_glue.c +++ b/linuxkm/lkcapi_sha_glue.c @@ -968,6 +968,7 @@ struct wc_linuxkm_drbg_ctx { struct wc_rng_inst { wolfSSL_Atomic_Int lock; WC_RNG rng; + int disabled_vec_ops; } *rngs; /* one per CPU ID */ }; @@ -1089,8 +1090,14 @@ static inline struct wc_rng_inst *get_drbg(struct crypto_rng *tfm) { for (;;) { int expected = 0; - if (likely(__atomic_compare_exchange_n(&ctx->rngs[n].lock, &expected, new_lock_value, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE))) - return &ctx->rngs[n]; + if (likely(__atomic_compare_exchange_n(&ctx->rngs[n].lock, &expected, new_lock_value, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE))) { + struct wc_rng_inst *drbg = &ctx->rngs[n]; + if (tfm == crypto_default_rng) + drbg->disabled_vec_ops = (DISABLE_VECTOR_REGISTERS() == 0); + else + drbg->disabled_vec_ops = 0; + return drbg; + } ++n; if (n >= (int)ctx->n_rngs) n = 0; @@ -1108,8 +1115,11 @@ static inline struct wc_rng_inst *get_drbg_n(struct wc_linuxkm_drbg_ctx *ctx, in for (;;) { int expected = 0; - if (likely(__atomic_compare_exchange_n(&ctx->rngs[n].lock, &expected, 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE))) - return &ctx->rngs[n]; + if (likely(__atomic_compare_exchange_n(&ctx->rngs[n].lock, &expected, 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE))) { + struct wc_rng_inst *drbg = &ctx->rngs[n]; + drbg->disabled_vec_ops = 0; + return drbg; + } if (can_sleep) { if (signal_pending(current)) return NULL; @@ -1127,6 +1137,10 @@ static inline void put_drbg(struct wc_rng_inst *drbg) { (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) int migration_disabled = (drbg->lock == 2); #endif + if (drbg->disabled_vec_ops) { + REENABLE_VECTOR_REGISTERS(); + drbg->disabled_vec_ops = 0; + } __atomic_store_n(&(drbg->lock),0,__ATOMIC_RELEASE); #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) @@ -1140,7 +1154,6 @@ static int wc_linuxkm_drbg_generate(struct crypto_rng *tfm, u8 *dst, unsigned int dlen) { int ret, retried = 0; - int need_fpu_restore; struct wc_rng_inst *drbg = get_drbg(tfm); if (! drbg) { @@ -1148,11 +1161,6 @@ static int wc_linuxkm_drbg_generate(struct crypto_rng *tfm, return -EFAULT; } - /* for the default RNG, make sure we don't cache an underlying SHA256 - * method that uses vector insns (forbidden from irq handlers). - */ - need_fpu_restore = (tfm == crypto_default_rng) ? (DISABLE_VECTOR_REGISTERS() == 0) : 0; - retry: if (slen > 0) { @@ -1186,8 +1194,6 @@ retry: out: - if (need_fpu_restore) - REENABLE_VECTOR_REGISTERS(); put_drbg(drbg); return ret; diff --git a/linuxkm/x86_vector_register_glue.c b/linuxkm/x86_vector_register_glue.c index 159162cd1..5ec17acfd 100644 --- a/linuxkm/x86_vector_register_glue.c +++ b/linuxkm/x86_vector_register_glue.c @@ -346,7 +346,14 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags) /* allow for nested calls */ if (pstate && (pstate->fpu_state != 0U)) { - if (unlikely(pstate->fpu_state & WC_FPU_INHIBITED_FLAG)) { + if (unlikely((pstate->fpu_state & WC_FPU_COUNT_MASK) + == WC_FPU_COUNT_MASK)) + { + pr_err("ERROR: wc_save_vector_registers_x86 recursion register overflow for " + "pid %d on CPU %d.\n", pstate->pid, raw_smp_processor_id()); + return BAD_STATE_E; + } + if (pstate->fpu_state & WC_FPU_INHIBITED_FLAG) { if (flags & WC_SVR_FLAG_INHIBIT) { /* allow recursive inhibit calls as long as the whole stack of * them is inhibiting. @@ -357,15 +364,12 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags) else return WC_ACCEL_INHIBIT_E; } - if (unlikely(flags & WC_SVR_FLAG_INHIBIT)) - return BAD_STATE_E; - if (unlikely((pstate->fpu_state & WC_FPU_COUNT_MASK) - == WC_FPU_COUNT_MASK)) - { - pr_err("ERROR: wc_save_vector_registers_x86 recursion register overflow for " - "pid %d on CPU %d.\n", pstate->pid, raw_smp_processor_id()); - return BAD_STATE_E; - } else { + if (flags & WC_SVR_FLAG_INHIBIT) { + ++pstate->fpu_state; + pstate->fpu_state |= WC_FPU_INHIBITED_FLAG; + return 0; + } + else { ++pstate->fpu_state; return 0; } @@ -475,7 +479,7 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags) __builtin_unreachable(); } -void wc_restore_vector_registers_x86(void) +void wc_restore_vector_registers_x86(enum wc_svr_flags flags) { struct wc_thread_fpu_count_ent *pstate; @@ -494,6 +498,14 @@ void wc_restore_vector_registers_x86(void) } if ((--pstate->fpu_state & WC_FPU_COUNT_MASK) > 0U) { + if (flags & WC_SVR_FLAG_INHIBIT) { + if (pstate->fpu_state & WC_FPU_INHIBITED_FLAG) + pstate->fpu_state &= ~WC_FPU_INHIBITED_FLAG; + else + VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called by pid %d on CPU %d " + "with _INHIBIT flag but saved state isn't _INHIBITED_.\n", task_pid_nr(current), + raw_smp_processor_id()); + } return; } @@ -505,6 +517,10 @@ void wc_restore_vector_registers_x86(void) #endif local_bh_enable(); } else if (unlikely(pstate->fpu_state & WC_FPU_INHIBITED_FLAG)) { + if (unlikely(! (flags & WC_SVR_FLAG_INHIBIT))) + VRG_PR_WARN_X("BUG: wc_restore_vector_registers_x86() called by pid %d on CPU %d " + "without _INHIBIT flag but saved state is _INHIBITED_.\n", task_pid_nr(current), + raw_smp_processor_id()); pstate->fpu_state = 0U; wc_linuxkm_fpu_state_release(pstate); local_bh_enable();