diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 3e039f6fc..8f41ad601 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -127,6 +127,7 @@ CONFIG_POSIX_API CONFIG_POSIX_THREADS CONFIG_PREEMPT_COUNT CONFIG_PTHREAD_IPC +CONFIG_SCHED_INFO CONFIG_SMP CONFIG_SNTP_TIME_SYNC_METHOD_SMOOTH CONFIG_TIMER_TASK_STACK_DEPTH diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 96e3be3a1..78dbdb84d 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -126,6 +126,15 @@ extern int wc_lkm_LockMutex(struct wolfSSL_Mutex* m); #endif + #ifndef WC_LINUXKM_INTR_SIGNALS + #define WC_LINUXKM_INTR_SIGNALS { SIGKILL, SIGABRT, SIGHUP, SIGINT } + #endif + extern int wc_linuxkm_check_for_intr_signals(void); + #ifndef WC_LINUXKM_MAX_NS_WITHOUT_YIELD + #define WC_LINUXKM_MAX_NS_WITHOUT_YIELD 1000000000 + #endif + extern void wc_linuxkm_relax_long_loop(void); + #ifdef BUILDING_WOLFSSL #if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)) || \ @@ -351,6 +360,8 @@ #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) /* for signal_pending() */ #include + /* for local_clock() */ + #include #endif #include @@ -424,6 +435,13 @@ #endif #endif + #ifndef WC_CHECK_FOR_INTR_SIGNALS + #define WC_CHECK_FOR_INTR_SIGNALS() wc_linuxkm_check_for_intr_signals() + #endif + #ifndef WC_RELAX_LONG_LOOP + #define WC_RELAX_LONG_LOOP() wc_linuxkm_relax_long_loop() + #endif + /* benchmarks.c uses floating point math, so needs a working * SAVE_VECTOR_REGISTERS(). */ @@ -875,6 +893,9 @@ typeof(wc_lkm_LockMutex) *wc_lkm_LockMutex; #endif + typeof(wc_linuxkm_check_for_intr_signals) *wc_linuxkm_check_for_intr_signals; + typeof(wc_linuxkm_relax_long_loop) *wc_linuxkm_relax_long_loop; + const void *_last_slot; }; @@ -1099,6 +1120,9 @@ */ #define spin_unlock_irqrestore(lock, flags) raw_spin_unlock_irqrestore(&((lock)->rlock), flags) + #define wc_linuxkm_check_for_intr_signals WC_LKM_INDIRECT_SYM(wc_linuxkm_check_for_intr_signals) + #define wc_linuxkm_relax_long_loop WC_LKM_INDIRECT_SYM(wc_linuxkm_relax_long_loop) + #endif /* __PIE__ */ #endif /* USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE */ diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index 8511ac421..b62b82a83 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -310,6 +310,7 @@ static int linuxkm_lkcapi_sysfs_deinstall(void) { return 0; } +static volatile int linuxkm_lkcapi_registering_now = 0; static int linuxkm_lkcapi_registered = 0; static int linuxkm_lkcapi_n_registered = 0; @@ -318,9 +319,11 @@ static int linuxkm_lkcapi_register(void) int ret = -1; int seen_err = 0; + linuxkm_lkcapi_registering_now = 1; + ret = linuxkm_lkcapi_sysfs_install(); if (ret) - return ret; + goto out; #if defined(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) || \ defined(CONFIG_CRYPTO_SELFTESTS_FULL) @@ -704,11 +707,14 @@ static int linuxkm_lkcapi_register(void) if (ret == -1) { /* no installations occurred */ - if (linuxkm_lkcapi_registered) - return -EEXIST; + if (linuxkm_lkcapi_registered) { + ret = -EEXIST; + goto out; + } else { linuxkm_lkcapi_registered = 1; - return 0; + ret = 0; + goto out; } } else { @@ -716,8 +722,15 @@ static int linuxkm_lkcapi_register(void) * occurred. */ linuxkm_lkcapi_registered = 1; - return seen_err; + ret = seen_err; + goto out; } + +out: + + linuxkm_lkcapi_registering_now = 0; + + return ret; } static int linuxkm_lkcapi_unregister(void) diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index 3d002e5d9..e0674868a 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -198,6 +198,66 @@ WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_deinstall_node(struct kobj_attri #include "linuxkm/lkcapi_glue.c" #endif +int wc_linuxkm_check_for_intr_signals(void) { + static const int intr_signals[] = WC_LINUXKM_INTR_SIGNALS; + if (preempt_count() != 0) + return 0; + +#if defined(HAVE_FIPS) && defined(LINUXKM_LKCAPI_REGISTER) + /* ignore signals during FIPS startup sequence -- failed alg tests cause + * kernel panics on FIPS kernels. + */ + if (linuxkm_lkcapi_registering_now) + return 0; +#endif + if (signal_pending(current)) { + int i; + for (i = 0; + i < (int)sizeof(intr_signals) / (int)sizeof(intr_signals[0]); + ++i) + { + if (sigismember(¤t->pending.signal, intr_signals[i])) { +#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG + pr_err("INFO: wc_linuxkm_check_for_intr_signals returning " + "INTERRUPTED_E on signal %d\n", intr_signals[i]); +#endif + return INTERRUPTED_E; + } + } + } + return 0; +} + +void wc_linuxkm_relax_long_loop(void) { + #if WC_LINUXKM_MAX_NS_WITHOUT_YIELD >= 0 + if (preempt_count() == 0) { + #if (WC_LINUXKM_MAX_NS_WITHOUT_YIELD == 0) || !defined(CONFIG_SCHED_INFO) + cond_resched(); + #else + /* note that local_clock() wraps a local_clock_noinstr() in a + * preempt_disable_notrace(), which sounds expensive but isn't -- + * preempt_disable_notrace() is actually just a nonlocking integer + * increment of current_thread_info()->preempt.count, protected only by + * various compiler optimizer barriers. + */ + u64 now = local_clock(); + u64 current_last_arrival = current->sched_info.last_arrival; + s64 delta = (s64)(now - current_last_arrival); + if (delta > WC_LINUXKM_MAX_NS_WITHOUT_YIELD) { + cond_resched(); + /* if nothing else is runnable, cond_resched() is a no-op and + * doesn't even update .last_arrival. we could force update by + * sleeping, but there's no need. we've been nice enough by just + * cond_resched()ing, and it's actually preferable to call + * cond_resched() frequently once computation has looped + * continuously for longer than WC_LINUXKM_MAX_NS_WITHOUT_YIELD. + */ + } + #endif + } + #endif +} + #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) #include "linuxkm/x86_vector_register_glue.c" #endif @@ -745,6 +805,9 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { wolfssl_linuxkm_pie_redirect_table.queued_spin_lock_slowpath = queued_spin_lock_slowpath; #endif + wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_check_for_intr_signals = wc_linuxkm_check_for_intr_signals; + wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_relax_long_loop = wc_linuxkm_relax_long_loop; + /* runtime assert that the table has no null slots after initialization. */ { unsigned long *i; diff --git a/linuxkm/x86_vector_register_glue.c b/linuxkm/x86_vector_register_glue.c index a6f6ebbac..68a0e230d 100644 --- a/linuxkm/x86_vector_register_glue.c +++ b/linuxkm/x86_vector_register_glue.c @@ -372,6 +372,14 @@ WARN_UNUSED_RESULT int save_vector_registers_x86(enum wc_svr_flags flags) __builtin_unreachable(); } + { + int ret = WC_CHECK_FOR_INTR_SIGNALS(); + if (ret) + return ret; + } + + WC_RELAX_LONG_LOOP(); + if (flags & WC_SVR_FLAG_INHIBIT) { if ((preempt_count() != 0) && !may_use_simd()) return WC_ACCEL_INHIBIT_E; /* not an error here, just a @@ -507,5 +515,7 @@ void restore_vector_registers_x86(void) migrate_enable(); #endif + WC_RELAX_LONG_LOOP(); + return; } diff --git a/wolfcrypt/src/curve25519.c b/wolfcrypt/src/curve25519.c index 1b383e7e6..41493ce95 100644 --- a/wolfcrypt/src/curve25519.c +++ b/wolfcrypt/src/curve25519.c @@ -60,9 +60,9 @@ #if defined(WOLFSSL_LINUXKM) && !defined(USE_INTEL_SPEEDUP) /* force off unneeded vector register save/restore. */ #undef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #undef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif const curve25519_set_type curve25519_sets[] = { diff --git a/wolfcrypt/src/dh.c b/wolfcrypt/src/dh.c index d27138ad7..e41dd73f4 100644 --- a/wolfcrypt/src/dh.c +++ b/wolfcrypt/src/dh.c @@ -60,9 +60,9 @@ #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM) /* force off unneeded vector register save/restore. */ #undef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #undef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif /* diff --git a/wolfcrypt/src/dsa.c b/wolfcrypt/src/dsa.c index cc455487e..2bbf4dbda 100644 --- a/wolfcrypt/src/dsa.c +++ b/wolfcrypt/src/dsa.c @@ -38,9 +38,9 @@ #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM) /* force off unneeded vector register save/restore. */ #undef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #undef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif #ifdef _MSC_VER diff --git a/wolfcrypt/src/ecc.c b/wolfcrypt/src/ecc.c index 03eaf138b..b90883029 100644 --- a/wolfcrypt/src/ecc.c +++ b/wolfcrypt/src/ecc.c @@ -224,9 +224,9 @@ ECC Curve Sizes: #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM) /* force off unneeded vector register save/restore. */ #undef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #undef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif #if !defined(WOLFSSL_ATECC508A) && !defined(WOLFSSL_ATECC608A) && \ diff --git a/wolfcrypt/src/eccsi.c b/wolfcrypt/src/eccsi.c index 2717607d3..db3ea2f56 100644 --- a/wolfcrypt/src/eccsi.c +++ b/wolfcrypt/src/eccsi.c @@ -39,9 +39,9 @@ #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM) /* force off unneeded vector register save/restore. */ #undef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #undef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif #ifndef WOLFSSL_HAVE_ECC_KEY_GET_PRIV diff --git a/wolfcrypt/src/error.c b/wolfcrypt/src/error.c index 3713dbaed..1d5c90d10 100644 --- a/wolfcrypt/src/error.c +++ b/wolfcrypt/src/error.c @@ -653,6 +653,9 @@ const char* wc_GetErrorString(int error) case BAD_INDEX_E: return "Bad index"; + case INTERRUPTED_E: + return "Process interrupted"; + case MAX_CODE_E: case WC_SPAN1_MIN_CODE_E: case MIN_CODE_E: diff --git a/wolfcrypt/src/rsa.c b/wolfcrypt/src/rsa.c index 488a7213e..e53d97037 100644 --- a/wolfcrypt/src/rsa.c +++ b/wolfcrypt/src/rsa.c @@ -59,9 +59,9 @@ RSA keys can be used to encrypt, decrypt, sign and verify data. #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM) /* force off unneeded vector register save/restore. */ #undef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #undef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif /* diff --git a/wolfcrypt/src/sakke.c b/wolfcrypt/src/sakke.c index 3e8db9231..35678ddac 100644 --- a/wolfcrypt/src/sakke.c +++ b/wolfcrypt/src/sakke.c @@ -40,9 +40,9 @@ #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM) /* force off unneeded vector register save/restore. */ #undef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #undef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif #ifndef WOLFSSL_HAVE_ECC_KEY_GET_PRIV diff --git a/wolfcrypt/src/sp_int.c b/wolfcrypt/src/sp_int.c index ebccf39f2..b5c832d3e 100644 --- a/wolfcrypt/src/sp_int.c +++ b/wolfcrypt/src/sp_int.c @@ -111,9 +111,9 @@ This library provides single precision (SP) integer math functions. #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM) /* force off unneeded vector register save/restore. */ #undef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #undef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif /* DECL_SP_INT: Declare one variable of type 'sp_int'. */ diff --git a/wolfssl/wolfcrypt/error-crypt.h b/wolfssl/wolfcrypt/error-crypt.h index 87692ddce..74e441c6a 100644 --- a/wolfssl/wolfcrypt/error-crypt.h +++ b/wolfssl/wolfcrypt/error-crypt.h @@ -306,9 +306,10 @@ enum wolfCrypt_ErrorCodes { ASCON_AUTH_E = -1001, /* ASCON Authentication check failure */ WC_ACCEL_INHIBIT_E = -1002, /* Crypto acceleration is currently inhibited */ BAD_INDEX_E = -1003, /* Bad index */ + INTERRUPTED_E = -1004, /* Process interrupted */ - WC_SPAN2_LAST_E = -1003, /* Update to indicate last used error code */ - WC_LAST_E = -1003, /* the last code used either here or in + WC_SPAN2_LAST_E = -1004, /* Update to indicate last used error code */ + WC_LAST_E = -1004, /* the last code used either here or in * error-ssl.h */ WC_SPAN2_MIN_CODE_E = -1999, /* Last usable code in span 2 */ diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 01ca1b7fe..5ae526dfc 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -1935,8 +1935,32 @@ WOLFSSL_API word32 CheckRunTimeSettings(void); #define wc_static_assert2(expr, msg) wc_static_assert(expr) #endif +#ifndef WC_RELAX_LONG_LOOP + #define WC_RELAX_LONG_LOOP() WC_DO_NOTHING +#endif +#ifndef WC_CHECK_FOR_INTR_SIGNALS + #define WC_CHECK_FOR_INTR_SIGNALS() 0 + #ifndef SAVE_NO_VECTOR_REGISTERS + #define SAVE_NO_VECTOR_REGISTERS(fail_clause) WC_RELAX_LONG_LOOP() + #endif +#else + #ifndef SAVE_NO_VECTOR_REGISTERS + #define SAVE_NO_VECTOR_REGISTERS(fail_clause) { \ + int _svr_ret = WC_CHECK_FOR_INTR_SIGNALS(); \ + if (_svr_ret != 0) { fail_clause } \ + WC_RELAX_LONG_LOOP(); \ + } + #endif +#endif +#ifndef SAVE_NO_VECTOR_REGISTERS2 + #define SAVE_NO_VECTOR_REGISTERS2() 0 +#endif +#ifndef RESTORE_NO_VECTOR_REGISTERS + #define RESTORE_NO_VECTOR_REGISTERS() WC_RELAX_LONG_LOOP() +#endif + #ifndef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING + #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause) #endif #ifndef SAVE_VECTOR_REGISTERS2 #define SAVE_VECTOR_REGISTERS2() 0 @@ -1956,8 +1980,9 @@ WOLFSSL_API word32 CheckRunTimeSettings(void); #define ASSERT_RESTORED_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING #endif #ifndef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING + #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS() #endif + #ifdef WOLFSSL_NO_ASM /* We define fallback no-op definitions for these only if asm is disabled, * otherwise the using code must detect that these macros are undefined and