Merge pull request #9035 from douzzer/20250725-wc_linuxkm_relax_long_loop

20250725-wc_linuxkm_relax_long_loop

.wolfssl_known_macro_extras

@@ -127,6 +127,7 @@ CONFIG_POSIX_API
 CONFIG_POSIX_THREADS
 CONFIG_PREEMPT_COUNT
 CONFIG_PTHREAD_IPC
+CONFIG_SCHED_INFO
 CONFIG_SMP
 CONFIG_SNTP_TIME_SYNC_METHOD_SMOOTH
 CONFIG_TIMER_TASK_STACK_DEPTH

linuxkm/linuxkm_wc_port.h

@@ -126,6 +126,15 @@
     extern int wc_lkm_LockMutex(struct wolfSSL_Mutex* m);
 #endif
 
+    #ifndef WC_LINUXKM_INTR_SIGNALS
+        #define WC_LINUXKM_INTR_SIGNALS { SIGKILL, SIGABRT, SIGHUP, SIGINT }
+    #endif
+    extern int wc_linuxkm_check_for_intr_signals(void);
+    #ifndef WC_LINUXKM_MAX_NS_WITHOUT_YIELD
+        #define WC_LINUXKM_MAX_NS_WITHOUT_YIELD 1000000000
+    #endif
+    extern void wc_linuxkm_relax_long_loop(void);
+
     #ifdef BUILDING_WOLFSSL
 
     #if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)) || \
@@ -351,6 +360,8 @@
     #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
         /* for signal_pending() */
         #include <linux/sched/signal.h>
+        /* for local_clock() */
+        #include <linux/sched/clock.h>
     #endif
     #include <linux/random.h>
 
@@ -424,6 +435,13 @@
         #endif
     #endif
 
+    #ifndef WC_CHECK_FOR_INTR_SIGNALS
+        #define WC_CHECK_FOR_INTR_SIGNALS() wc_linuxkm_check_for_intr_signals()
+    #endif
+    #ifndef WC_RELAX_LONG_LOOP
+        #define WC_RELAX_LONG_LOOP() wc_linuxkm_relax_long_loop()
+    #endif
+
     /* benchmarks.c uses floating point math, so needs a working
      * SAVE_VECTOR_REGISTERS().
      */
@@ -875,6 +893,9 @@
         typeof(wc_lkm_LockMutex) *wc_lkm_LockMutex;
         #endif
 
+        typeof(wc_linuxkm_check_for_intr_signals) *wc_linuxkm_check_for_intr_signals;
+        typeof(wc_linuxkm_relax_long_loop) *wc_linuxkm_relax_long_loop;
+
         const void *_last_slot;
     };
 
@@ -1099,6 +1120,9 @@
      */
     #define spin_unlock_irqrestore(lock, flags) raw_spin_unlock_irqrestore(&((lock)->rlock), flags)
 
+    #define wc_linuxkm_check_for_intr_signals WC_LKM_INDIRECT_SYM(wc_linuxkm_check_for_intr_signals)
+    #define wc_linuxkm_relax_long_loop WC_LKM_INDIRECT_SYM(wc_linuxkm_relax_long_loop)
+
     #endif /* __PIE__ */
 
     #endif /* USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE */

linuxkm/lkcapi_glue.c

@@ -310,6 +310,7 @@ static int linuxkm_lkcapi_sysfs_deinstall(void) {
     return 0;
 }
 
+static volatile int linuxkm_lkcapi_registering_now = 0;
 static int linuxkm_lkcapi_registered = 0;
 static int linuxkm_lkcapi_n_registered = 0;
 
@@ -318,9 +319,11 @@ static int linuxkm_lkcapi_register(void)
     int ret = -1;
     int seen_err = 0;
 
+    linuxkm_lkcapi_registering_now = 1;
+
     ret = linuxkm_lkcapi_sysfs_install();
     if (ret)
-        return ret;
+        goto out;
 
 #if defined(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) || \
     defined(CONFIG_CRYPTO_SELFTESTS_FULL)
@@ -704,11 +707,14 @@ static int linuxkm_lkcapi_register(void)
 
     if (ret == -1) {
         /* no installations occurred */
-        if (linuxkm_lkcapi_registered)
-            return -EEXIST;
+        if (linuxkm_lkcapi_registered) {
+            ret = -EEXIST;
+            goto out;
+        }
         else {
             linuxkm_lkcapi_registered = 1;
-            return 0;
+            ret = 0;
+            goto out;
         }
     }
     else {
@@ -716,8 +722,15 @@ static int linuxkm_lkcapi_register(void)
          * occurred.
          */
         linuxkm_lkcapi_registered = 1;
-        return seen_err;
+        ret = seen_err;
+        goto out;
     }
+
+out:
+
+    linuxkm_lkcapi_registering_now = 0;
+
+    return ret;
 }
 
 static int linuxkm_lkcapi_unregister(void)

linuxkm/module_hooks.c

@@ -198,6 +198,66 @@ WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_deinstall_node(struct kobj_attri
     #include "linuxkm/lkcapi_glue.c"
 #endif
 
+int wc_linuxkm_check_for_intr_signals(void) {
+    static const int intr_signals[] = WC_LINUXKM_INTR_SIGNALS;
+    if (preempt_count() != 0)
+        return 0;
+
+#if defined(HAVE_FIPS) && defined(LINUXKM_LKCAPI_REGISTER)
+    /* ignore signals during FIPS startup sequence -- failed alg tests cause
+     * kernel panics on FIPS kernels.
+     */
+    if (linuxkm_lkcapi_registering_now)
+        return 0;
+#endif
+    if (signal_pending(current)) {
+        int i;
+        for (i = 0;
+             i < (int)sizeof(intr_signals) / (int)sizeof(intr_signals[0]);
+             ++i)
+        {
+            if (sigismember(&current->pending.signal, intr_signals[i])) {
+#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
+                pr_err("INFO: wc_linuxkm_check_for_intr_signals returning "
+                       "INTERRUPTED_E on signal %d\n", intr_signals[i]);
+#endif
+                return INTERRUPTED_E;
+            }
+        }
+    }
+    return 0;
+}
+
+void wc_linuxkm_relax_long_loop(void) {
+    #if WC_LINUXKM_MAX_NS_WITHOUT_YIELD >= 0
+    if (preempt_count() == 0) {
+        #if (WC_LINUXKM_MAX_NS_WITHOUT_YIELD == 0) || !defined(CONFIG_SCHED_INFO)
+        cond_resched();
+        #else
+        /* note that local_clock() wraps a local_clock_noinstr() in a
+         * preempt_disable_notrace(), which sounds expensive but isn't --
+         * preempt_disable_notrace() is actually just a nonlocking integer
+         * increment of current_thread_info()->preempt.count, protected only by
+         * various compiler optimizer barriers.
+         */
+        u64 now = local_clock();
+        u64 current_last_arrival = current->sched_info.last_arrival;
+        s64 delta = (s64)(now - current_last_arrival);
+        if (delta > WC_LINUXKM_MAX_NS_WITHOUT_YIELD) {
+            cond_resched();
+            /* if nothing else is runnable, cond_resched() is a no-op and
+             * doesn't even update .last_arrival.  we could force update by
+             * sleeping, but there's no need.  we've been nice enough by just
+             * cond_resched()ing, and it's actually preferable to call
+             * cond_resched() frequently once computation has looped
+             * continuously for longer than WC_LINUXKM_MAX_NS_WITHOUT_YIELD.
+             */
+        }
+        #endif
+    }
+    #endif
+}
+
 #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
     #include "linuxkm/x86_vector_register_glue.c"
 #endif
@@ -745,6 +805,9 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     wolfssl_linuxkm_pie_redirect_table.queued_spin_lock_slowpath = queued_spin_lock_slowpath;
 #endif
 
+    wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_check_for_intr_signals = wc_linuxkm_check_for_intr_signals;
+    wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_relax_long_loop = wc_linuxkm_relax_long_loop;
+
     /* runtime assert that the table has no null slots after initialization. */
     {
         unsigned long *i;

linuxkm/x86_vector_register_glue.c

@@ -372,6 +372,14 @@ WARN_UNUSED_RESULT int save_vector_registers_x86(enum wc_svr_flags flags)
         __builtin_unreachable();
     }
 
+    {
+        int ret = WC_CHECK_FOR_INTR_SIGNALS();
+        if (ret)
+            return ret;
+    }
+
+    WC_RELAX_LONG_LOOP();
+
     if (flags & WC_SVR_FLAG_INHIBIT) {
         if ((preempt_count() != 0) && !may_use_simd())
             return WC_ACCEL_INHIBIT_E; /* not an error here, just a
@@ -507,5 +515,7 @@ void restore_vector_registers_x86(void)
     migrate_enable();
     #endif
 
+    WC_RELAX_LONG_LOOP();
+
     return;
 }

wolfcrypt/src/curve25519.c

@@ -60,9 +60,9 @@
 #if defined(WOLFSSL_LINUXKM) && !defined(USE_INTEL_SPEEDUP)
     /* force off unneeded vector register save/restore. */
     #undef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
     #undef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
 
 const curve25519_set_type curve25519_sets[] = {

wolfcrypt/src/dh.c

@@ -60,9 +60,9 @@
 #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM)
     /* force off unneeded vector register save/restore. */
     #undef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
     #undef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
 
 /*

wolfcrypt/src/dsa.c

@@ -38,9 +38,9 @@
 #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM)
     /* force off unneeded vector register save/restore. */
     #undef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
     #undef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
 
 #ifdef _MSC_VER

wolfcrypt/src/ecc.c

@@ -224,9 +224,9 @@ ECC Curve Sizes:
 #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM)
     /* force off unneeded vector register save/restore. */
     #undef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
     #undef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
 
 #if !defined(WOLFSSL_ATECC508A) && !defined(WOLFSSL_ATECC608A) && \

wolfcrypt/src/eccsi.c

@@ -39,9 +39,9 @@
 #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM)
     /* force off unneeded vector register save/restore. */
     #undef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
     #undef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
 
 #ifndef WOLFSSL_HAVE_ECC_KEY_GET_PRIV

wolfcrypt/src/error.c

@@ -653,6 +653,9 @@ const char* wc_GetErrorString(int error)
     case BAD_INDEX_E:
         return "Bad index";
 
+    case INTERRUPTED_E:
+        return "Process interrupted";
+
     case MAX_CODE_E:
     case WC_SPAN1_MIN_CODE_E:
     case MIN_CODE_E:

wolfcrypt/src/rsa.c

@@ -59,9 +59,9 @@ RSA keys can be used to encrypt, decrypt, sign and verify data.
 #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM)
     /* force off unneeded vector register save/restore. */
     #undef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
     #undef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
 
 /*

wolfcrypt/src/sakke.c

@@ -40,9 +40,9 @@
 #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM)
     /* force off unneeded vector register save/restore. */
     #undef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
     #undef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
 
 #ifndef WOLFSSL_HAVE_ECC_KEY_GET_PRIV

wolfcrypt/src/sp_int.c

@@ -111,9 +111,9 @@ This library provides single precision (SP) integer math functions.
 #if defined(WOLFSSL_LINUXKM) && !defined(WOLFSSL_SP_ASM)
     /* force off unneeded vector register save/restore. */
     #undef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
     #undef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
 
 /* DECL_SP_INT: Declare one variable of type 'sp_int'. */

wolfssl/wolfcrypt/error-crypt.h

@@ -306,9 +306,10 @@ enum wolfCrypt_ErrorCodes {
     ASCON_AUTH_E        = -1001, /* ASCON Authentication check failure */
     WC_ACCEL_INHIBIT_E  = -1002, /* Crypto acceleration is currently inhibited */
     BAD_INDEX_E         = -1003, /* Bad index */
+    INTERRUPTED_E       = -1004, /* Process interrupted */
 
-    WC_SPAN2_LAST_E     = -1003, /* Update to indicate last used error code */
-    WC_LAST_E           = -1003, /* the last code used either here or in
+    WC_SPAN2_LAST_E     = -1004, /* Update to indicate last used error code */
+    WC_LAST_E           = -1004, /* the last code used either here or in
                                   * error-ssl.h */
 
     WC_SPAN2_MIN_CODE_E = -1999, /* Last usable code in span 2 */

wolfssl/wolfcrypt/types.h

@@ -1935,8 +1935,32 @@ WOLFSSL_API word32 CheckRunTimeSettings(void);
         #define wc_static_assert2(expr, msg) wc_static_assert(expr)
 #endif
 
+#ifndef WC_RELAX_LONG_LOOP
+    #define WC_RELAX_LONG_LOOP() WC_DO_NOTHING
+#endif
+#ifndef WC_CHECK_FOR_INTR_SIGNALS
+    #define WC_CHECK_FOR_INTR_SIGNALS() 0
+    #ifndef SAVE_NO_VECTOR_REGISTERS
+        #define SAVE_NO_VECTOR_REGISTERS(fail_clause) WC_RELAX_LONG_LOOP()
+    #endif
+#else
+    #ifndef SAVE_NO_VECTOR_REGISTERS
+        #define SAVE_NO_VECTOR_REGISTERS(fail_clause) {     \
+                int _svr_ret = WC_CHECK_FOR_INTR_SIGNALS(); \
+                if (_svr_ret != 0) { fail_clause }          \
+                WC_RELAX_LONG_LOOP();                       \
+            }
+    #endif
+#endif
+#ifndef SAVE_NO_VECTOR_REGISTERS2
+    #define SAVE_NO_VECTOR_REGISTERS2() 0
+#endif
+#ifndef RESTORE_NO_VECTOR_REGISTERS
+    #define RESTORE_NO_VECTOR_REGISTERS() WC_RELAX_LONG_LOOP()
+#endif
+
 #ifndef SAVE_VECTOR_REGISTERS
-    #define SAVE_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
+    #define SAVE_VECTOR_REGISTERS(fail_clause) SAVE_NO_VECTOR_REGISTERS(fail_clause)
 #endif
 #ifndef SAVE_VECTOR_REGISTERS2
     #define SAVE_VECTOR_REGISTERS2() 0
@@ -1956,8 +1980,9 @@ WOLFSSL_API word32 CheckRunTimeSettings(void);
     #define ASSERT_RESTORED_VECTOR_REGISTERS(fail_clause) WC_DO_NOTHING
 #endif
 #ifndef RESTORE_VECTOR_REGISTERS
-    #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING
+    #define RESTORE_VECTOR_REGISTERS() RESTORE_NO_VECTOR_REGISTERS()
 #endif
+
 #ifdef WOLFSSL_NO_ASM
     /* We define fallback no-op definitions for these only if asm is disabled,
      * otherwise the using code must detect that these macros are undefined and