Merge pull request #4407 from douzzer/linuxkm-SIMD-IRQ

linuxkm-SIMD-IRQ

configure.ac

@@ -180,6 +180,42 @@ AS_IF([test "$ax_enable_debug" = "yes"],
 ENABLED_CERTS="no"
 
 
+# Support for forcing 32-bit mode
+# To force 32-bit instructions use:
+#   ./configure CFLAGS="-m32" LDFLAGS="-m32" && make
+# The checks for sizeof long and long/long are run at the top of configure and require "-m32" to be set directly in the ./configure statement.
+AC_ARG_ENABLE([32bit],
+    [AS_HELP_STRING([--enable-32bit],[Enables 32-bit support (default: disabled)])],
+    [ ENABLED_32BIT=$enableval ],
+    [ ENABLED_32BIT=no ]
+    )
+
+# 16-bit compiler support
+AC_ARG_ENABLE([16bit],
+    [AS_HELP_STRING([--enable-16bit],[Enables 16-bit support (default: disabled)])],
+    [ ENABLED_16BIT=$enableval ],
+    [ ENABLED_16BIT=no ]
+    )
+if test "$ENABLED_16BIT" = "yes"
+then
+    AM_CFLAGS="$AM_CFLAGS -DWC_16BIT_CPU"
+fi
+
+
+
+# Support for disabling all ASM
+AC_ARG_ENABLE([asm],
+    [AS_HELP_STRING([--enable-asm],[Enables option for assembly (default: enabled)])],
+    [ ENABLED_ASM=$enableval ],
+    [ ENABLED_ASM=yes ]
+    )
+
+if test "$ENABLED_ASM" = "no"
+then
+    AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM"
+fi
+AC_SUBST([ENABLED_ASM])
+
 
 # FIPS
 AC_ARG_ENABLE([fips],
@@ -373,9 +409,7 @@ then
     test "$enable_aescbc_length_checks" = "" && enable_aescbc_length_checks=yes
     test "$enable_camellia" = "" && enable_camellia=yes
     test "$enable_ripemd" = "" && enable_ripemd=yes
-    test "$enable_sha512" = "" && enable_sha512=yes
     test "$enable_sha224" = "" && enable_sha224=yes
-    test "$enable_sha3" = "" && enable_sha3=yes
     test "$enable_sessioncerts" = "" && enable_sessioncerts=yes
     test "$enable_keygen" = "" && enable_keygen=yes
     test "$enable_certgen" = "" && enable_certgen=yes
@@ -431,6 +465,12 @@ then
     test "$enable_anon" = "" && enable_anon=yes
     test "$enable_mcast" = "" && enable_mcast=yes
 
+    if test "$ENABLED_32BIT" != "yes"
+    then
+        test "$enable_sha512" = "" && enable_sha512=yes
+        test "$enable_sha3" = "" && enable_sha3=yes
+    fi
+
     if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
     then
         test "$enable_compkey" = "" && enable_compkey=yes
@@ -447,8 +487,11 @@ then
         if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
         then
             if test "$ENABLED_FIPS" = "no"
+            then
+                if test "$ENABLED_32BIT" != "yes"
                 then
                     test "$enable_openssh" = "" && enable_openssh=yes
+                fi
                 # S/MIME support requires PKCS7, which requires no FIPS.
                 test "$enable_smime" = "" && enable_smime=yes
             fi
@@ -474,10 +517,13 @@ then
         test "$enable_xchacha" = "" && enable_xchacha=yes
         test "$enable_scep" = "" && enable_scep=yes
         test "$enable_pkcs7" = "" && enable_pkcs7=yes
+        if test "$ENABLED_32BIT" != "yes"
+        then
             test "$enable_ed25519" = "" && enable_ed25519=yes
             test "$enable_ed25519_stream" = "" && enable_ed25519_stream=yes
             test "$enable_ed448" = "" && enable_ed448=yes
             test "$enable_ed448_stream" = "" && enable_ed448_stream=yes
+        fi
 
         if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
         then
@@ -525,9 +571,7 @@ then
     test "$enable_aescfb" = "" && enable_aescfb=yes
     test "$enable_camellia" = "" && enable_camellia=yes
     test "$enable_ripemd" = "" && enable_ripemd=yes
-    test "$enable_sha512" = "" && enable_sha512=yes
     test "$enable_sha224" = "" && enable_sha224=yes
-    test "$enable_sha3" = "" && enable_sha3=yes
     test "$enable_sessioncerts" = "" && enable_sessioncerts=yes
     test "$enable_keygen" = "" && enable_keygen=yes
     test "$enable_certgen" = "" && enable_certgen=yes
@@ -569,6 +613,12 @@ then
     test "$enable_cryptocb" = "" && enable_cryptocb=yes
     test "$enable_anon" = "" && enable_anon=yes
 
+    if test "$ENABLED_32BIT" != "yes"
+    then
+        test "$enable_sha512" = "" && enable_sha512=yes
+        test "$enable_sha3" = "" && enable_sha3=yes
+    fi
+
     if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
     then
         test "$enable_compkey" = "" && enable_compkey=yes
@@ -586,11 +636,14 @@ then
     if test "$ENABLED_FIPS" = "no"
     then
         test "$enable_xchacha" = "" && enable_xchacha=yes
+        test "$enable_pkcs7" = "" && enable_pkcs7=yes
+        if test "$ENABLED_32BIT" != "yes"
+        then
             test "$enable_ed25519" = "" && enable_ed25519=yes
             test "$enable_ed25519_stream" = "" && enable_ed25519_stream=yes
             test "$enable_ed448" = "" && enable_ed448=yes
             test "$enable_ed448_stream" = "" && enable_ed448_stream=yes
-        test "$enable_pkcs7" = "" && enable_pkcs7=yes
+        fi
 
         if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
         then
@@ -615,43 +668,6 @@ then
 fi
 
 
-# Support for forcing 32-bit mode
-# To force 32-bit instructions use:
-#   ./configure CFLAGS="-m32" LDFLAGS="-m32" && make
-# The checks for sizeof long and long/long are run at the top of configure and require "-m32" to be set directly in the ./configure statement.
-AC_ARG_ENABLE([32bit],
-    [AS_HELP_STRING([--enable-32bit],[Enables 32-bit support (default: disabled)])],
-    [ ENABLED_32BIT=$enableval ],
-    [ ENABLED_32BIT=no ]
-    )
-
-# 16-bit compiler support
-AC_ARG_ENABLE([16bit],
-    [AS_HELP_STRING([--enable-16bit],[Enables 16-bit support (default: disabled)])],
-    [ ENABLED_16BIT=$enableval ],
-    [ ENABLED_16BIT=no ]
-    )
-if test "$ENABLED_16BIT" = "yes"
-then
-    AM_CFLAGS="$AM_CFLAGS -DWC_16BIT_CPU"
-fi
-
-
-
-# Support for disabling all ASM
-AC_ARG_ENABLE([asm],
-    [AS_HELP_STRING([--enable-asm],[Enables option for assembly (default: enabled)])],
-    [ ENABLED_ASM=$enableval ],
-    [ ENABLED_ASM=yes ]
-    )
-
-if test "$ENABLED_ASM" = "no"
-then
-    AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM"
-fi
-AC_SUBST([ENABLED_ASM])
-
-
 # SINGLE THREADED
 AC_ARG_ENABLE([singlethreaded],
     [AS_HELP_STRING([--enable-singlethreaded],[Enable wolfSSL single threaded (default: disabled)])],
@@ -1837,7 +1853,7 @@ fi
 
 # set sha3 default
 SHA3_DEFAULT=no
-if test "$host_cpu" = "x86_64" || test "$host_cpu" = "aarch64"
+if (test "$host_cpu" = "x86_64" || test "$host_cpu" = "aarch64") && test "$ENABLED_32BIT" = "no"
 then
     if test "x$ENABLED_FIPS" = "xno" || test "x$FIPS_VERSION" = "xv2"
     then
@@ -6451,7 +6467,8 @@ AS_IF([test "x$ENABLED_CERT_REQ" = "xyes" && \
 
 # ed25519 requires sha512
 AS_IF([test "x$ENABLED_ED25519" = "xyes" && \
-       test "x$ENABLED_SHA512" = "xno"],
+       test "x$ENABLED_SHA512" = "xno" && \
+       test "x$ENABLED_32BIT" = "xno"],
       [AC_MSG_ERROR([cannot enable ed25519 without enabling sha512.])])
 
 # ed25519 stream requires ed25519
@@ -6942,7 +6959,9 @@ AM_CONDITIONAL([BUILD_OCTEON_SYNC],[test "x$ENABLED_OCTEON_SYNC" = "xyes"])
 AM_CONDITIONAL([BUILD_INTEL_QA],[test "x$ENABLED_INTEL_QA" = "xyes"])
 AM_CONDITIONAL([BUILD_INTEL_QA_SYNC],[test "x$ENABLED_INTEL_QA_SYNC" = "xyes"])
 AM_CONDITIONAL([BUILD_SP],[test "x$ENABLED_SP" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
-AM_CONDITIONAL([BUILD_SP_C],[(test "x$ENABLED_SP" = "xyes" && test "x$ENABLED_SP_ASM" = "xno") || test "x$ENABLED_USERSETTINGS" = "xyes"])
+AM_CONDITIONAL([BUILD_SP_C],[((test "$ENABLED_SP_RSA" = "yes" || test "$ENABLED_SP_DH" = "yes" \
+                               || test "$ENABLED_SP_ECC" = "yes") && test "x$ENABLED_SP_ASM" = "xno") \
+                             || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_SP_ARM64],[test "x$ENABLED_SP_ARM64_ASM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_SP_ARM32],[test "x$ENABLED_SP_ARM32_ASM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_SP_ARM_THUMB],[test "x$ENABLED_SP_ARM_THUMB_ASM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])

linuxkm/Kbuild

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
-#/
+#
 
 SHELL=/bin/bash
 
@@ -48,6 +48,7 @@ $(obj)/linuxkm/module_exports.o: $(WOLFSSL_OBJ_TARGETS)
 # this mechanism only works in kernel 5.x+ (fallback to hardcoded value)
 hostprogs := linuxkm/get_thread_size
 always-y := $(hostprogs)
+always := $(hostprogs)
 HOST_EXTRACFLAGS += $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(KBUILD_CFLAGS) -static
 
 # this rule is needed to get build to succeed in 4.x (get_thread_size still doesn't get built)
@@ -91,14 +92,20 @@ endif
 
 asflags-y := $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPUSIMD_DISABLE)
 
-# vectorized algorithms equipped with {SAVE,RESTORE}_VECTOR_REGISTERS()
+# vectorized algorithms protected by {SAVE,RESTORE}_VECTOR_REGISTERS() can be
-# can be safely included here:
+# safely included here, though many of these are not yet kernel-compatible:
 $(obj)/wolfcrypt/src/aes_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
 $(obj)/wolfcrypt/src/aes_gcm_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
+$(obj)/wolfcrypt/src/sha256_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
+$(obj)/wolfcrypt/src/sp_x86_64_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
+$(obj)/wolfcrypt/src/sha512_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
+$(obj)/wolfcrypt/src/poly1305_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
+$(obj)/wolfcrypt/src/chacha_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
 
-# these _asms are kernel-compatible, but they still irritate objtool:
+# these _asms are known kernel-compatible, but they still irritate objtool:
 $(obj)/wolfcrypt/src/aes_asm.o: OBJECT_FILES_NON_STANDARD := y
 $(obj)/wolfcrypt/src/aes_gcm_asm.o: OBJECT_FILES_NON_STANDARD := y
+$(obj)/wolfcrypt/src/sp_x86_64_asm.o: OBJECT_FILES_NON_STANDARD := y
 
 ifeq "$(ENABLED_LINUXKM_PIE)" "yes"
 
@@ -158,7 +165,7 @@ $(src)/linuxkm/module_exports.c: $(src)/linuxkm/module_exports.c.template $(WOLF
 	@cp $< $@
 	@readelf --symbols --wide $(WOLFSSL_OBJ_TARGETS) |				\
 		awk '/^ *[0-9]+: / {							\
-		  if ($$8 !~ /^(wc_|wolf)/){next;}					\
+		  if ($$8 !~ /^(wc_|wolf|WOLF|TLSX_)/){next;}				\
 		  if (($$4 == "FUNC") && ($$5 == "GLOBAL") && ($$6 == "DEFAULT")) {	\
 		    print "EXPORT_SYMBOL_NS(" $$8 ", WOLFSSL);";    		  	\
 		  }									\

linuxkm/Makefile

@@ -21,7 +21,7 @@
 
 SHELL=/bin/bash
 
-all: libwolfssl.ko
+all: libwolfssl.ko libwolfssl.ko.signed
 
 .PHONY: libwolfssl.ko
 
@@ -61,7 +61,42 @@ libwolfssl.ko:
 	@if test -z "$(src_libwolfssl_la_OBJECTS)"; then echo '$$src_libwolfssl_la_OBJECTS is unset.' >&2; exit 1; fi
 	@mkdir -p linuxkm src wolfcrypt/src wolfcrypt/test
 	@if test ! -h $(SRC_TOP)/Kbuild; then ln -s $(MODULE_TOP)/Kbuild $(SRC_TOP)/Kbuild; fi
+ifeq "$(ENABLED_LINUXKM_PIE)" "yes"
+	+$(MAKE) -C $(KERNEL_ROOT) M=$(MODULE_TOP) src=$(SRC_TOP) CC_FLAGS_FTRACE=
+else
 	+$(MAKE) -C $(KERNEL_ROOT) M=$(MODULE_TOP) src=$(SRC_TOP)
+endif
+
+libwolfssl.ko.signed: libwolfssl.ko
+	@cd '$(KERNEL_ROOT)' || exit $$?;							\
+	while read configline; do								\
+		case "$$configline" in								\
+		CONFIG_MODULE_SIG_KEY=*)							\
+				CONFIG_MODULE_SIG_KEY="$${configline#CONFIG_MODULE_SIG_KEY=}"	\
+			;;									\
+		CONFIG_MODULE_SIG_HASH=*)							\
+				CONFIG_MODULE_SIG_HASH="$${configline#CONFIG_MODULE_SIG_HASH=}"	\
+			;;									\
+		esac;										\
+	done < .config || exit $$?;								\
+	if [[ -n "$${CONFIG_MODULE_SIG_KEY}" && -n "$${CONFIG_MODULE_SIG_HASH}" &&		\
+			( ! -f '$(MODULE_TOP)/$@' ||						\
+			'$(MODULE_TOP)/$<' -nt '$(MODULE_TOP)/$@' ) ]]; then			\
+		CONFIG_MODULE_SIG_KEY="$${CONFIG_MODULE_SIG_KEY#\"}";				\
+		CONFIG_MODULE_SIG_KEY="$${CONFIG_MODULE_SIG_KEY%\"}";				\
+		CONFIG_MODULE_SIG_HASH="$${CONFIG_MODULE_SIG_HASH#\"}";				\
+		CONFIG_MODULE_SIG_HASH="$${CONFIG_MODULE_SIG_HASH%\"}";				\
+		cp -p '$(MODULE_TOP)/$<' '$(MODULE_TOP)/$@' || exit $$?;			\
+		./scripts/sign-file "$${CONFIG_MODULE_SIG_HASH}"				\
+				    "$${CONFIG_MODULE_SIG_KEY}"					\
+				    "$${CONFIG_MODULE_SIG_KEY/%.pem/.x509}"			\
+				    '$(MODULE_TOP)/$@' ||					\
+					$(RM) -f '$(MODULE_TOP)/$@' || exit $$?;		\
+		if [[ "$(quiet)" != "silent_" ]]; then						\
+			echo "  Module $@ signed by $${CONFIG_MODULE_SIG_KEY}.";		\
+		fi										\
+	fi
+
 
 .PHONY: install modules_install
 install modules_install:

linuxkm/module_hooks.c

@@ -19,6 +19,10 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  */
 
+#ifndef WOLFSSL_LICENSE
+#define WOLFSSL_LICENSE "GPL v2"
+#endif
+
 #define FIPS_NO_WRAPPERS
 
 #ifdef HAVE_CONFIG_H
@@ -41,13 +45,13 @@ static int libwolfssl_cleanup(void) {
 #ifdef WOLFCRYPT_ONLY
     ret = wolfCrypt_Cleanup();
     if (ret != 0)
-        pr_err("wolfCrypt_Cleanup() failed: %s", wc_GetErrorString(ret));
+        pr_err("wolfCrypt_Cleanup() failed: %s\n", wc_GetErrorString(ret));
     else
         pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n");
 #else
     ret = wolfSSL_Cleanup();
     if (ret != WOLFSSL_SUCCESS)
-        pr_err("wolfSSL_Cleanup() failed: %s", wc_GetErrorString(ret));
+        pr_err("wolfSSL_Cleanup() failed: %s\n", wc_GetErrorString(ret));
     else
         pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n");
 #endif
@@ -89,8 +93,8 @@ static void lkmFipsCb(int ok, int err, const char* hash)
     if ((! ok) || (err != 0))
         pr_err("libwolfssl FIPS error: %s\n", wc_GetErrorString(err));
     if (err == IN_CORE_FIPS_E) {
-        pr_err("In-core integrity hash check failure.\n");
+        pr_err("In-core integrity hash check failure.\n"
-        pr_err("Update verifyCore[] in fips_test.c with new hash \"%s\" and rebuild.\n",
+               "Update verifyCore[] in fips_test.c with new hash \"%s\" and rebuild.\n",
                hash ? hash : "<null>");
     }
 }
@@ -104,6 +108,13 @@ static int wolfssl_init(void)
 {
     int ret;
 
+#ifdef CONFIG_MODULE_SIG
+    if (THIS_MODULE->sig_ok == false) {
+        pr_err("wolfSSL module load aborted -- bad or missing module signature with CONFIG_MODULE_SIG kernel.\n");
+        return -ECANCELED;
+    }
+#endif
+
 #ifdef USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE
     ret = set_up_wolfssl_linuxkm_pie_redirect_table();
     if (ret < 0)
@@ -112,6 +123,17 @@ static int wolfssl_init(void)
 #endif
 
 #ifdef HAVE_LINUXKM_PIE_SUPPORT
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)
+    #define THIS_MODULE_BASE (THIS_MODULE->core_layout.base)
+    #define THIS_MODULE_TEXT_SIZE (THIS_MODULE->core_layout.text_size)
+    #define THIS_MODULE_RO_SIZE (THIS_MODULE->core_layout.ro_size)
+#else
+    #define THIS_MODULE_BASE (THIS_MODULE->module_core)
+    #define THIS_MODULE_TEXT_SIZE (THIS_MODULE->core_text_size)
+    #define THIS_MODULE_RO_SIZE (THIS_MODULE->core_ro_size)
+#endif
+
     {
         char *pie_text_start = (char *)wolfCrypt_PIE_first_function;
         char *pie_text_end = (char *)wolfCrypt_PIE_last_function;
@@ -120,8 +142,8 @@ static int wolfssl_init(void)
         unsigned int text_hash, rodata_hash;
 
         if ((pie_text_start < pie_text_end) &&
-            (pie_text_start >= (char *)(THIS_MODULE->core_layout.base)) &&
+            (pie_text_start >= (char *)THIS_MODULE_BASE) &&
-            (pie_text_end - (char *)(THIS_MODULE->core_layout.base) <= THIS_MODULE->core_layout.text_size))
+            (pie_text_end - (char *)THIS_MODULE_BASE <= THIS_MODULE_TEXT_SIZE))
         {
             text_hash = hash_span(pie_text_start, pie_text_end);
         } else {
@@ -130,14 +152,14 @@ static int wolfssl_init(void)
                     pie_text_start,
                     pie_text_end,
                     pie_text_end-pie_text_start,
-                    THIS_MODULE->core_layout.base,
+                    THIS_MODULE_BASE,
-                    (char *)(THIS_MODULE->core_layout.base) + THIS_MODULE->core_layout.text_size);
+                    (char *)THIS_MODULE_BASE + THIS_MODULE_TEXT_SIZE);
             text_hash = 0;
         }
 
         if ((pie_rodata_start < pie_rodata_end) &&
-            (pie_rodata_start >= (char *)(THIS_MODULE->core_layout.base) + THIS_MODULE->core_layout.text_size) &&
+            (pie_rodata_start >= (char *)THIS_MODULE_BASE + THIS_MODULE_TEXT_SIZE) &&
-            (pie_rodata_end - (char *)(THIS_MODULE->core_layout.base) <= THIS_MODULE->core_layout.ro_size))
+            (pie_rodata_end - (char *)THIS_MODULE_BASE <= THIS_MODULE_RO_SIZE))
         {
             rodata_hash = hash_span(pie_rodata_start, pie_rodata_end);
         } else {
@@ -146,8 +168,8 @@ static int wolfssl_init(void)
                     pie_rodata_start,
                     pie_rodata_end,
                     pie_rodata_end-pie_rodata_start,
-                    (char *)(THIS_MODULE->core_layout.base) + THIS_MODULE->core_layout.text_size,
+                    (char *)THIS_MODULE_BASE + THIS_MODULE_TEXT_SIZE,
-                    (char *)(THIS_MODULE->core_layout.base) + THIS_MODULE->core_layout.ro_size);
+                    (char *)THIS_MODULE_BASE + THIS_MODULE_RO_SIZE);
             rodata_hash = 0;
         }
 
@@ -158,20 +180,20 @@ static int wolfssl_init(void)
         pr_info("wolfCrypt container hashes (spans): %x (%lu) %x (%lu), module base %pK\n",
                 text_hash, pie_text_end-pie_text_start,
                 rodata_hash, pie_rodata_end-pie_rodata_start,
-                THIS_MODULE->core_layout.base);
+                THIS_MODULE_BASE);
     }
 #endif /* HAVE_LINUXKM_PIE_SUPPORT */
 
 #ifdef HAVE_FIPS
     ret = wolfCrypt_SetCb_fips(lkmFipsCb);
     if (ret != 0) {
-        pr_err("wolfCrypt_SetCb_fips() failed: %s", wc_GetErrorString(ret));
+        pr_err("wolfCrypt_SetCb_fips() failed: %s\n", wc_GetErrorString(ret));
         return -ECANCELED;
     }
     fipsEntry();
     ret = wolfCrypt_GetStatus_fips();
     if (ret != 0) {
-        pr_err("wolfCrypt_GetStatus_fips() failed: %s", wc_GetErrorString(ret));
+        pr_err("wolfCrypt_GetStatus_fips() failed: %s\n", wc_GetErrorString(ret));
         if (ret == IN_CORE_FIPS_E) {
             const char *newhash = wolfCrypt_GetCoreHash_fips();
             pr_err("Update verifyCore[] in fips_test.c with new hash \"%s\" and rebuild.\n",
@@ -198,13 +220,13 @@ static int wolfssl_init(void)
 #ifdef WOLFCRYPT_ONLY
     ret = wolfCrypt_Init();
     if (ret != 0) {
-        pr_err("wolfCrypt_Init() failed: %s", wc_GetErrorString(ret));
+        pr_err("wolfCrypt_Init() failed: %s\n", wc_GetErrorString(ret));
         return -ECANCELED;
     }
 #else
     ret = wolfSSL_Init();
     if (ret != WOLFSSL_SUCCESS) {
-        pr_err("wolfSSL_Init() failed: %s", wc_GetErrorString(ret));
+        pr_err("wolfSSL_Init() failed: %s\n", wc_GetErrorString(ret));
         return -ECANCELED;
     }
 #endif
@@ -212,7 +234,7 @@ static int wolfssl_init(void)
 #ifndef NO_CRYPT_TEST
     ret = wolfcrypt_test(NULL);
     if (ret < 0) {
-        pr_err("wolfcrypt self-test failed with return code %d.", ret);
+        pr_err("wolfcrypt self-test failed with return code %d.\n", ret);
         (void)libwolfssl_cleanup();
         msleep(10);
         return -ECANCELED;
@@ -221,12 +243,20 @@ static int wolfssl_init(void)
 #endif
 
 #ifdef WOLFCRYPT_ONLY
-    pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " loaded. See https://www.wolfssl.com/ for information.\n");
+    pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " loaded"
-#else
+#ifdef CONFIG_MODULE_SIG
-    pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " loaded. See https://www.wolfssl.com/ for information.\n");
+            " with valid module signature"
+#endif
+            ".\nSee https://www.wolfssl.com/ for more information.\n"
+            "wolfCrypt Copyright (C) 2006-present wolfSSL Inc.  Licensed under " WOLFSSL_LICENSE ".\n");
+#else
+    pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " loaded"
+#ifdef CONFIG_MODULE_SIG
+            " with valid module signature"
+#endif
+            ".\nSee https://www.wolfssl.com/ for more information.\n"
+            "wolfSSL Copyright (C) 2006-present wolfSSL Inc.  Licensed under " WOLFSSL_LICENSE ".\n");
 #endif
-
-    pr_info("Copyright (C) 2006-2020 wolfSSL Inc. All Rights Reserved.\n");
 
     return 0;
 }
@@ -240,18 +270,38 @@ static void wolfssl_exit(void)
 #endif
 {
     (void)libwolfssl_cleanup();
+
     return;
 }
 
 module_exit(wolfssl_exit);
 
-MODULE_LICENSE("GPL v2");
+MODULE_LICENSE(WOLFSSL_LICENSE);
 MODULE_AUTHOR("https://www.wolfssl.com/");
 MODULE_DESCRIPTION("libwolfssl cryptographic and protocol facilities");
 MODULE_VERSION(LIBWOLFSSL_VERSION_STRING);
 
 #ifdef USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE
 
+/* get_current() is an inline or macro, depending on the target -- sidestep the whole issue with a wrapper func. */
+static struct task_struct *my_get_current_thread(void) {
+    return get_current();
+}
+
+/* ditto for preempt_count(). */
+static int my_preempt_count(void) {
+    return preempt_count();
+}
+
+#if defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0))
+static int my_copy_fpregs_to_fpstate(struct fpu *fpu) {
+    return copy_fpregs_to_fpstate(fpu);
+}
+static void my_copy_kernel_to_fpregs(union fpregs_state *fpstate) {
+    copy_kernel_to_fpregs(fpstate);
+}
+#endif
+
 static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     memset(
         &wolfssl_linuxkm_pie_redirect_table,
@@ -310,12 +360,22 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
         kmalloc_order_trace;
 
     wolfssl_linuxkm_pie_redirect_table.get_random_bytes = get_random_bytes;
-    wolfssl_linuxkm_pie_redirect_table.ktime_get_real_seconds =
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
-        ktime_get_real_seconds;
+        wolfssl_linuxkm_pie_redirect_table.getnstimeofday =
-    wolfssl_linuxkm_pie_redirect_table.ktime_get_with_offset =
+            getnstimeofday;
-        ktime_get_with_offset;
+    #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+        wolfssl_linuxkm_pie_redirect_table.current_kernel_time64 =
+            current_kernel_time64;
+    #else
+        wolfssl_linuxkm_pie_redirect_table.ktime_get_coarse_real_ts64 =
+            ktime_get_coarse_real_ts64;
+    #endif
 
-#if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP)
+    wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread;
+    wolfssl_linuxkm_pie_redirect_table.preempt_count = my_preempt_count;
+
+#ifdef WOLFSSL_LINUXKM_SIMD_X86
+    wolfssl_linuxkm_pie_redirect_table.irq_fpu_usable = irq_fpu_usable;
     #ifdef kernel_fpu_begin
     wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask =
         kernel_fpu_begin_mask;
@@ -324,11 +384,30 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
         kernel_fpu_begin;
     #endif
     wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
+    #ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
+        #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
+            wolfssl_linuxkm_pie_redirect_table.copy_fpregs_to_fpstate = my_copy_fpregs_to_fpstate;
+            wolfssl_linuxkm_pie_redirect_table.copy_kernel_to_fpregs = my_copy_kernel_to_fpregs;
+        #else
+            wolfssl_linuxkm_pie_redirect_table.save_fpregs_to_fpstate = save_fpregs_to_fpstate;
+            wolfssl_linuxkm_pie_redirect_table.__restore_fpregs_from_fpstate = __restore_fpregs_from_fpstate;
+            wolfssl_linuxkm_pie_redirect_table.xfeatures_mask_all = &xfeatures_mask_all;
+        #endif
+        wolfssl_linuxkm_pie_redirect_table.cpu_number = &cpu_number;
+        wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids;
+#endif /* WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
 #endif
 
     wolfssl_linuxkm_pie_redirect_table.__mutex_init = __mutex_init;
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
+        wolfssl_linuxkm_pie_redirect_table.mutex_lock_nested = mutex_lock_nested;
+    #else
         wolfssl_linuxkm_pie_redirect_table.mutex_lock = mutex_lock;
+    #endif
     wolfssl_linuxkm_pie_redirect_table.mutex_unlock = mutex_unlock;
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
+        wolfssl_linuxkm_pie_redirect_table.mutex_destroy = mutex_destroy;
+    #endif
 
 #ifdef HAVE_FIPS
     wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_first =

src/internal.c

@@ -8269,11 +8269,9 @@ ProtocolVersion MakeDTLSv1_2(void)
     }
 
 #elif defined(WOLFSSL_LINUXKM)
-    #include <linux/time.h>
-    #include <linux/ktime.h>
     word32 LowResTimer(void)
     {
-        return (word32)ktime_get_real_ns();
+        return (word32)time(NULL);
     }
 
 #else

src/ssl.c

@@ -52105,7 +52105,7 @@ WOLFSSL_BN_ULONG wolfSSL_BN_mod_word(const WOLFSSL_BIGNUM *bn,
 
     if (w <= MP_MASK) {
         mp_digit bn_ret;
-        if (mp_mod_d((mp_int*)bn->internal, (WOLFSSL_BN_ULONG)w, &bn_ret) != MP_OKAY) {
+        if (mp_mod_d((mp_int*)bn->internal, (mp_digit)w, &bn_ret) != MP_OKAY) {
             WOLFSSL_MSG("mp_add_d error");
             return (WOLFSSL_BN_ULONG)WOLFSSL_FATAL_ERROR;
         }

src/tls13.c

@@ -1558,19 +1558,25 @@ end:
         return (word32)(uTaskerSystemTick / (TICK_RESOLUTION / 1000));
     }
 #elif defined(WOLFSSL_LINUXKM)
-    /* The time in milliseconds.
-     * Used for tickets to represent difference between when first seen and when
-     * sending.
-     *
-     * returns the time in milliseconds as a 32-bit value.
-     */
     word32 TimeNowInMilliseconds(void)
     {
-    #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+        s64 t;
-        return (word32)(ktime_get_real_ns() / (s64)1000000);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
-    #else
+        struct timespec ts;
-        return (word32)(ktime_get_real_ns() / (ktime_t)1000000);
+        getnstimeofday(&ts);
-    #endif
+        t = ts.tv_sec * (s64)1000;
+        t += ts.tv_nsec / (s64)1000000;
+#else
+        struct timespec64 ts;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+        ts = current_kernel_time64();
+#else
+        ktime_get_coarse_real_ts64(&ts);
+#endif
+        t = ts.tv_sec * 1000L;
+        t += ts.tv_nsec / 1000000L;
+#endif
+        return (word32)t;
     }
 #elif defined(WOLFSSL_QNX_CAAM)
     word32 TimeNowInMilliseconds(void)

tests/api.c

@@ -3804,7 +3804,11 @@ static int nonblocking_accept_read(void* args, WOLFSSL* ssl, SOCKET_T* sockfd)
 #endif
 
 #if defined(OPENSSL_ALL) || defined(WOLFSSL_HAPROXY) || defined(WOLFSSL_WPAS)
+    #ifdef WC_SHA512_DIGEST_SIZE
         #define MD_MAX_SIZE WC_SHA512_DIGEST_SIZE
+    #else
+        #define MD_MAX_SIZE WC_SHA256_DIGEST_SIZE
+    #endif
     byte server_side_msg1[MD_MAX_SIZE] = {0};/* msg sent by server */
     byte server_side_msg2[MD_MAX_SIZE] = {0};/* msg received from client */
     byte client_side_msg1[MD_MAX_SIZE] = {0};/* msg sent by client */
@@ -6525,7 +6529,7 @@ static void verify_alpn_matching_http1(WOLFSSL* ssl)
     AssertIntEQ(0, XMEMCMP(nego_proto, proto, protoSz));
 }
 
-static void test_wolfSSL_set_alpn_protos()
+static void test_wolfSSL_set_alpn_protos(void)
 {
     unsigned long i;
     callback_functions callbacks[] = {
@@ -23188,8 +23192,8 @@ static int test_wc_ecc_shared_secret (void)
 #if defined(HAVE_ECC) && defined(HAVE_ECC_DHE) && !defined(WC_NO_RNG)
     ecc_key     key, pubKey;
     WC_RNG      rng;
-    int         keySz = KEY16;
+    byte        out[KEY32];
-    byte        out[KEY16];
+    int         keySz = sizeof(out);
     word32      outlen = (word32)sizeof(out);
 
 #if defined(HAVE_ECC) && !defined(NO_ECC256)
@@ -44450,7 +44454,11 @@ static void test_wolfSSL_SMIME_read_PKCS7(void)
 
 #ifdef WOLFSSL_TLS13
 #if defined(WOLFSSL_SEND_HRR_COOKIE) && !defined(NO_WOLFSSL_SERVER)
-static byte fixedKey[WC_SHA384_DIGEST_SIZE] = { 0, };
+#ifdef WC_SHA384_DIGEST_SIZE
+    static byte fixedKey[WC_SHA384_DIGEST_SIZE] = { 0, };
+#else
+    static byte fixedKey[WC_SHA256_DIGEST_SIZE] = { 0, };
+#endif
 #endif
 #ifdef WOLFSSL_EARLY_DATA
 static const char earlyData[] = "Early Data";

wolfcrypt/benchmark/benchmark.c

@@ -5770,7 +5770,7 @@ void bench_eccEncrypt(int curveId)
     ecc_key userA, userB;
     int     keySize;
     byte    msg[48];
-    byte    out[80];
+    byte    out[sizeof(msg) + WC_SHA256_DIGEST_SIZE + (MAX_ECC_BITS+3)/4 + 2];
     word32  outSz   = sizeof(out);
     word32  bench_plainSz = BENCH_SIZE;
     int     ret, i, count;

wolfcrypt/src/aes.c

@@ -775,6 +775,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
                                                     const int bits, Aes* aes)
         {
             int nr;
+            int ret;
 #ifdef WOLFSSL_SMALL_STACK
             Aes *temp_key;
 #else
@@ -805,7 +806,12 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
             nr = temp_key->rounds;
             aes->rounds = nr;
 
-            SAVE_VECTOR_REGISTERS();
+            if ((ret = SAVE_VECTOR_REGISTERS()) != 0) {
+#ifdef WOLFSSL_SMALL_STACK
+                XFREE(temp_key, aes->heap, DYNAMIC_TYPE_AES);
+#endif
+                return ret;
+            }
 
             Key_Schedule[nr] = Temp_Key_Schedule[0];
             Key_Schedule[nr-1] = _mm_aesimc_si128(Temp_Key_Schedule[1]);
@@ -1738,10 +1744,8 @@ static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
             tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN));
 
             XMEMCPY(tmp_align, inBlock, AES_BLOCK_SIZE);
-            SAVE_VECTOR_REGISTERS();
             AES_ECB_encrypt(tmp_align, tmp_align, AES_BLOCK_SIZE,
                     (byte*)aes->key, aes->rounds);
-            RESTORE_VECTOR_REGISTERS();
             XMEMCPY(outBlock, tmp_align, AES_BLOCK_SIZE);
             XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
             return;
@@ -1751,10 +1755,8 @@ static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
         #endif
         }
 
-        SAVE_VECTOR_REGISTERS();
         AES_ECB_encrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key,
                         aes->rounds);
-        RESTORE_VECTOR_REGISTERS();
 
         return;
     }
@@ -2089,10 +2091,8 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
         /* if input and output same will overwrite input iv */
         if ((const byte*)aes->tmp != inBlock)
             XMEMCPY(aes->tmp, inBlock, AES_BLOCK_SIZE);
-        SAVE_VECTOR_REGISTERS();
         AES_ECB_decrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key,
                         aes->rounds);
-        RESTORE_VECTOR_REGISTERS();
         return;
     }
     else {
@@ -3060,6 +3060,64 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
     #elif defined(WOLFSSL_DEVCRYPTO_AES)
         /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
 
+    #elif defined(WOLFSSL_LINUXKM)
+
+        #ifdef WOLFSSL_AESNI
+
+        __must_check int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
+        {
+            if (haveAESNI && aes->use_aesni) {
+                int ret = SAVE_VECTOR_REGISTERS();
+                if (ret != 0)
+                    return ret;
+            }
+            wc_AesEncrypt(aes, in, out);
+            if (haveAESNI && aes->use_aesni)
+                RESTORE_VECTOR_REGISTERS();
+            return 0;
+        }
+        /* vector reg save/restore is explicit in all below calls to
+         * wc_Aes{En,De}cryptDirect(), so bypass the public version with a
+         * macro.
+         */
+        #define wc_AesEncryptDirect(aes, out, in) wc_AesEncrypt(aes, in, out)
+        #ifdef HAVE_AES_DECRYPT
+        /* Allow direct access to one block decrypt */
+        __must_check int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
+        {
+            if (haveAESNI && aes->use_aesni) {
+                int ret = SAVE_VECTOR_REGISTERS();
+                if (ret != 0)
+                    return ret;
+            }
+            wc_AesDecrypt(aes, in, out);
+            if (haveAESNI && aes->use_aesni)
+                RESTORE_VECTOR_REGISTERS();
+            return 0;
+        }
+        #define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out)
+        #endif /* HAVE_AES_DECRYPT */
+
+        #else /* !WOLFSSL_AESNI */
+
+        __must_check int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
+        {
+            wc_AesEncrypt(aes, in, out);
+            return 0;
+        }
+        #define wc_AesEncryptDirect(aes, out, in) wc_AesEncrypt(aes, in, out)
+        #ifdef HAVE_AES_DECRYPT
+        /* Allow direct access to one block decrypt */
+        __must_check int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
+        {
+            wc_AesDecrypt(aes, in, out);
+            return 0;
+        }
+        #define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out)
+        #endif /* HAVE_AES_DECRYPT */
+
+        #endif /* WOLFSSL_AESNI */
+
     #else
         /* Allow direct access to one block encrypt */
         void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
@@ -3814,6 +3872,7 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
 
     #ifdef WOLFSSL_AESNI
         if (haveAESNI) {
+            int ret;
             #ifdef DEBUG_AESNI
                 printf("about to aes cbc encrypt\n");
                 printf("in  = %p\n", in);
@@ -3834,7 +3893,10 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
 
                 tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN));
                 XMEMCPY(tmp_align, in, sz);
-                SAVE_VECTOR_REGISTERS();
+                if ((ret = SAVE_VECTOR_REGISTERS()) != 0) {
+                    XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
+                    return ret;
+                }
                 AES_CBC_encrypt(tmp_align, tmp_align, (byte*)aes->reg, sz,
                                                   (byte*)aes->key, aes->rounds);
                 RESTORE_VECTOR_REGISTERS();
@@ -3850,7 +3912,8 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
             #endif
             }
 
-            SAVE_VECTOR_REGISTERS();
+            if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+                return ret;
             AES_CBC_encrypt(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
                             aes->rounds);
             RESTORE_VECTOR_REGISTERS();
@@ -3935,6 +3998,7 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
 
     #ifdef WOLFSSL_AESNI
         if (haveAESNI) {
+            int ret;
             #ifdef DEBUG_AESNI
                 printf("about to aes cbc decrypt\n");
                 printf("in  = %p\n", in);
@@ -3947,7 +4011,8 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
 
             /* if input and output same will overwrite input iv */
             XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
-            SAVE_VECTOR_REGISTERS();
+            if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+                return ret;
             #if defined(WOLFSSL_AESNI_BY4)
             AES_CBC_decrypt_by4(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
                             aes->rounds);
@@ -7519,7 +7584,9 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
 #ifdef WOLFSSL_AESNI
     #ifdef HAVE_INTEL_AVX2
     if (IS_INTEL_AVX2(intel_flags)) {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         AES_GCM_encrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
                                  authTagSz, (const byte*)aes->key, aes->rounds);
         RESTORE_VECTOR_REGISTERS();
@@ -7529,7 +7596,9 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
     #endif
     #ifdef HAVE_INTEL_AVX1
     if (IS_INTEL_AVX1(intel_flags)) {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         AES_GCM_encrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
                                  authTagSz, (const byte*)aes->key, aes->rounds);
         RESTORE_VECTOR_REGISTERS();
@@ -8041,7 +8110,9 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
 #ifdef WOLFSSL_AESNI
     #ifdef HAVE_INTEL_AVX2
     if (IS_INTEL_AVX2(intel_flags)) {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         AES_GCM_decrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
                                  authTagSz, (byte*)aes->key, aes->rounds, &res);
         RESTORE_VECTOR_REGISTERS();
@@ -8053,7 +8124,9 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
     #endif
     #ifdef HAVE_INTEL_AVX1
     if (IS_INTEL_AVX1(intel_flags)) {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         AES_GCM_decrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
                                  authTagSz, (byte*)aes->key, aes->rounds, &res);
         RESTORE_VECTOR_REGISTERS();
@@ -8296,7 +8369,7 @@ extern void AES_GCM_encrypt_final_aesni(unsigned char* tag,
  * @param [in]      iv    IV/nonce buffer.
  * @param [in]      ivSz  Length of IV/nonce data.
  */
-static void AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
+static int AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
 {
     /* Reset state fields. */
     aes->aSz = 0;
@@ -8309,7 +8382,9 @@ static void AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
 
 #ifdef HAVE_INTEL_AVX2
     if (IS_INTEL_AVX2(intel_flags)) {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         AES_GCM_init_avx2((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
                           AES_COUNTER(aes), AES_INITCTR(aes));
         RESTORE_VECTOR_REGISTERS();
@@ -8318,7 +8393,9 @@ static void AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
 #endif
 #ifdef HAVE_INTEL_AVX1
     if (IS_INTEL_AVX1(intel_flags)) {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         AES_GCM_init_avx1((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
                           AES_COUNTER(aes), AES_INITCTR(aes));
         RESTORE_VECTOR_REGISTERS();
@@ -8326,11 +8403,14 @@ static void AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
     else
 #endif
     {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return BAD_STATE_E;
         AES_GCM_init_aesni((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
                            AES_COUNTER(aes), AES_INITCTR(aes));
         RESTORE_VECTOR_REGISTERS();
     }
+    return 0;
 }
 
 /* Update the AES GCM for encryption with authentication data.
@@ -8458,13 +8538,15 @@ static void AesGcmAadUpdate_aesni(Aes* aes, const byte* a, word32 aSz, int endA)
  * @param [in]      a    Buffer holding authentication data.
  * @param [in]      aSz  Length of authentication data in bytes.
  */
-static void AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
+static int AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
     word32 cSz, const byte* a, word32 aSz)
 {
     word32 blocks;
     int partial;
 
-    SAVE_VECTOR_REGISTERS();
+    int ret = SAVE_VECTOR_REGISTERS();
+    if (ret != 0)
+        return ret;
     /* Hash in A, the Authentication Data */
     AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL));
 
@@ -8573,6 +8655,7 @@ static void AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
         }
     }
     RESTORE_VECTOR_REGISTERS();
+    return 0;
 }
 
 /* Finalize the AES GCM for encryption and calculate the authentication tag.
@@ -8584,12 +8667,14 @@ static void AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
  * @param [in]      authTagSz  Length of authentication tag in bytes.
  * @return  0 on success.
  */
-static void AesGcmEncryptFinal_aesni(Aes* aes, byte* authTag, word32 authTagSz)
+static int AesGcmEncryptFinal_aesni(Aes* aes, byte* authTag, word32 authTagSz)
 {
     /* AAD block incomplete when > 0 */
     byte over = aes->aOver;
 
-    SAVE_VECTOR_REGISTERS();
+    int ret = SAVE_VECTOR_REGISTERS();
+    if (ret != 0)
+        return ret;
     if (aes->cOver > 0) {
         /* Cipher text block incomplete. */
         over = aes->cOver;
@@ -8635,6 +8720,7 @@ static void AesGcmEncryptFinal_aesni(Aes* aes, byte* authTag, word32 authTagSz)
             aes->aSz, aes->H, AES_INITCTR(aes));
     }
     RESTORE_VECTOR_REGISTERS();
+    return 0;
 }
 
 #if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
@@ -8680,13 +8766,15 @@ extern void AES_GCM_decrypt_final_aesni(unsigned char* tag,
  * @param [in]      a    Buffer holding authentication data.
  * @param [in]      aSz  Length of authentication data in bytes.
  */
-static void AesGcmDecryptUpdate_aesni(Aes* aes, byte* p, const byte* c,
+static int AesGcmDecryptUpdate_aesni(Aes* aes, byte* p, const byte* c,
     word32 cSz, const byte* a, word32 aSz)
 {
     word32 blocks;
     int partial;
 
-    SAVE_VECTOR_REGISTERS();
+    int ret = SAVE_VECTOR_REGISTERS();
+    if (ret != 0)
+        return ret;
     /* Hash in A, the Authentication Data */
     AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL));
 
@@ -8797,6 +8885,7 @@ static void AesGcmDecryptUpdate_aesni(Aes* aes, byte* p, const byte* c,
         }
     }
     RESTORE_VECTOR_REGISTERS();
+    return 0;
 }
 
 /* Finalize the AES GCM for decryption and check the authentication tag.
@@ -8819,7 +8908,8 @@ static int AesGcmDecryptFinal_aesni(Aes* aes, const byte* authTag,
     byte over = aes->aOver;
     byte *lastBlock = AES_LASTGBLOCK(aes);
 
-    SAVE_VECTOR_REGISTERS();
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
     if (aes->cOver > 0) {
         /* Cipher text block incomplete. */
         over = aes->cOver;
@@ -8940,7 +9030,7 @@ int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv,
                 || IS_INTEL_AVX1(intel_flags)
             #endif
                 ) {
-                AesGcmInit_aesni(aes, iv, ivSz);
+                ret = AesGcmInit_aesni(aes, iv, ivSz);
             }
             else
         #endif
@@ -9052,7 +9142,7 @@ int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz,
             || IS_INTEL_AVX1(intel_flags)
         #endif
             ) {
-            AesGcmEncryptUpdate_aesni(aes, out, in, sz, authIn, authInSz);
+            ret = AesGcmEncryptUpdate_aesni(aes, out, in, sz, authIn, authInSz);
         }
         else
     #endif
@@ -9818,13 +9908,14 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
     B[15] = 1;
 #ifdef WOLFSSL_AESNI
     if (haveAESNI && aes->use_aesni) {
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         while (inSz >= AES_BLOCK_SIZE * 4) {
             AesCcmCtrIncSet4(B, lenSz);
 
-            SAVE_VECTOR_REGISTERS();
             AES_ECB_encrypt(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key,
                             aes->rounds);
-            RESTORE_VECTOR_REGISTERS();
 
             xorbuf(A, in, AES_BLOCK_SIZE * 4);
             XMEMCPY(out, A, AES_BLOCK_SIZE * 4);
@@ -9835,6 +9926,7 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
 
             AesCcmCtrInc4(B, lenSz);
         }
+        RESTORE_VECTOR_REGISTERS();
     }
 #endif
     while (inSz >= AES_BLOCK_SIZE) {
@@ -9903,13 +9995,14 @@ int  wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
 
 #ifdef WOLFSSL_AESNI
     if (haveAESNI && aes->use_aesni) {
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         while (oSz >= AES_BLOCK_SIZE * 4) {
             AesCcmCtrIncSet4(B, lenSz);
 
-            SAVE_VECTOR_REGISTERS();
             AES_ECB_encrypt(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key,
                             aes->rounds);
-            RESTORE_VECTOR_REGISTERS();
 
             xorbuf(A, in, AES_BLOCK_SIZE * 4);
             XMEMCPY(o, A, AES_BLOCK_SIZE * 4);
@@ -9920,6 +10013,7 @@ int  wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
 
             AesCcmCtrInc4(B, lenSz);
         }
+        RESTORE_VECTOR_REGISTERS();
     }
 #endif
     while (oSz >= AES_BLOCK_SIZE) {
@@ -10274,12 +10368,10 @@ int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 #else
 
 /* Software AES - ECB */
-int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
+static int _AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 {
     word32 blocks = sz / AES_BLOCK_SIZE;
 
-    if ((in == NULL) || (out == NULL) || (aes == NULL))
-      return BAD_FUNC_ARG;
 #ifdef WOLFSSL_IMXRT_DCP
     if (aes->keylen == 16)
         return DCPAesEcbEncrypt(aes, out, in, sz);
@@ -10293,13 +10385,10 @@ int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
     return 0;
 }
 
-
+static int _AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
-int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 {
     word32 blocks = sz / AES_BLOCK_SIZE;
 
-    if ((in == NULL) || (out == NULL) || (aes == NULL))
-      return BAD_FUNC_ARG;
 #ifdef WOLFSSL_IMXRT_DCP
     if (aes->keylen == 16)
         return DCPAesEcbDecrypt(aes, out, in, sz);
@@ -10312,6 +10401,36 @@ int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
     }
     return 0;
 }
+
+int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
+{
+    int ret;
+
+    if ((in == NULL) || (out == NULL) || (aes == NULL))
+      return BAD_FUNC_ARG;
+
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
+    ret = _AesEcbEncrypt(aes, out, in, sz);
+    RESTORE_VECTOR_REGISTERS();
+
+    return ret;
+}
+
+int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
+{
+    int ret;
+
+    if ((in == NULL) || (out == NULL) || (aes == NULL))
+      return BAD_FUNC_ARG;
+
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
+    ret = _AesEcbDecrypt(aes, out, in, sz);
+    RESTORE_VECTOR_REGISTERS();
+
+    return ret;
+}
 #endif
 #endif /* HAVE_AES_ECB */
 
@@ -10335,6 +10454,7 @@ static int wc_AesFeedbackEncrypt(Aes* aes, byte* out, const byte* in,
 #ifdef WOLFSSL_AES_CFB
     byte*  reg = NULL;
 #endif
+    int ret;
 
     if (aes == NULL || out == NULL || in == NULL) {
         return BAD_FUNC_ARG;
@@ -10360,6 +10480,9 @@ static int wc_AesFeedbackEncrypt(Aes* aes, byte* out, const byte* in,
         sz--;
     }
 
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
+
     while (sz >= AES_BLOCK_SIZE) {
         /* Using aes->tmp here for inline case i.e. in=out */
         wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
@@ -10406,6 +10529,7 @@ static int wc_AesFeedbackEncrypt(Aes* aes, byte* out, const byte* in,
             aes->left--;
         }
     }
+    RESTORE_VECTOR_REGISTERS();
 
     return 0;
 }
@@ -10427,6 +10551,7 @@ static int wc_AesFeedbackDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
         byte mode)
 {
     byte*  tmp;
+    int ret;
 
     if (aes == NULL || out == NULL || in == NULL) {
         return BAD_FUNC_ARG;
@@ -10448,6 +10573,9 @@ static int wc_AesFeedbackDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
         sz--;
     }
 
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
+
     while (sz > AES_BLOCK_SIZE) {
         /* Using aes->tmp here for inline case i.e. in=out */
         wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
@@ -10491,6 +10619,7 @@ static int wc_AesFeedbackDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
             aes->left--;
         }
     }
+    RESTORE_VECTOR_REGISTERS();
 
     return 0;
 }
@@ -10563,6 +10692,7 @@ static int wc_AesFeedbackCFB8(Aes* aes, byte* out, const byte* in,
         word32 sz, byte dir)
 {
     byte *pt;
+    int ret;
 
     if (aes == NULL || out == NULL || in == NULL) {
         return BAD_FUNC_ARG;
@@ -10572,6 +10702,9 @@ static int wc_AesFeedbackCFB8(Aes* aes, byte* out, const byte* in,
         return 0;
     }
 
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
+
     while (sz > 0) {
         wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
         if (dir == AES_DECRYPTION) {
@@ -10600,6 +10733,8 @@ static int wc_AesFeedbackCFB8(Aes* aes, byte* out, const byte* in,
         sz  -= 1;
     }
 
+    RESTORE_VECTOR_REGISTERS();
+
     return 0;
 }
 
@@ -10612,6 +10747,7 @@ static int wc_AesFeedbackCFB1(Aes* aes, byte* out, const byte* in,
     byte cur = 0; /* hold current work in order to handle inline in=out */
     byte* pt;
     int bit = 7;
+    int ret;
 
     if (aes == NULL || out == NULL || in == NULL) {
         return BAD_FUNC_ARG;
@@ -10621,6 +10757,9 @@ static int wc_AesFeedbackCFB1(Aes* aes, byte* out, const byte* in,
         return 0;
     }
 
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
+
     while (sz > 0) {
         wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
         if (dir == AES_DECRYPTION) {
@@ -10667,6 +10806,7 @@ static int wc_AesFeedbackCFB1(Aes* aes, byte* out, const byte* in,
     if (bit > 0 && bit < 7) {
         out[0] = cur;
     }
+    RESTORE_VECTOR_REGISTERS();
 
     return 0;
 }
@@ -10823,6 +10963,8 @@ int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
     byte t[KEYWRAP_BLOCK_SIZE];
     byte tmp[AES_BLOCK_SIZE];
 
+    int ret;
+
     /* n must be at least 2 64-bit blocks, output size is (n + 1) 8 bytes (64-bit) */
     if (aes == NULL || in  == NULL || inSz < 2*KEYWRAP_BLOCK_SIZE ||
         out == NULL || outSz < (inSz + KEYWRAP_BLOCK_SIZE))
@@ -10843,6 +10985,9 @@ int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
         XMEMCPY(tmp, iv, KEYWRAP_BLOCK_SIZE);
     }
 
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
+
     for (j = 0; j <= 5; j++) {
         for (i = 1; i <= inSz / KEYWRAP_BLOCK_SIZE; i++) {
             /* load R[i] */
@@ -10860,6 +11005,7 @@ int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
         }
         r = out + KEYWRAP_BLOCK_SIZE;
     }
+    RESTORE_VECTOR_REGISTERS();
 
     /* C[0] = A */
     XMEMCPY(out, tmp, KEYWRAP_BLOCK_SIZE);
@@ -10925,6 +11071,8 @@ int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
         0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6
     };
 
+    int ret;
+
     if (aes == NULL || in == NULL || inSz < 3 * KEYWRAP_BLOCK_SIZE ||
         out == NULL || outSz < (inSz - KEYWRAP_BLOCK_SIZE))
         return BAD_FUNC_ARG;
@@ -10944,6 +11092,9 @@ int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
     XMEMCPY(out, in + KEYWRAP_BLOCK_SIZE, inSz - KEYWRAP_BLOCK_SIZE);
     XMEMSET(t, 0, sizeof(t));
 
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
+
     /* initialize counter to 6n */
     n = (inSz - 1) / KEYWRAP_BLOCK_SIZE;
     InitKeyWrapCounter(t, 6 * n);
@@ -10964,6 +11115,7 @@ int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
             XMEMCPY(r, tmp + KEYWRAP_BLOCK_SIZE, KEYWRAP_BLOCK_SIZE);
         }
     }
+    RESTORE_VECTOR_REGISTERS();
 
     /* verify IV */
     if (XMEMCMP(tmp, expIv, KEYWRAP_BLOCK_SIZE) != 0)
@@ -11178,10 +11330,10 @@ static int _AesXtsHelper(Aes* aes, byte* out, const byte* in, word32 sz, int dir
 
     xorbuf(out, in, totalSz);
     if (dir == AES_ENCRYPTION) {
-        return wc_AesEcbEncrypt(aes, out, out, totalSz);
+        return _AesEcbEncrypt(aes, out, out, totalSz);
     }
     else {
-        return wc_AesEcbDecrypt(aes, out, out, totalSz);
+        return _AesEcbDecrypt(aes, out, out, totalSz);
     }
 }
 #endif /* HAVE_AES_ECB */
@@ -11224,6 +11376,9 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
         XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES
                                           * key setup passed to encrypt direct*/
 
+        if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+            return ret;
+
         wc_AesEncryptDirect(tweak, tmp, i);
 
     #ifdef HAVE_AES_ECB
@@ -11231,6 +11386,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
         if (in != out) { /* can not handle inline */
             XMEMCPY(out, tmp, AES_BLOCK_SIZE);
             if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0) {
+                RESTORE_VECTOR_REGISTERS();
                 return ret;
             }
         }
@@ -11285,6 +11441,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
             wc_AesEncryptDirect(aes, out - AES_BLOCK_SIZE, buf);
             xorbuf(out - AES_BLOCK_SIZE, tmp, AES_BLOCK_SIZE);
         }
+        RESTORE_VECTOR_REGISTERS();
     }
     else {
         WOLFSSL_MSG("Plain text input too small for encryption");
@@ -11335,6 +11492,9 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
         XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES
                                           * key setup passed to decrypt direct*/
 
+        if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+            return ret;
+
         wc_AesEncryptDirect(tweak, tmp, i);
 
         /* if Stealing then break out of loop one block early to handle special
@@ -11348,6 +11508,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
         if (in != out) { /* can not handle inline */
             XMEMCPY(out, tmp, AES_BLOCK_SIZE);
             if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0) {
+                RESTORE_VECTOR_REGISTERS();
                 return ret;
             }
         }
@@ -11416,6 +11577,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
             /* Make buffer with end of cipher text | last */
             XMEMCPY(buf, tmp2, AES_BLOCK_SIZE);
             if (sz >= AES_BLOCK_SIZE) { /* extra sanity check before copy */
+                RESTORE_VECTOR_REGISTERS();
                 return BUFFER_E;
             }
             XMEMCPY(buf, in,   sz);
@@ -11426,6 +11588,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
             xorbuf(tmp2, tmp, AES_BLOCK_SIZE);
             XMEMCPY(out - AES_BLOCK_SIZE, tmp2, AES_BLOCK_SIZE);
         }
+        RESTORE_VECTOR_REGISTERS();
     }
     else {
         WOLFSSL_MSG("Plain text input too small for encryption");

wolfcrypt/src/chacha.c

@@ -418,14 +418,18 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input,
 
     #ifdef HAVE_INTEL_AVX2
     if (IS_INTEL_AVX2(cpuidFlags)) {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         chacha_encrypt_avx2(ctx, input, output, msglen);
         RESTORE_VECTOR_REGISTERS();
         return 0;
     }
     #endif
     if (IS_INTEL_AVX1(cpuidFlags)) {
-        SAVE_VECTOR_REGISTERS();
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
         chacha_encrypt_avx1(ctx, input, output, msglen);
         RESTORE_VECTOR_REGISTERS();
         return 0;

wolfcrypt/src/cmac.c

@@ -119,10 +119,19 @@ int wc_InitCmac_ex(Cmac* cmac, const byte* key, word32 keySz,
         byte l[AES_BLOCK_SIZE];
 
         XMEMSET(l, 0, AES_BLOCK_SIZE);
+#ifdef WOLFSSL_LINUXKM
+        ret =
+#endif
             wc_AesEncryptDirect(&cmac->aes, l, l);
+#ifdef WOLFSSL_LINUXKM
+        if (ret == 0) {
+#endif
             ShiftAndXorRb(cmac->k1, l);
             ShiftAndXorRb(cmac->k2, cmac->k1);
             ForceZero(l, AES_BLOCK_SIZE);
+#ifdef WOLFSSL_LINUXKM
+        }
+#endif
     }
     return ret;
 }
@@ -172,9 +181,18 @@ int wc_CmacUpdate(Cmac* cmac, const byte* in, word32 inSz)
             if (cmac->totalSz != 0) {
                 xorbuf(cmac->buffer, cmac->digest, AES_BLOCK_SIZE);
             }
+#ifdef WOLFSSL_LINUXKM
+            ret =
+#endif
                 wc_AesEncryptDirect(&cmac->aes, cmac->digest, cmac->buffer);
+#ifdef WOLFSSL_LINUXKM
+            if (ret == 0) {
+#endif
                 cmac->totalSz += AES_BLOCK_SIZE;
                 cmac->bufferSz = 0;
+#ifdef WOLFSSL_LINUXKM
+            }
+#endif
         }
     }
 
@@ -221,9 +239,17 @@ int wc_CmacFinal(Cmac* cmac, byte* out, word32* outSz)
     }
     xorbuf(cmac->buffer, cmac->digest, AES_BLOCK_SIZE);
     xorbuf(cmac->buffer, subKey, AES_BLOCK_SIZE);
+#ifdef WOLFSSL_LINUXKM
+    ret =
+#endif
         wc_AesEncryptDirect(&cmac->aes, cmac->digest, cmac->buffer);
-
+#ifdef WOLFSSL_LINUXKM
+    if (ret == 0) {
+#endif
         XMEMCPY(out, cmac->digest, *outSz);
+#ifdef WOLFSSL_LINUXKM
+    }
+#endif
 
     ForceZero(cmac, sizeof(Cmac));
 

wolfcrypt/src/curve25519.c

@@ -128,15 +128,12 @@ int wc_curve25519_make_pub(int public_size, byte* pub, int private_size,
 #else
     fe_init();
 
-    #if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
-        SAVE_VECTOR_REGISTERS();
+        return ret;
-    #endif
 
     ret = curve25519(pub, priv, kCurve25519BasePoint);
 
-    #if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
     RESTORE_VECTOR_REGISTERS();
-    #endif
 #endif
 
     return ret;
@@ -174,15 +171,12 @@ int wc_curve25519_generic(int public_size, byte* pub,
 
     fe_init();
 
-    #if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
-        SAVE_VECTOR_REGISTERS();
+        return ret;
-    #endif
 
     ret = curve25519(pub, priv, basepoint);
 
-    #if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
     RESTORE_VECTOR_REGISTERS();
-    #endif
 
     return ret;
 #endif /* FREESCALE_LTC_ECC */
@@ -295,16 +289,13 @@ int wc_curve25519_shared_secret_ex(curve25519_key* private_key,
         ret = nxp_ltc_curve25519(&o, private_key->k, &public_key->p,
                                  kLTC_Curve25519);
     #else
-        #if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
+        if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
-            SAVE_VECTOR_REGISTERS();
+            return ret;
-        #endif
 
         ret = curve25519(o.point, private_key->k, public_key->p.point);
 
-        #if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
         RESTORE_VECTOR_REGISTERS();
     #endif
-    #endif
     if (ret != 0) {
         ForceZero(&o, sizeof(o));
         return ret;

wolfcrypt/src/des3.c

@@ -1422,7 +1422,7 @@
             byte* const  pc1m = buffer;            /* place to modify pc1 into */
             byte* const  pcr  = pc1m + 56;         /* place to rotate pc1 into */
             byte* const  ks   = pcr  + 56;
-            register int i, j, l;
+            int i, j, l;
             int          m;
 
             for (j = 0; j < 56; j++) {             /* convert pc1 to bits of key  */

wolfcrypt/src/memory.c

@@ -1133,3 +1133,176 @@ void __attribute__((no_instrument_function))
 }
 #endif
 
+#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
+union fpregs_state **wolfcrypt_irq_fpu_states = NULL;
+#endif
+
+#if defined(WOLFSSL_LINUXKM_SIMD_X86) && defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED)
+
+    static WARN_UNUSED_RESULT inline int am_in_hard_interrupt_handler(void)
+    {
+        return (preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0;
+    }
+
+    WARN_UNUSED_RESULT int allocate_wolfcrypt_irq_fpu_states(void)
+    {
+        wolfcrypt_irq_fpu_states =
+            (union fpregs_state **)kzalloc(nr_cpu_ids
+                                           * sizeof(struct fpu_state *),
+                                           GFP_KERNEL);
+        if (! wolfcrypt_irq_fpu_states) {
+            pr_err("warning, allocation of %lu bytes for "
+                   "wolfcrypt_irq_fpu_states failed.\n",
+                   nr_cpu_ids * sizeof(struct fpu_state *));
+            return MEMORY_E;
+        }
+        {
+            typeof(nr_cpu_ids) i;
+            for (i=0; i<nr_cpu_ids; ++i) {
+                _Static_assert(sizeof(union fpregs_state) <= PAGE_SIZE,
+                               "union fpregs_state is larger than expected.");
+                wolfcrypt_irq_fpu_states[i] =
+                    (union fpregs_state *)kzalloc(PAGE_SIZE
+                                                  /* sizeof(union fpregs_state) */,
+                                                  GFP_KERNEL);
+                if (! wolfcrypt_irq_fpu_states[i])
+                    break;
+                /* double-check that the allocation is 64-byte-aligned as needed
+                 * for xsave.
+                 */
+                if ((unsigned long)wolfcrypt_irq_fpu_states[i] & 63UL) {
+                    pr_err("warning, allocation for wolfcrypt_irq_fpu_states "
+                           "was not properly aligned (%px).\n",
+                           wolfcrypt_irq_fpu_states[i]);
+                    kfree(wolfcrypt_irq_fpu_states[i]);
+                    wolfcrypt_irq_fpu_states[i] = 0;
+                    break;
+                }
+            }
+            if (i < nr_cpu_ids) {
+                pr_err("warning, only %u/%u allocations succeeded for "
+                       "wolfcrypt_irq_fpu_states.\n",
+                       i, nr_cpu_ids);
+                return MEMORY_E;
+            }
+        }
+        return 0;
+    }
+
+    void free_wolfcrypt_irq_fpu_states(void)
+    {
+        if (wolfcrypt_irq_fpu_states) {
+            typeof(nr_cpu_ids) i;
+            for (i=0; i<nr_cpu_ids; ++i) {
+                if (wolfcrypt_irq_fpu_states[i])
+                    kfree(wolfcrypt_irq_fpu_states[i]);
+            }
+            kfree(wolfcrypt_irq_fpu_states);
+            wolfcrypt_irq_fpu_states = 0;
+        }
+    }
+
+    WARN_UNUSED_RESULT int save_vector_registers_x86(void)
+    {
+        preempt_disable();
+        if (! irq_fpu_usable()) {
+            if (am_in_hard_interrupt_handler()) {
+                int processor_id;
+
+                if (! wolfcrypt_irq_fpu_states) {
+                    static int warned_on_null_wolfcrypt_irq_fpu_states = 0;
+                    preempt_enable();
+                    if (! warned_on_null_wolfcrypt_irq_fpu_states) {
+                        warned_on_null_wolfcrypt_irq_fpu_states = 1;
+                        pr_err("save_vector_registers_x86 with null "
+                               "wolfcrypt_irq_fpu_states.\n");
+                    }
+                    return BAD_STATE_E;
+                }
+
+                processor_id = __smp_processor_id();
+
+                if (! wolfcrypt_irq_fpu_states[processor_id]) {
+                    static int _warned_on_null = -1;
+                    preempt_enable();
+                    if (_warned_on_null < processor_id) {
+                        _warned_on_null = processor_id;
+                        pr_err("save_vector_registers_x86 for cpu id %d with "
+                               "null wolfcrypt_irq_fpu_states[id].\n",
+                               processor_id);
+                    }
+                    return BAD_STATE_E;
+                }
+
+                /* check for nested interrupts -- doesn't exist on x86, but make
+                 * sure, in case something changes.
+                 */
+                if (((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] != 0) {
+                    preempt_enable();
+                    pr_err("save_vector_registers_x86 called recursively for "
+                           "cpu id %d.\n", processor_id);
+                    return BAD_STATE_E;
+                }
+
+                /* note, fpregs_lock() is not needed here, because
+                 * interrupts/preemptions are already disabled here.
+                 */
+                {
+                    /* save_fpregs_to_fpstate() only accesses fpu->state, which
+                     * has stringent alignment requirements (64 byte cache
+                     * line), but takes a pointer to the parent struct.  work
+                     * around this.
+                     */
+                    struct fpu *fake_fpu_pointer =
+                        (struct fpu *)(((char *)wolfcrypt_irq_fpu_states[processor_id])
+                                       - offsetof(struct fpu, state));
+                #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
+                    copy_fpregs_to_fpstate(fake_fpu_pointer);
+                #else
+                    save_fpregs_to_fpstate(fake_fpu_pointer);
+                #endif
+                }
+                /* mark the slot as used. */
+                ((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] = 1;
+                /* note, not preempt_enable()ing, mirroring kernel_fpu_begin()
+                 * semantics.
+                 */
+                return 0;
+            }
+            preempt_enable();
+            return BAD_STATE_E;
+        } else {
+            kernel_fpu_begin();
+            preempt_enable(); /* kernel_fpu_begin() does its own
+                               * preempt_disable().  decrement ours.
+                               */
+            return 0;
+        }
+    }
+    void restore_vector_registers_x86(void)
+    {
+        if (am_in_hard_interrupt_handler()) {
+            int processor_id = __smp_processor_id();
+            if ((wolfcrypt_irq_fpu_states == NULL) ||
+                (wolfcrypt_irq_fpu_states[processor_id] == NULL) ||
+                (((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] == 0))
+            {
+                pr_err("restore_vector_registers_x86 called for cpu id %d "
+                       "without saved context.\n", processor_id);
+                preempt_enable(); /* just in case */
+                return;
+            } else {
+            #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
+                copy_kernel_to_fpregs(wolfcrypt_irq_fpu_states[processor_id]);
+            #else
+                __restore_fpregs_from_fpstate(wolfcrypt_irq_fpu_states[processor_id],
+                                              xfeatures_mask_all);
+            #endif
+                ((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] = 0;
+                preempt_enable();
+                return;
+            }
+        }
+        kernel_fpu_end();
+    }
+#endif /* WOLFSSL_LINUXKM_SIMD_X86 && WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */

wolfcrypt/src/misc.c

@@ -299,7 +299,7 @@ WC_STATIC WC_INLINE void xorbuf(void* buf, const void* mask, word32 count)
 #ifndef WOLFSSL_NO_FORCE_ZERO
 /* This routine fills the first len bytes of the memory area pointed by mem
    with zeros. It ensures compiler optimizations doesn't skip it  */
-WC_STATIC WC_INLINE void ForceZero(const void* mem, word32 len)
+WC_STATIC WC_INLINE void ForceZero(void* mem, word32 len)
 {
     volatile byte* z = (volatile byte*)mem;
 

wolfcrypt/src/poly1305.c

@@ -262,14 +262,17 @@ static WC_INLINE void u32tole64(const word32 inLe32, byte outLe64[8])
 This local function operates on a message with a given number of bytes
 with a given ctx pointer to a Poly1305 structure.
 */
-static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
+static int poly1305_blocks(Poly1305* ctx, const unsigned char *m,
                      size_t bytes)
 {
 #ifdef USE_INTEL_SPEEDUP
     /* AVX2 is handled in wc_Poly1305Update. */
-    SAVE_VECTOR_REGISTERS();
+    int ret = SAVE_VECTOR_REGISTERS();
+    if (ret != 0)
+        return ret;
     poly1305_blocks_avx(ctx, m, bytes);
     RESTORE_VECTOR_REGISTERS();
+    return 0;
 #elif defined(POLY130564)
     const word64 hibit = (ctx->finished) ? 0 : ((word64)1 << 40); /* 1 << 128 */
     word64 r0,r1,r2;
@@ -320,6 +323,8 @@ static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
     ctx->h[1] = h1;
     ctx->h[2] = h2;
 
+    return 0;
+
 #else /* if not 64 bit then use 32 bit */
     const word32 hibit = (ctx->finished) ? 0 : ((word32)1 << 24); /* 1 << 128 */
     word32 r0,r1,r2,r3,r4;
@@ -385,6 +390,8 @@ static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
     ctx->h[3] = h3;
     ctx->h[4] = h4;
 
+    return 0;
+
 #endif /* end of 64 bit cpu blocks or 32 bit cpu */
 }
 
@@ -392,15 +399,18 @@ static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
 This local function is used for the last call when a message with a given
 number of bytes is less than the block size.
 */
-static void poly1305_block(Poly1305* ctx, const unsigned char *m)
+static int poly1305_block(Poly1305* ctx, const unsigned char *m)
 {
 #ifdef USE_INTEL_SPEEDUP
     /* No call to poly1305_block when AVX2, AVX2 does 4 blocks at a time. */
-    SAVE_VECTOR_REGISTERS();
+    int ret= SAVE_VECTOR_REGISTERS();
+    if (ret != 0)
+        return ret;
     poly1305_block_avx(ctx, m);
     RESTORE_VECTOR_REGISTERS();
+    return 0;
 #else
-    poly1305_blocks(ctx, m, POLY1305_BLOCK_SIZE);
+    return poly1305_blocks(ctx, m, POLY1305_BLOCK_SIZE);
 #endif
 }
 #endif /* !defined(WOLFSSL_ARMASM) || !defined(__aarch64__) */
@@ -434,7 +444,11 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz)
         intel_flags = cpuid_get_flags();
         cpu_flags_set = 1;
     }
-    SAVE_VECTOR_REGISTERS();
+    {
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
+    }
     #ifdef HAVE_INTEL_AVX2
     if (IS_INTEL_AVX2(intel_flags))
         poly1305_setkey_avx2(ctx, key);
@@ -497,6 +511,7 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz)
 int wc_Poly1305Final(Poly1305* ctx, byte* mac)
 {
 #ifdef USE_INTEL_SPEEDUP
+    int ret;
 #elif defined(POLY130564)
 
     word64 h0,h1,h2,c;
@@ -516,7 +531,8 @@ int wc_Poly1305Final(Poly1305* ctx, byte* mac)
         return BAD_FUNC_ARG;
 
 #ifdef USE_INTEL_SPEEDUP
-    SAVE_VECTOR_REGISTERS();
+    if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+        return ret;
     #ifdef HAVE_INTEL_AVX2
     if (IS_INTEL_AVX2(intel_flags))
         poly1305_final_avx2(ctx, mac);
@@ -704,7 +720,12 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes)
 #ifdef USE_INTEL_SPEEDUP
     #ifdef HAVE_INTEL_AVX2
     if (IS_INTEL_AVX2(intel_flags)) {
+        int ret = SAVE_VECTOR_REGISTERS();
+        if (ret != 0)
+            return ret;
+
         /* handle leftover */
+
         if (ctx->leftover) {
             size_t want = sizeof(ctx->buffer) - ctx->leftover;
             if (want > bytes)
@@ -718,15 +739,11 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes)
             if (ctx->leftover < sizeof(ctx->buffer))
                 return 0;
 
-            SAVE_VECTOR_REGISTERS();
             if (!ctx->started)
                 poly1305_calc_powers_avx2(ctx);
             poly1305_blocks_avx2(ctx, ctx->buffer, sizeof(ctx->buffer));
             ctx->leftover = 0;
         }
-        else {
-            SAVE_VECTOR_REGISTERS();
-        }
 
         /* process full blocks */
         if (bytes >= sizeof(ctx->buffer)) {
@@ -769,8 +786,11 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes)
 
         /* process full blocks */
         if (bytes >= POLY1305_BLOCK_SIZE) {
+            int ret;
             size_t want = (bytes & ~(POLY1305_BLOCK_SIZE - 1));
-            poly1305_blocks(ctx, m, want);
+            ret = poly1305_blocks(ctx, m, want);
+            if (ret != 0)
+                return ret;
             m += want;
             bytes -= (word32)want;
         }

wolfcrypt/src/pwdbased.c

@@ -368,11 +368,17 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen,
     byte*  buffer = staticBuffer;
 
 #ifdef WOLFSSL_SMALL_STACK
-    byte*  Ai;
+    byte*  Ai = NULL;
-    byte*  B;
+    byte*  B = NULL;
+    mp_int *B1 = NULL;
+    mp_int *i1 = NULL;
+    mp_int *res = NULL;
 #else
     byte   Ai[WC_MAX_DIGEST_SIZE];
     byte   B[WC_MAX_BLOCK_SIZE];
+    mp_int B1[1];
+    mp_int i1[1];
+    mp_int res[1];
 #endif
     enum wc_HashType hashT;
 
@@ -449,9 +455,20 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen,
     for (i = 0; i < (int)pLen; i++)
         P[i] = passwd[i % passLen];
 
+#ifdef WOLFSSL_SMALL_STACK
+    if (((B1 = (mp_int *)XMALLOC(sizeof(*B1), heap, DYNAMIC_TYPE_TMP_BUFFER))
+         == NULL) ||
+        ((i1 = (mp_int *)XMALLOC(sizeof(*i1), heap, DYNAMIC_TYPE_TMP_BUFFER))
+         == NULL) ||
+        ((res = (mp_int *)XMALLOC(sizeof(*res), heap, DYNAMIC_TYPE_TMP_BUFFER))
+         == NULL)) {
+        ret = MEMORY_E;
+        goto out;
+    }
+#endif
+
     while (kLen > 0) {
         word32 currentLen;
-        mp_int B1;
 
         ret = DoPKCS12Hash(hashType, buffer, totalLen, Ai, u, iterations);
         if (ret < 0)
@@ -460,55 +477,53 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen,
         for (i = 0; i < (int)v; i++)
             B[i] = Ai[i % u];
 
-        if (mp_init(&B1) != MP_OKAY)
+        if (mp_init(B1) != MP_OKAY)
             ret = MP_INIT_E;
-        else if (mp_read_unsigned_bin(&B1, B, v) != MP_OKAY)
+        else if (mp_read_unsigned_bin(B1, B, v) != MP_OKAY)
             ret = MP_READ_E;
-        else if (mp_add_d(&B1, (mp_digit)1, &B1) != MP_OKAY)
+        else if (mp_add_d(B1, (mp_digit)1, B1) != MP_OKAY)
             ret = MP_ADD_E;
 
         if (ret != 0) {
-            mp_clear(&B1);
+            mp_clear(B1);
             break;
         }
 
         for (i = 0; i < (int)iLen; i += v) {
             int    outSz;
-            mp_int i1;
-            mp_int res;
 
-            if (mp_init_multi(&i1, &res, NULL, NULL, NULL, NULL) != MP_OKAY) {
+            if (mp_init_multi(i1, res, NULL, NULL, NULL, NULL) != MP_OKAY) {
                 ret = MP_INIT_E;
                 break;
             }
-            if (mp_read_unsigned_bin(&i1, I + i, v) != MP_OKAY)
+            if (mp_read_unsigned_bin(i1, I + i, v) != MP_OKAY)
                 ret = MP_READ_E;
-            else if (mp_add(&i1, &B1, &res) != MP_OKAY)
+            else if (mp_add(i1, B1, res) != MP_OKAY)
                 ret = MP_ADD_E;
-            else if ( (outSz = mp_unsigned_bin_size(&res)) < 0)
+            else if ( (outSz = mp_unsigned_bin_size(res)) < 0)
                 ret = MP_TO_E;
             else {
                 if (outSz > (int)v) {
                     /* take off MSB */
                     byte  tmp[WC_MAX_BLOCK_SIZE + 1];
-                    ret = mp_to_unsigned_bin(&res, tmp);
+                    ret = mp_to_unsigned_bin(res, tmp);
                     XMEMCPY(I + i, tmp + 1, v);
                 }
                 else if (outSz < (int)v) {
                     XMEMSET(I + i, 0, v - outSz);
-                    ret = mp_to_unsigned_bin(&res, I + i + v - outSz);
+                    ret = mp_to_unsigned_bin(res, I + i + v - outSz);
                 }
                 else
-                    ret = mp_to_unsigned_bin(&res, I + i);
+                    ret = mp_to_unsigned_bin(res, I + i);
             }
 
-            mp_clear(&i1);
+            mp_clear(i1);
-            mp_clear(&res);
+            mp_clear(res);
             if (ret < 0) break;
         }
 
         if (ret < 0) {
-            mp_clear(&B1);
+            mp_clear(B1);
             break;
         }
 
@@ -516,16 +531,27 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen,
         XMEMCPY(output, Ai, currentLen);
         output += currentLen;
         kLen   -= currentLen;
-        mp_clear(&B1);
+        mp_clear(B1);
     }
 
-    if (dynamic) XFREE(buffer, heap, DYNAMIC_TYPE_KEY);
-
 #ifdef WOLFSSL_SMALL_STACK
+  out:
+
+    if (Ai)
         XFREE(Ai, heap, DYNAMIC_TYPE_TMP_BUFFER);
+    if (B)
         XFREE(B,  heap, DYNAMIC_TYPE_TMP_BUFFER);
+    if (B1)
+        XFREE(B1, heap, DYNAMIC_TYPE_TMP_BUFFER);
+    if (i1)
+        XFREE(i1, heap, DYNAMIC_TYPE_TMP_BUFFER);
+    if (res)
+        XFREE(res, heap, DYNAMIC_TYPE_TMP_BUFFER);
 #endif
 
+    if (dynamic)
+        XFREE(buffer, heap, DYNAMIC_TYPE_KEY);
+
     return ret;
 }
 

wolfcrypt/src/rsa.c

@@ -4097,7 +4097,11 @@ int wc_CheckProbablePrime_ex(const byte* pRaw, word32 pRawSz,
                           const byte* eRaw, word32 eRawSz,
                           int nlen, int* isPrime, WC_RNG* rng)
 {
-    mp_int p, q, e;
+#ifdef WOLFSSL_SMALL_STACK
+    mp_int *p = NULL, *q = NULL, *e = NULL;
+#else
+    mp_int p[1], q[1], e[1];
+#endif
     mp_int* Q = NULL;
     int ret;
 
@@ -4111,30 +4115,54 @@ int wc_CheckProbablePrime_ex(const byte* pRaw, word32 pRawSz,
     if ((qRaw != NULL && qRawSz == 0) || (qRaw == NULL && qRawSz != 0))
         return BAD_FUNC_ARG;
 
-    ret = mp_init_multi(&p, &q, &e, NULL, NULL, NULL);
+#ifdef WOLFSSL_SMALL_STACK
+    if (((p = (mp_int *)XMALLOC(sizeof(*p), NULL, DYNAMIC_TYPE_RSA_BUFFER)) == NULL) ||
+        ((q = (mp_int *)XMALLOC(sizeof(*q), NULL, DYNAMIC_TYPE_RSA_BUFFER)) == NULL) ||
+        ((e = (mp_int *)XMALLOC(sizeof(*e), NULL, DYNAMIC_TYPE_RSA_BUFFER)) == NULL))
+        ret = MEMORY_E;
+    else
+        ret = 0;
+    if (ret == 0)
+#endif
+        ret = mp_init_multi(p, q, e, NULL, NULL, NULL);
 
     if (ret == MP_OKAY)
-        ret = mp_read_unsigned_bin(&p, pRaw, pRawSz);
+        ret = mp_read_unsigned_bin(p, pRaw, pRawSz);
 
     if (ret == MP_OKAY) {
         if (qRaw != NULL) {
-            ret = mp_read_unsigned_bin(&q, qRaw, qRawSz);
+            ret = mp_read_unsigned_bin(q, qRaw, qRawSz);
             if (ret == MP_OKAY)
-                Q = &q;
+                Q = q;
         }
     }
 
     if (ret == MP_OKAY)
-        ret = mp_read_unsigned_bin(&e, eRaw, eRawSz);
+        ret = mp_read_unsigned_bin(e, eRaw, eRawSz);
 
     if (ret == MP_OKAY)
-        ret = _CheckProbablePrime(&p, Q, &e, nlen, isPrime, rng);
+        ret = _CheckProbablePrime(p, Q, e, nlen, isPrime, rng);
 
     ret = (ret == MP_OKAY) ? 0 : PRIME_GEN_E;
 
-    mp_clear(&p);
+#ifdef WOLFSSL_SMALL_STACK
-    mp_clear(&q);
+    if (p) {
-    mp_clear(&e);
+        mp_clear(p);
+        XFREE(p, NULL, DYNAMIC_TYPE_RSA_BUFFER);
+    }
+    if (q) {
+        mp_clear(q);
+        XFREE(q, NULL, DYNAMIC_TYPE_RSA_BUFFER);
+    }
+    if (e) {
+        mp_clear(e);
+        XFREE(e, NULL, DYNAMIC_TYPE_RSA_BUFFER);
+    }
+#else
+    mp_clear(p);
+    mp_clear(q);
+    mp_clear(e);
+#endif
 
     return ret;
 }

wolfcrypt/src/sha256.c

@@ -319,8 +319,10 @@ static int InitSha256(wc_Sha256* sha256)
 
     static WC_INLINE int inline_XTRANSFORM(wc_Sha256* S, const byte* D) {
         int ret;
-        if (Transform_Sha256_is_vectorized)
+        if (Transform_Sha256_is_vectorized) {
-            SAVE_VECTOR_REGISTERS();
+            if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+                return ret;
+        }
         ret = (*Transform_Sha256_p)(S, D);
         if (Transform_Sha256_is_vectorized)
             RESTORE_VECTOR_REGISTERS();
@@ -330,8 +332,10 @@ static int InitSha256(wc_Sha256* sha256)
 
     static WC_INLINE int inline_XTRANSFORM_LEN(wc_Sha256* S, const byte* D, word32 L) {
         int ret;
-        if (Transform_Sha256_is_vectorized)
+        if (Transform_Sha256_is_vectorized) {
-            SAVE_VECTOR_REGISTERS();
+            if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+                return ret;
+        }
         ret = (*Transform_Sha256_Len_p)(S, D, L);
         if (Transform_Sha256_is_vectorized)
             RESTORE_VECTOR_REGISTERS();

wolfcrypt/src/sha512.c

@@ -448,8 +448,10 @@ static int InitSha512_256(wc_Sha512* sha512)
 
     static WC_INLINE int Transform_Sha512(wc_Sha512 *sha512) {
         int ret;
-        if (Transform_Sha512_is_vectorized)
+        if (Transform_Sha512_is_vectorized) {
-            SAVE_VECTOR_REGISTERS();
+            if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+                return ret;
+        }
         ret = (*Transform_Sha512_p)(sha512);
         if (Transform_Sha512_is_vectorized)
             RESTORE_VECTOR_REGISTERS();
@@ -457,8 +459,10 @@ static int InitSha512_256(wc_Sha512* sha512)
     }
     static WC_INLINE int Transform_Sha512_Len(wc_Sha512 *sha512, word32 len) {
         int ret;
-        if (Transform_Sha512_is_vectorized)
+        if (Transform_Sha512_is_vectorized) {
-            SAVE_VECTOR_REGISTERS();
+            if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
+                return ret;
+        }
         ret = (*Transform_Sha512_Len_p)(sha512, len);
         if (Transform_Sha512_is_vectorized)
             RESTORE_VECTOR_REGISTERS();

wolfcrypt/src/wc_port.c

@@ -157,6 +157,15 @@ int wolfCrypt_Init(void)
         }
     #endif
 
+    #if defined(WOLFSSL_LINUXKM_SIMD_X86) \
+        && defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED)
+        ret = allocate_wolfcrypt_irq_fpu_states();
+        if (ret != 0) {
+            WOLFSSL_MSG("allocate_wolfcrypt_irq_fpu_states failed");
+            return ret;
+        }
+    #endif
+
     #if WOLFSSL_CRYPT_HW_MUTEX
         /* If crypto hardware mutex protection is enabled, then initialize it */
         ret = wolfSSL_CryptHwMutexInit();
@@ -356,6 +365,10 @@ int wolfCrypt_Cleanup(void)
         rpcmem_deinit();
         wolfSSL_CleanupHandle();
     #endif
+    #if defined(WOLFSSL_LINUXKM_SIMD_X86) \
+        && defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED)
+        free_wolfcrypt_irq_fpu_states();
+    #endif
     }
 
     return ret;
@@ -2438,9 +2451,15 @@ time_t time(time_t * timer)
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
     struct timespec ts;
     getnstimeofday(&ts);
-    ret = ts.tv_sec * 1000000000LL + ts.tv_nsec;
+    ret = ts.tv_sec;
 #else
-    ret = ktime_get_real_seconds();
+    struct timespec64 ts;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+    ts = current_kernel_time64();
+#else
+    ktime_get_coarse_real_ts64(&ts);
+#endif
+    ret = ts.tv_sec;
 #endif
     if (timer)
         *timer = ret;

wolfcrypt/test/test.c

@@ -9084,7 +9084,12 @@ WOLFSSL_TEST_SUBROUTINE int aes_test(void)
         ret = wc_AesSetKey(enc, niKey, sizeof(niKey), cipher, AES_ENCRYPTION);
         if (ret != 0)
             ERROR_OUT(-5943, out);
+#ifdef WOLFSSL_LINUXKM
+        if (wc_AesEncryptDirect(enc, cipher, niPlain) != 0)
+            ERROR_OUT(-5950, out);
+#else
         wc_AesEncryptDirect(enc, cipher, niPlain);
+#endif
         if (XMEMCMP(cipher, niCipher, AES_BLOCK_SIZE) != 0)
             ERROR_OUT(-5944, out);
 
@@ -9092,7 +9097,12 @@ WOLFSSL_TEST_SUBROUTINE int aes_test(void)
         ret = wc_AesSetKey(dec, niKey, sizeof(niKey), plain, AES_DECRYPTION);
         if (ret != 0)
             ERROR_OUT(-5945, out);
+#ifdef WOLFSSL_LINUXKM
+        if (wc_AesDecryptDirect(dec, plain, niCipher) != 0)
+            ERROR_OUT(-5951, out);
+#else
         wc_AesDecryptDirect(dec, plain, niCipher);
+#endif
         if (XMEMCMP(plain, niPlain, AES_BLOCK_SIZE) != 0)
             ERROR_OUT(-5946, out);
     }

wolfssl/wolfcrypt/aes.h

@@ -351,8 +351,13 @@ WOLFSSL_API int wc_AesEcbDecrypt(Aes* aes, byte* out,
 #endif
 /* AES-DIRECT */
 #if defined(WOLFSSL_AES_DIRECT)
+#ifdef WOLFSSL_LINUXKM
+ WOLFSSL_API __must_check int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in);
+ WOLFSSL_API __must_check int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in);
+#else
  WOLFSSL_API void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in);
  WOLFSSL_API void wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in);
+#endif
  WOLFSSL_API int  wc_AesSetKeyDirect(Aes* aes, const byte* key, word32 len,
                                 const byte* iv, int dir);
 #endif

wolfssl/wolfcrypt/curve25519.h

@@ -73,7 +73,7 @@ typedef struct {
 #endif
 
 /* A CURVE25519 Key */
-typedef struct curve25519_key {
+struct curve25519_key {
     int idx;            /* Index into the ecc_sets[] for the parameters of
                            this curve if -1, this key is using user supplied
                            curve in dp */
@@ -92,7 +92,7 @@ typedef struct curve25519_key {
     /* bit fields */
     byte pubSet:1;
     byte privSet:1;
-} curve25519_key;
+};
 
 enum {
     EC25519_LITTLE_ENDIAN=0,

wolfssl/wolfcrypt/curve448.h

@@ -49,7 +49,7 @@
 #endif
 
 /* A CURVE448 Key */
-typedef struct curve448_key {
+struct curve448_key {
     byte p[CURVE448_PUB_KEY_SIZE];  /* public key  */
     byte k[CURVE448_KEY_SIZE];      /* private key */
 
@@ -60,7 +60,7 @@ typedef struct curve448_key {
     /* bit fields */
     byte pubSet:1;
     byte privSet:1;
-} curve448_key;
+};
 
 enum {
     EC448_LITTLE_ENDIAN = 0,

wolfssl/wolfcrypt/misc.h

@@ -66,7 +66,7 @@ WOLFSSL_LOCAL
 void xorbuf(void*, const void*, word32);
 
 WOLFSSL_LOCAL
-void ForceZero(const void*, word32);
+void ForceZero(void*, word32);
 
 WOLFSSL_LOCAL
 int ConstantCompare(const byte*, const byte*, int);

wolfssl/wolfcrypt/types.h

@@ -320,6 +320,16 @@ decouple library dependencies with standard string, memory and so on.
         #define FALL_THROUGH
     #endif
 
+    #ifndef WARN_UNUSED_RESULT
+        #if defined(WOLFSSL_LINUXKM) && defined(__must_check)
+            #define WARN_UNUSED_RESULT __must_check
+        #elif defined(__GNUC__) && (__GNUC__ >= 4)
+            #define WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+        #else
+            #define WARN_UNUSED_RESULT
+        #endif
+    #endif /* WARN_UNUSED_RESULT */
+
     /* Micrium will use Visual Studio for compilation but not the Win32 API */
     #if defined(_WIN32) && !defined(MICRIUM) && !defined(FREERTOS) && \
         !defined(FREERTOS_TCP) && !defined(EBSNET) && \

wolfssl/wolfcrypt/wc_port.h

@@ -122,29 +122,47 @@
     #endif
     #include <linux/net.h>
     #include <linux/slab.h>
-    #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP)
+    #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_SP_X86_64_ASM)
+        #ifndef CONFIG_X86
+            #error X86 SIMD extensions requested, but CONFIG_X86 is not set.
+        #endif
+        #define WOLFSSL_LINUXKM_SIMD
+        #define WOLFSSL_LINUXKM_SIMD_X86
         #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
             #include <asm/i387.h>
         #else
             #include <asm/simd.h>
         #endif
+        #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)
+            #include <asm/fpu/internal.h>
+        #endif
         #ifndef SAVE_VECTOR_REGISTERS
-            #define SAVE_VECTOR_REGISTERS() kernel_fpu_begin()
+            #define SAVE_VECTOR_REGISTERS() save_vector_registers_x86()
         #endif
         #ifndef RESTORE_VECTOR_REGISTERS
-            #define RESTORE_VECTOR_REGISTERS() kernel_fpu_end()
+            #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_x86()
         #endif
-    #elif defined(WOLFSSL_ARMASM)
+    #elif defined(WOLFSSL_ARMASM) || defined(WOLFSSL_SP_ARM32_ASM) || \
+          defined(WOLFSSL_SP_ARM64_ASM) || defined(WOLFSSL_SP_ARM_THUMB_ASM) ||\
+          defined(WOLFSSL_SP_ARM_CORTEX_M_ASM)
+        #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
+            #error ARM SIMD extensions requested, but CONFIG_ARM* is not set.
+        #endif
+        #define WOLFSSL_LINUXKM_SIMD
+        #define WOLFSSL_LINUXKM_SIMD_ARM
         #include <asm/fpsimd.h>
         #ifndef SAVE_VECTOR_REGISTERS
-            #define SAVE_VECTOR_REGISTERS() ({ preempt_disable(); fpsimd_preserve_current_state(); })
+            #define SAVE_VECTOR_REGISTERS() save_vector_registers_arm()
         #endif
         #ifndef RESTORE_VECTOR_REGISTERS
-            #define RESTORE_VECTOR_REGISTERS() ({ fpsimd_restore_current_state(); preempt_enable(); })
+            #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm()
         #endif
     #else
+        #ifndef WOLFSSL_NO_ASM
+            #define WOLFSSL_NO_ASM
+        #endif
         #ifndef SAVE_VECTOR_REGISTERS
-            #define SAVE_VECTOR_REGISTERS() ({})
+            #define SAVE_VECTOR_REGISTERS() 0
         #endif
         #ifndef RESTORE_VECTOR_REGISTERS
             #define RESTORE_VECTOR_REGISTERS() ({})
@@ -247,10 +265,19 @@
         typeof(kmalloc_order_trace) *kmalloc_order_trace;
 
         typeof(get_random_bytes) *get_random_bytes;
-        typeof(ktime_get_real_seconds) *ktime_get_real_seconds;
+        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
-        typeof(ktime_get_with_offset) *ktime_get_with_offset;
+            typeof(getnstimeofday) *getnstimeofday;
+        #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+            typeof(current_kernel_time64) *current_kernel_time64;
+        #else
+            typeof(ktime_get_coarse_real_ts64) *ktime_get_coarse_real_ts64;
+        #endif
 
-        #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP)
+        struct task_struct *(*get_current)(void);
+        int (*preempt_count)(void);
+
+        #ifdef WOLFSSL_LINUXKM_SIMD_X86
+        typeof(irq_fpu_usable) *irq_fpu_usable;
         /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
          * released in kernel 5.11, backported to 5.4.93
          */
@@ -260,11 +287,32 @@
             typeof(kernel_fpu_begin) *kernel_fpu_begin;
         #endif
         typeof(kernel_fpu_end) *kernel_fpu_end;
+
+        #ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
+        #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
+            typeof(copy_fpregs_to_fpstate) *copy_fpregs_to_fpstate;
+            typeof(copy_kernel_to_fpregs) *copy_kernel_to_fpregs;
+        #else
+            typeof(save_fpregs_to_fpstate) *save_fpregs_to_fpstate;
+            typeof(__restore_fpregs_from_fpstate) *__restore_fpregs_from_fpstate;
+            typeof(xfeatures_mask_all) *xfeatures_mask_all;
         #endif
+        typeof(cpu_number) *cpu_number;
+        typeof(nr_cpu_ids) *nr_cpu_ids;
+        #endif /* WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
+
+        #endif /* WOLFSSL_LINUXKM_SIMD_X86 */
 
         typeof(__mutex_init) *__mutex_init;
+        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
+            typeof(mutex_lock_nested) *mutex_lock_nested;
+        #else
             typeof(mutex_lock) *mutex_lock;
+        #endif
         typeof(mutex_unlock) *mutex_unlock;
+        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
+            typeof(mutex_destroy) *mutex_destroy;
+        #endif
 
         #ifdef HAVE_FIPS
         typeof(wolfCrypt_FIPS_first) *wolfCrypt_FIPS_first;
@@ -326,6 +374,7 @@
     #define kfree (wolfssl_linuxkm_get_pie_redirect_table()->kfree)
     #define ksize (wolfssl_linuxkm_get_pie_redirect_table()->ksize)
     #define krealloc (wolfssl_linuxkm_get_pie_redirect_table()->krealloc)
+    #define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
     #ifdef HAVE_KVMALLOC
         #define kvmalloc_node (wolfssl_linuxkm_get_pie_redirect_table()->kvmalloc_node)
         #define kvfree (wolfssl_linuxkm_get_pie_redirect_table()->kvfree)
@@ -335,21 +384,51 @@
     #define kmalloc_order_trace (wolfssl_linuxkm_get_pie_redirect_table()->kmalloc_order_trace)
 
     #define get_random_bytes (wolfssl_linuxkm_get_pie_redirect_table()->get_random_bytes)
-    #define ktime_get_real_seconds (wolfssl_linuxkm_get_pie_redirect_table()->ktime_get_real_seconds)
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
-    #define ktime_get_with_offset (wolfssl_linuxkm_get_pie_redirect_table()->ktime_get_with_offset)
+        #define getnstimeofday (wolfssl_linuxkm_get_pie_redirect_table()->getnstimeofday)
+    #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
+        #define current_kernel_time64 (wolfssl_linuxkm_get_pie_redirect_table()->current_kernel_time64)
+    #else
+        #define ktime_get_coarse_real_ts64 (wolfssl_linuxkm_get_pie_redirect_table()->ktime_get_coarse_real_ts64)
+    #endif
 
-    #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP)
+    #undef get_current
+    #define get_current (wolfssl_linuxkm_get_pie_redirect_table()->get_current)
+    #undef preempt_count
+    #define preempt_count (wolfssl_linuxkm_get_pie_redirect_table()->preempt_count)
+
+    #ifdef WOLFSSL_LINUXKM_SIMD_X86
+        #define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable)
         #ifdef kernel_fpu_begin
             #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
         #else
             #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
         #endif
         #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
+        #ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
+            #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
+                #define copy_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->copy_fpregs_to_fpstate)
+                #define copy_kernel_to_fpregs (wolfssl_linuxkm_get_pie_redirect_table()->copy_kernel_to_fpregs)
+            #else
+                #define save_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->save_fpregs_to_fpstate)
+                #define __restore_fpregs_from_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->__restore_fpregs_from_fpstate)
+                #define xfeatures_mask_all (*(wolfssl_linuxkm_get_pie_redirect_table()->xfeatures_mask_all))
+            #endif
+            #define cpu_number (*(wolfssl_linuxkm_get_pie_redirect_table()->cpu_number))
+            #define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids))
+        #endif /* WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
     #endif
 
     #define __mutex_init (wolfssl_linuxkm_get_pie_redirect_table()->__mutex_init)
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
+        #define mutex_lock_nested (wolfssl_linuxkm_get_pie_redirect_table()->mutex_lock_nested)
+    #else
         #define mutex_lock (wolfssl_linuxkm_get_pie_redirect_table()->mutex_lock)
+    #endif
     #define mutex_unlock (wolfssl_linuxkm_get_pie_redirect_table()->mutex_unlock)
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
+        #define mutex_destroy (wolfssl_linuxkm_get_pie_redirect_table()->mutex_destroy)
+    #endif
 
     /* per linux/ctype.h, tolower() and toupper() are macros bound to static inlines
      * that use macros that bring in the _ctype global.  for __PIE__, this needs to
@@ -371,6 +450,55 @@
 
     #endif /* USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE */
 
+#ifdef WOLFSSL_LINUXKM_SIMD
+
+#ifdef WOLFSSL_LINUXKM_SIMD_X86
+
+#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
+    extern __must_check int allocate_wolfcrypt_irq_fpu_states(void);
+    extern void free_wolfcrypt_irq_fpu_states(void);
+    extern __must_check int save_vector_registers_x86(void);
+    extern void restore_vector_registers_x86(void);
+#else /* !WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
+    #define save_vector_registers_x86() ({ \
+        int _ret;                          \
+        preempt_disable();                 \
+        if (! irq_fpu_usable()) {          \
+            preempt_enable();              \
+            _ret = BAD_STATE_E;            \
+        } else {                           \
+            kernel_fpu_begin();            \
+            preempt_enable(); /* kernel_fpu_begin() does its own preempt_disable().  decrement ours. */ \
+            _ret = 0;                      \
+        }                                  \
+        _ret;                              \
+    })
+    #define restore_vector_registers_x86() kernel_fpu_end()
+#endif /* !WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
+
+#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+
+    static WARN_UNUSED_RESULT inline int save_vector_registers_arm(void)
+    {
+        preempt_disable();
+        if (! may_use_simd()) {
+            preempt_enable();
+            return BAD_STATE_E;
+        } else {
+            fpsimd_preserve_current_state();
+            return 0;
+        }
+    }
+    static inline void restore_vector_registers_arm(void)
+    {
+        fpsimd_restore_current_state();
+        preempt_enable();
+    }
+
+#endif
+
+#endif /* WOLFSSL_LINUXKM_SIMD */
+
     /* Linux headers define these using C expressions, but we need
      * them to be evaluable by the preprocessor, for use in sp_int.h.
      */
@@ -466,8 +594,6 @@
         extern void fipsEntry(void);
     #endif
 
-    #endif /* BUILDING_WOLFSSL */
-
     /* needed to suppress inclusion of stdio.h in wolfssl/wolfcrypt/types.h */
     #define XSNPRINTF snprintf
 
@@ -481,10 +607,20 @@
           (int)_xatoi_res;                              \
         })
 
+
+    /* suppress false-positive "writing 1 byte into a region of size 0" warnings
+     * building old kernels with new gcc:
+     */
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
+    _Pragma("GCC diagnostic ignored \"-Wstringop-overflow\"");
+    #endif
+
+    #endif /* BUILDING_WOLFSSL */
+
 #else /* ! WOLFSSL_LINUXKM */
 
     #ifndef SAVE_VECTOR_REGISTERS
-        #define SAVE_VECTOR_REGISTERS() do{}while(0)
+        #define SAVE_VECTOR_REGISTERS() 0
     #endif
     #ifndef RESTORE_VECTOR_REGISTERS
         #define RESTORE_VECTOR_REGISTERS() do{}while(0)