Merge pull request #7180 from douzzer/20240126-LINUXKM_LKCAPI_REGISTER

20240126-LINUXKM_LKCAPI_REGISTER

configure.ac

@@ -745,7 +745,7 @@ then
     test "$enable_psk" = "" && enable_psk=yes
     test "$enable_cmac" = "" && enable_cmac=yes
     test "$enable_siphash" = "" && enable_siphash=yes
-    test "$enable_xts" = "" && enable_xts=yes
+    test "$enable_aesxts" = "" && enable_aesxts=yes
     test "$enable_ocsp" = "" && enable_ocsp=yes
     test "$enable_ocspstapling" = "" && test "$enable_ocsp" != "no" && enable_ocspstapling=yes
     test "$enable_ocspstapling2" = "" && test "$enable_ocsp" != "no" && enable_ocspstapling2=yes
@@ -933,7 +933,7 @@ then
     test "$enable_psk" = "" && enable_psk=yes
     test "$enable_cmac" = "" && enable_cmac=yes
     test "$enable_siphash" = "" && enable_siphash=yes
-    test "$enable_xts" = "" && enable_xts=yes
+    test "$enable_aesxts" = "" && enable_aesxts=yes
     test "$enable_ocsp" = "" && enable_ocsp=yes
     test "$enable_ocspstapling" = "" && test "$enable_ocsp" != "no" && enable_ocspstapling=yes
     test "$enable_ocspstapling2" = "" && test "$enable_ocsp" != "no" && enable_ocspstapling2=yes
@@ -4836,17 +4836,23 @@ AS_IF([test "x$ENABLED_CMAC" = "xyes"],
 
 
 # AES-XTS
-AC_ARG_ENABLE([xts],
+AC_ARG_ENABLE([aesxts],
-    [AS_HELP_STRING([--enable-xts],[Enable XTS (default: disabled)])],
+    [AS_HELP_STRING([--enable-aesxts],[Enable AES XTS (default: disabled)])],
-    [ ENABLED_XTS=$enableval ],
+    [ ENABLED_AESXTS=$enableval ],
-    [ ENABLED_XTS=no ]
+    [ ENABLED_AESXTS=no ]
     )
 
-AS_IF([test "x$ENABLED_XTS" = "xyes"],
+# legacy old option name, for compatibility:
+AC_ARG_ENABLE([xts],
+    [AS_HELP_STRING([--enable-xts],[Please use --enable-aesxts])],
+    [ ENABLED_AESXTS=$enableval ]
+    )
+
+AS_IF([test "x$ENABLED_AESXTS" = "xyes"],
       [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_AES_XTS -DWOLFSSL_AES_DIRECT"])
-AS_IF([test "x$ENABLED_XTS" = "xyes" && test "x$ENABLED_INTELASM" = "xyes"],
+AS_IF([test "x$ENABLED_AESXTS" = "xyes" && test "x$ENABLED_INTELASM" = "xyes"],
       [AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_AES_XTS"])
-AS_IF([test "x$ENABLED_XTS" = "xyes" && test "x$ENABLED_AESNI" = "xyes"],
+AS_IF([test "x$ENABLED_AESXTS" = "xyes" && test "x$ENABLED_AESNI" = "xyes"],
       [AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_AES_XTS"])
 
 # Web Server Build
@@ -8028,6 +8034,37 @@ if test -n "$MPI_MAX_KEY_BITS" -o -n "$WITH_MAX_ECC_BITS"; then
    fi
 fi
 
+AC_ARG_ENABLE([linuxkm-lkcapi-register],
+    [AS_HELP_STRING([--enable-linuxkm-lkcapi-register],[Register wolfCrypt implementations with the Linux Kernel Crypto API backplane.  Possible values are "none", "all", "cbc(aes)", "cfb(aes)", "gcm(aes)", and "xts(aes)", or a comma-separate combination. (default: none)])],
+    [ENABLED_LINUXKM_LKCAPI_REGISTER=$enableval],
+    [ENABLED_LINUXKM_LKCAPI_REGISTER=none]
+    )
+if test "$ENABLED_LINUXKM_LKCAPI_REGISTER" != "none"
+then
+    AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER"
+
+    if test "$ENABLED_AESGCM" != "no" && test "$ENABLED_AESGCM_STREAM" = "no" && test "$ENABLED_AESNI" = "no" && test "$ENABLED_ARMASM" = "no" && test "$ENABLED_FIPS" = "no"; then
+        ENABLED_AESGCM_STREAM=yes
+    fi
+
+    for lkcapi_alg in $(echo "$ENABLED_LINUXKM_LKCAPI_REGISTER" | tr ',' ' ')
+    do
+        case "$lkcapi_alg" in
+        all) AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_ALL"           ;;
+        'cbc(aes)') test "$ENABLED_AESCBC" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-CBC implementation not enabled.])
+                    AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESCBC" ;;
+        'cfb(aes)') test "$ENABLED_AESCFB" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-CFB implementation not enabled.])
+                    AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESCFB" ;;
+        'gcm(aes)') test "$ENABLED_AESGCM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-GCM implementation not enabled.])
+                    test "$ENABLED_AESGCM_STREAM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: --enable-aesgcm-stream is required for LKCAPI.])
+                    AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESGCM" ;;
+        'xts(aes)') test "$ENABLED_AESXTS" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-XTS implementation not enabled.])
+                    AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESXTS" ;;
+        *) AC_MSG_ERROR([Unsupported LKCAPI algorithm "$lkcapi_alg".])      ;;
+        esac
+    done
+fi
+
 # Library Suffix
 LIBSUFFIX=""
 AC_ARG_WITH([libsuffix],
@@ -8958,7 +8995,7 @@ AM_CONDITIONAL([BUILD_SNIFFER],  [ test "x$ENABLED_SNIFFER"   = "xyes" || test "
 AM_CONDITIONAL([BUILD_SNIFFTEST],[ test "x$ENABLED_SNIFFTEST" = "xyes"])
 AM_CONDITIONAL([BUILD_AESGCM],[test "x$ENABLED_AESGCM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_AESCCM],[test "x$ENABLED_AESCCM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
-AM_CONDITIONAL([BUILD_XTS],[test "x$ENABLED_XTS" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
+AM_CONDITIONAL([BUILD_AESXTS],[test "x$ENABLED_AESXTS" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_ARMASM],[test "x$ENABLED_ARMASM" = "xyes"])
 AM_CONDITIONAL([BUILD_ARMASM_INLINE],[test "x$ENABLED_ARMASM_INLINE" = "xyes"])
 AM_CONDITIONAL([BUILD_ARMASM_CRYPTO],[test "x$ENABLED_ARMASM_CRYPTO" = "xyes"])
@@ -9397,6 +9434,7 @@ echo "   * AES-CCM:                    $ENABLED_AESCCM"
 echo "   * AES-CTR:                    $ENABLED_AESCTR"
 echo "   * AES-CFB:                    $ENABLED_AESCFB"
 echo "   * AES-OFB:                    $ENABLED_AESOFB"
+echo "   * AES-XTS:                    $ENABLED_AESXTS"
 echo "   * AES-SIV:                    $ENABLED_AESSIV"
 echo "   * AES-EAX:                    $ENABLED_AESEAX"
 echo "   * AES Bitspliced:             $ENABLED_AESBS"

linuxkm/Kbuild

@@ -32,6 +32,10 @@ WOLFSSL_CFLAGS += -ffreestanding -Wframe-larger-than=$(MAX_STACK_FRAME_SIZE) -is
 
 ifeq "$(KERNEL_ARCH)" "x86"
     WOLFSSL_CFLAGS += -mpreferred-stack-boundary=4
+else ifeq "$(KERNEL_ARCH)" "aarch64"
+    WOLFSSL_CFLAGS += -mno-outline-atomics
+else ifeq "$(KERNEL_ARCH)" "arm64"
+    WOLFSSL_CFLAGS += -mno-outline-atomics
 endif
 
 obj-m := libwolfssl.o
@@ -47,9 +51,14 @@ $(obj)/linuxkm/module_exports.o: $(WOLFSSL_OBJ_TARGETS)
 # this mechanism only works in kernel 5.x+ (fallback to hardcoded value)
 hostprogs := linuxkm/get_thread_size
 always-y := $(hostprogs)
+
+HOST_EXTRACFLAGS += $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(KBUILD_CFLAGS) -static -fno-omit-frame-pointer
+
 # "-mindirect-branch=keep -mfunction-return=keep" to avoid "undefined reference
 #  to `__x86_return_thunk'" on CONFIG_RETHUNK kernels (5.19.0-rc7)
-HOST_EXTRACFLAGS += $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(KBUILD_CFLAGS) -static -fno-omit-frame-pointer -mindirect-branch=keep -mfunction-return=keep
+ifeq "$(KERNEL_ARCH)" "x86"
+  HOST_EXTRACFLAGS += -mindirect-branch=keep -mfunction-return=keep
+endif
 
 # this rule is needed to get build to succeed in 4.x (get_thread_size still doesn't get built)
 $(obj)/linuxkm/get_thread_size: $(src)/linuxkm/get_thread_size.c
@@ -149,10 +158,10 @@ ifneq "$(quiet)" "silent_"
 endif
 	@cd "$(obj)" || exit $$?; \
 	for file in $(WOLFCRYPT_PIE_FILES); do \
-	    $(OBJCOPY) --rename-section .text=.text.wolfcrypt --rename-section .data=.data.wolfcrypt "$$file" || exit $$?; \
+	    $(OBJCOPY) --rename-section .text=.text.wolfcrypt --rename-section .data=.data.wolfcrypt --rename-section .rodata=.rodata.wolfcrypt "$$file" || exit $$?; \
 	done
 ifneq "$(quiet)" "silent_"
-	@echo '  wolfCrypt .{text,data} sections containerized to .{text,data}.wolfcrypt'
+	@echo '  wolfCrypt .{text,data,rodata} sections containerized to .{text,data,rodata}.wolfcrypt'
 endif
 
 $(src)/linuxkm/module_exports.c: rename-pie-text-and-data-sections

linuxkm/include.am

@@ -12,4 +12,5 @@ EXTRA_DIST += m4/ax_linuxkm.m4 \
 	      linuxkm/pie_redirect_table.c \
 	      linuxkm/pie_last.c \
 	      linuxkm/linuxkm_memory.c \
-	      linuxkm/linuxkm_wc_port.h
+	      linuxkm/linuxkm_wc_port.h \
+	      linuxkm/lkcapi_glue.c

linuxkm/linuxkm_memory.c

@@ -275,7 +275,7 @@ WARN_UNUSED_RESULT int save_vector_registers_x86(void)
 {
     struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(1);
     if (pstate == NULL)
-        return ENOMEM;
+        return MEMORY_E;
 
     /* allow for nested calls */
     if (pstate->fpu_state != 0U) {
@@ -314,7 +314,7 @@ WARN_UNUSED_RESULT int save_vector_registers_x86(void)
         if (! warned_fpu_forbidden)
             pr_err("save_vector_registers_x86 called from IRQ handler.\n");
         wc_linuxkm_fpu_state_release(pstate);
-        return EPERM;
+        return BAD_STATE_E;
     } else {
 #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
     (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
@@ -380,3 +380,11 @@ void my__show_free_areas(
     return;
 }
 #endif
+
+#if defined(__PIE__) && defined(CONFIG_FORTIFY_SOURCE)
+/* needed because FORTIFY_SOURCE inline implementations call fortify_panic(). */
+void __my_fortify_panic(const char *name) {
+    pr_emerg("__my_fortify_panic in %s\n", name);
+    BUG();
+}
+#endif

linuxkm/linuxkm_wc_port.h

@@ -65,8 +65,8 @@
           (int)_xatoi_res;                              \
         })
 
-    /* Kbuild+gcc on x86 doesn't consistently honor the default ALIGN16 on stack objects,
+    /* Kbuild+gcc on x86 doesn't consistently honor the default ALIGN16 on stack
-     *   but gives adequate alignment with "32".
+     * objects, but gives adequate alignment with "32".
      */
     #if defined(CONFIG_X86) && !defined(ALIGN16)
         #define ALIGN16 __attribute__ ( (aligned (32)))
@@ -119,8 +119,133 @@
     #include <linux/kconfig.h>
     #include <linux/kernel.h>
     #include <linux/ctype.h>
+
+    #ifdef CONFIG_FORTIFY_SOURCE
+        #ifdef __PIE__
+            /* the inline definitions in fortify-string.h use non-inline
+             * fortify_panic().
+             */
+            extern void __my_fortify_panic(const char *name) __noreturn __cold;
+            #define fortify_panic __my_fortify_panic
+        #endif
+
+        /* the _FORTIFY_SOURCE macros and implementations for several string
+         * functions are incompatible with libwolfssl, so just reimplement with
+         * inlines and remap with macros.
+         */
+
+        #define __ARCH_STRLEN_NO_REDIRECT
+        #define __ARCH_MEMCPY_NO_REDIRECT
+        #define __ARCH_MEMSET_NO_REDIRECT
+        #define __ARCH_MEMMOVE_NO_REDIRECT
+
+        /* the inline definitions in fortify-string.h use non-inline
+         * strlen().
+         */
+        static inline size_t strlen(const char *s) {
+            const char *s_start = s;
+            while (*s)
+                ++s;
+            return (size_t)((uintptr_t)s - (uintptr_t)s_start);
+        }
+
+        #include <linux/string.h>
+
+        #undef strlen
+        #define strlen(s) \
+            ((__builtin_constant_p(s) && __builtin_constant_p(*(s))) ? \
+             (sizeof(s) - 1) : strlen(s))
+
+        static inline void *my_memcpy(void *dest, const void *src, size_t n) {
+            if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n)
+                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
+            {
+                uintptr_t *src_longs = (uintptr_t *)src,
+                    *dest_longs = (uintptr_t *)dest,
+                    *endp = (uintptr_t *)((u8 *)src + n);
+                while (src_longs < endp)
+                    *dest_longs++ = *src_longs++;
+            } else {
+                u8 *src_bytes = (u8 *)src,
+                    *dest_bytes = (u8 *)dest,
+                    *endp = src_bytes + n;
+                while (src_bytes < endp)
+                    *dest_bytes++ = *src_bytes++;
+            }
+            return dest;
+        }
+        #undef memcpy
+        #define memcpy my_memcpy
+
+        static inline void *my_memset(void *dest, int c, size_t n) {
+            if (! (((uintptr_t)dest | (uintptr_t)n)
+                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
+            {
+                uintptr_t c_long = __builtin_choose_expr(
+                    sizeof(uintptr_t) == 8,
+                    (uintptr_t)(u8)c * 0x0101010101010101UL,
+                    (uintptr_t)(u8)c * 0x01010101U
+                    );
+                uintptr_t *dest_longs = (uintptr_t *)dest,
+                    *endp = (uintptr_t *)((u8 *)dest_longs + n);
+                while (dest_longs < endp)
+                    *dest_longs++ = c_long;
+            } else {
+                u8 *dest_bytes = (u8 *)dest, *endp = dest_bytes + n;
+                while (dest_bytes < endp)
+                    *dest_bytes++ = (u8)c;
+            }
+            return dest;
+        }
+        #undef memset
+        #define memset my_memset
+
+        static inline void *my_memmove(void *dest, const void *src, size_t n) {
+            if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n)
+                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
+            {
+                uintptr_t *src_longs = (uintptr_t *)src,
+                    *dest_longs = (uintptr_t *)dest;
+                n >>= __builtin_choose_expr(
+                    sizeof(uintptr_t) == 8,
+                    3U,
+                    2U);
+                if (src_longs < dest_longs) {
+                    uintptr_t *startp = src_longs;
+                    src_longs += n - 1;
+                    dest_longs += n - 1;
+                    while (src_longs >= startp)
+                        *dest_longs-- = *src_longs--;
+                } else if (src_longs > dest_longs) {
+                    uintptr_t *endp = src_longs + n;
+                    while (src_longs < endp)
+                        *dest_longs++ = *src_longs++;
+                }
+            } else {
+                u8 *src_bytes = (u8 *)src, *dest_bytes = (u8 *)dest;
+                if (src_bytes < dest_bytes) {
+                    u8 *startp = src_bytes;
+                    src_bytes += n - 1;
+                    dest_bytes += n - 1;
+                    while (src_bytes >= startp)
+                        *dest_bytes-- = *src_bytes--;
+                } else if (src_bytes > dest_bytes) {
+                    u8 *endp = src_bytes + n;
+                    while (src_bytes < endp)
+                        *dest_bytes++ = *src_bytes++;
+                }
+            }
+            return dest;
+        }
+        #undef memmove
+        #define memmove my_memmove
+
+    #endif /* CONFIG_FORTIFY_SOURCE */
+
     #include <linux/init.h>
     #include <linux/module.h>
+    #include <linux/delay.h>
+
     #ifdef __PIE__
         /* without this, mm.h brings in static, but not inline, pmd_to_page(),
          * with direct references to global vmem variables.
@@ -146,7 +271,33 @@
     #include <linux/net.h>
     #include <linux/slab.h>
 
-    #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_SP_X86_64_ASM)
+    #ifdef LINUXKM_LKCAPI_REGISTER
+        #include <linux/crypto.h>
+        #include <linux/scatterlist.h>
+        #include <crypto/scatterwalk.h>
+        #include <crypto/internal/aead.h>
+        #include <crypto/internal/skcipher.h>
+
+        /* the LKCAPI assumes that expanded encrypt and decrypt keys will stay
+         * loaded simultaneously, and the Linux in-tree implementations have two
+         * AES key structs in each context, one for each direction.  in
+         * linuxkm/lkcapi_glue.c (used for CBC, CFB, and GCM), we do the same
+         * thing with "struct km_AesCtx".  however, wolfCrypt struct AesXts
+         * already has two AES expanded keys, the main and tweak, and the tweak
+         * is always used in the encrypt direction regardless of the main
+         * direction.  to avoid allocating and computing a duplicate second
+         * tweak encrypt key, we set
+         * WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS, which adds a second
+         * Aes slot to wolfCrypt's struct AesXts, and activates support for
+         * AES_ENCRYPTION_AND_DECRYPTION on AES-XTS.
+         */
+        #ifndef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+        #define WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+        #endif
+    #endif
+
+    #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || \
+        defined(WOLFSSL_SP_X86_64_ASM)
         #ifndef CONFIG_X86
             #error X86 SIMD extensions requested, but CONFIG_X86 is not set.
         #endif
@@ -172,21 +323,39 @@
         #endif
     #endif
 
-    /* benchmarks.c uses floating point math, so needs a working SAVE_VECTOR_REGISTERS(). */
+    /* benchmarks.c uses floating point math, so needs a working
-    #if defined(WOLFSSL_LINUXKM_BENCHMARKS) && !defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS)
+     * SAVE_VECTOR_REGISTERS().
+     */
+    #if defined(WOLFSSL_LINUXKM_BENCHMARKS) && \
+        !defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS)
         #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
     #endif
 
-    #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
+    #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && \
+        defined(CONFIG_X86)
         #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
             #include <asm/i387.h>
         #else
             #include <asm/simd.h>
         #endif
         #ifndef SAVE_VECTOR_REGISTERS
-            #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_x86(); if (_svr_ret != 0) { fail_clause } }
+            #define SAVE_VECTOR_REGISTERS(fail_clause) {    \
+                int _svr_ret = save_vector_registers_x86(); \
+                if (_svr_ret != 0) {                        \
+                    fail_clause                             \
+                }                                           \
+            }
+            #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
+                #define SAVE_VECTOR_REGISTERS2() ({                    \
+                    int _fuzzer_ret = SAVE_VECTOR_REGISTERS2_fuzzer(); \
+                    (_fuzzer_ret == 0) ?                               \
+                     save_vector_registers_x86() :                     \
+                     _fuzzer_ret;                                      \
+                })
+            #else
                 #define SAVE_VECTOR_REGISTERS2() save_vector_registers_x86()
             #endif
+        #endif
         #ifndef RESTORE_VECTOR_REGISTERS
             #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_x86()
         #endif
@@ -291,6 +460,11 @@
         #else
             typeof(printk) *printk;
         #endif
+
+#ifdef CONFIG_FORTIFY_SOURCE
+        typeof(__warn_printk) *__warn_printk;
+#endif
+
         typeof(snprintf) *snprintf;
 
         const unsigned char *_ctype;
@@ -434,6 +608,11 @@
     #else
         #define printk (wolfssl_linuxkm_get_pie_redirect_table()->printk)
     #endif
+
+    #ifdef CONFIG_FORTIFY_SOURCE
+        #define __warn_printk (wolfssl_linuxkm_get_pie_redirect_table()->__warn_printk)
+    #endif
+
     #define snprintf (wolfssl_linuxkm_get_pie_redirect_table()->snprintf)
 
     #define _ctype (wolfssl_linuxkm_get_pie_redirect_table()->_ctype)
@@ -643,8 +822,9 @@
         #define realloc(ptr, newsize) krealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), GFP_KERNEL)
     #endif
 
-#ifdef WOLFSSL_TRACK_MEMORY
     #include <wolfssl/wolfcrypt/memory.h>
+
+#ifdef WOLFSSL_TRACK_MEMORY
     #define XMALLOC(s, h, t)     ({(void)(h); (void)(t); wolfSSL_Malloc(s);})
     #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK
         #define XFREE(p, h, t)       ({(void)(h); (void)(t); wolfSSL_Free(p);})

linuxkm/module_hooks.c

@@ -47,7 +47,6 @@
 #endif
 #ifndef NO_CRYPT_TEST
     #include <wolfcrypt/test/test.h>
-    #include <linux/delay.h>
 #endif
 
 static int libwolfssl_cleanup(void) {
@@ -71,6 +70,8 @@ static int libwolfssl_cleanup(void) {
 
 #ifdef HAVE_LINUXKM_PIE_SUPPORT
 
+#ifdef DEBUG_LINUXKM_PIE_SUPPORT
+
 extern int wolfCrypt_PIE_first_function(void);
 extern int wolfCrypt_PIE_last_function(void);
 extern const unsigned int wolfCrypt_PIE_rodata_start[];
@@ -90,6 +91,8 @@ static unsigned int hash_span(char *start, char *end) {
     return sum;
 }
 
+#endif /* DEBUG_LINUXKM_PIE_SUPPORT */
+
 #ifdef USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE
 extern struct wolfssl_linuxkm_pie_redirect_table wolfssl_linuxkm_pie_redirect_table;
 static int set_up_wolfssl_linuxkm_pie_redirect_table(void);
@@ -118,7 +121,6 @@ static int updateFipsHash(void);
 #endif
 
 #ifdef WOLFSSL_LINUXKM_BENCHMARKS
-#undef HAVE_PTHREAD
 #define STRING_USER
 #define NO_MAIN_FUNCTION
 #define current_time benchmark_current_time
@@ -126,6 +128,10 @@ static int updateFipsHash(void);
 #include "wolfcrypt/benchmark/benchmark.c"
 #endif /* WOLFSSL_LINUXKM_BENCHMARKS */
 
+#ifdef LINUXKM_LKCAPI_REGISTER
+    #include "linuxkm/lkcapi_glue.c"
+#endif
+
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)
 static int __init wolfssl_init(void)
 #else
@@ -152,7 +158,7 @@ static int wolfssl_init(void)
         return ret;
 #endif
 
-#ifdef HAVE_LINUXKM_PIE_SUPPORT
+#if defined(HAVE_LINUXKM_PIE_SUPPORT) && defined(DEBUG_LINUXKM_PIE_SUPPORT)
 
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)
     /* see linux commit ac3b432839 */
@@ -219,7 +225,7 @@ static int wolfssl_init(void)
                 text_hash, pie_text_end-pie_text_start,
                 rodata_hash, pie_rodata_end-pie_rodata_start);
     }
-#endif /* HAVE_LINUXKM_PIE_SUPPORT */
+#endif /* HAVE_LINUXKM_PIE_SUPPORT && DEBUG_LINUXKM_PIE_SUPPORT */
 
 #ifdef HAVE_FIPS
     ret = wolfCrypt_SetCb_fips(lkmFipsCb);
@@ -239,19 +245,32 @@ static int wolfssl_init(void)
         return -ECANCELED;
     }
 
-    pr_info("wolfCrypt FIPS ["
+    pr_info("FIPS 140-3 wolfCrypt-fips v%d.%d.%d%s%s startup "
-
+            "self-test succeeded.\n",
-#if defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION == 3)
+#ifdef HAVE_FIPS_VERSION_MAJOR
-            "ready"
+            HAVE_FIPS_VERSION_MAJOR,
-#elif defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION == 2) \
-    && defined(WOLFCRYPT_FIPS_RAND)
-            "140-2 rand"
-#elif defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION == 2)
-            "140-2"
 #else
-            "140"
+            HAVE_FIPS_VERSION,
 #endif
-            "] POST succeeded.\n");
+#ifdef HAVE_FIPS_VERSION_MINOR
+            HAVE_FIPS_VERSION_MINOR,
+#else
+            0,
+#endif
+#ifdef HAVE_FIPS_VERSION_PATCH
+            HAVE_FIPS_VERSION_PATCH,
+#else
+            0,
+#endif
+#ifdef HAVE_FIPS_VERSION_PORT
+            "-",
+            HAVE_FIPS_VERSION_PORT
+#else
+            "",
+            ""
+#endif
+        );
+
 #endif /* HAVE_FIPS */
 
 #ifdef WC_RNG_SEED_CB
@@ -287,6 +306,21 @@ static int wolfssl_init(void)
         return -ECANCELED;
     }
     pr_info("wolfCrypt self-test passed.\n");
+#else
+    pr_info("skipping full wolfcrypt_test() "
+            "(configure with --enable-crypttests to enable).\n");
+#endif
+
+#ifdef LINUXKM_LKCAPI_REGISTER
+    ret = linuxkm_lkcapi_register();
+
+    if (ret) {
+        pr_err("linuxkm_lkcapi_register() failed with return code %d.\n", ret);
+        linuxkm_lkcapi_unregister();
+        (void)libwolfssl_cleanup();
+        msleep(10);
+        return -ECANCELED;
+    }
 #endif
 
 #ifdef WOLFSSL_LINUXKM_BENCHMARKS
@@ -326,6 +360,10 @@ static void __exit wolfssl_exit(void)
 static void wolfssl_exit(void)
 #endif
 {
+#ifdef LINUXKM_LKCAPI_REGISTER
+    linuxkm_lkcapi_unregister();
+#endif
+
     (void)libwolfssl_cleanup();
 
     return;
@@ -375,6 +413,7 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
 #ifndef __ARCH_MEMCMP_NO_REDIRECT
     wolfssl_linuxkm_pie_redirect_table.memcmp = memcmp;
 #endif
+#ifndef CONFIG_FORTIFY_SOURCE
 #ifndef __ARCH_MEMCPY_NO_REDIRECT
     wolfssl_linuxkm_pie_redirect_table.memcpy = memcpy;
 #endif
@@ -384,6 +423,7 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
 #ifndef __ARCH_MEMMOVE_NO_REDIRECT
     wolfssl_linuxkm_pie_redirect_table.memmove = memmove;
 #endif
+#endif /* !CONFIG_FORTIFY_SOURCE */
 #ifndef __ARCH_STRCMP_NO_REDIRECT
     wolfssl_linuxkm_pie_redirect_table.strcmp = strcmp;
 #endif
@@ -415,6 +455,11 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     #else
         wolfssl_linuxkm_pie_redirect_table.printk = printk;
     #endif
+
+#ifdef CONFIG_FORTIFY_SOURCE
+    wolfssl_linuxkm_pie_redirect_table.__warn_printk = __warn_printk;
+#endif
+
     wolfssl_linuxkm_pie_redirect_table.snprintf = snprintf;
 
     wolfssl_linuxkm_pie_redirect_table._ctype = _ctype;
@@ -516,11 +561,15 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     /* runtime assert that the table has no null slots after initialization. */
     {
         unsigned long *i;
+        static_assert(sizeof(unsigned long) == sizeof(void *),
+                      "unexpected pointer size");
         for (i = (unsigned long *)&wolfssl_linuxkm_pie_redirect_table;
              i < (unsigned long *)&wolfssl_linuxkm_pie_redirect_table._last_slot;
              ++i)
             if (*i == 0) {
-                pr_err("wolfCrypt container redirect table initialization was incomplete.\n");
+                pr_err("wolfCrypt container redirect table initialization was "
+                       "incomplete [%lu].\n",
+                       i-(unsigned long *)&wolfssl_linuxkm_pie_redirect_table);
                 return -EFAULT;
             }
     }
@@ -711,11 +760,19 @@ static int updateFipsHash(void)
         }
     }
 
-    if (XMEMCMP(hash, binVerify, WC_SHA256_DIGEST_SIZE) == 0)
+    if (XMEMCMP(hash, binVerify, WC_SHA256_DIGEST_SIZE) == 0) {
+#if defined(DEBUG_LINUXKM_PIE_SUPPORT) || defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG)
+        pr_info("updateFipsHash: verifyCore already matches [%s]\n", verifyCore);
+#else
         pr_info("updateFipsHash: verifyCore already matches.\n");
-    else {
+#endif
+    } else {
         XMEMCPY(verifyCore, base16_hash, WC_SHA256_DIGEST_SIZE*2 + 1);
+#if defined(DEBUG_LINUXKM_PIE_SUPPORT) || defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG)
+        pr_info("updateFipsHash: verifyCore updated [%s].\n", base16_hash);
+#else
         pr_info("updateFipsHash: verifyCore updated.\n");
+#endif
     }
 
     ret = 0;

wolfcrypt/src/aes.c

@@ -6383,7 +6383,6 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
 #endif
         return BAD_FUNC_ARG;
     }
-
 #ifdef OPENSSL_EXTRA
     XMEMSET(aes->gcm.aadH, 0, sizeof(aes->gcm.aadH));
     aes->gcm.aadLen = 0;
@@ -12250,9 +12249,17 @@ int wc_AesKeyUnWrap(const byte* key, word32 keySz, const byte* in, word32 inSz,
 
 #ifdef WOLFSSL_AES_XTS
 
-/* Galios Field to use */
+/* Galois Field to use */
 #define GF_XTS 0x87
 
+/* Set up keys for encryption and/or decryption.
+ *
+ * aes   buffer holding aes subkeys
+ * heap  heap hint to use for memory. Can be NULL
+ * devId id to use with async crypto. Can be 0
+ *
+ * return 0 on success
+ */
 int wc_AesXtsInit(XtsAes* aes, void* heap, int devId)
 {
     int    ret = 0;
@@ -12265,22 +12272,28 @@ int wc_AesXtsInit(XtsAes* aes, void* heap, int devId)
         return ret;
     }
     if ((ret = wc_AesInit(&aes->aes, heap, devId)) != 0) {
+        (void)wc_AesFree(&aes->tweak);
         return ret;
     }
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    if ((ret = wc_AesInit(&aes->aes_decrypt, heap, devId)) != 0) {
+        (void)wc_AesFree(&aes->tweak);
+        (void)wc_AesFree(&aes->aes);
+        return ret;
+    }
+#endif
 
     return 0;
 }
 
-/* This is to help with setting keys to correct encrypt or decrypt type.
+/* Set up keys for encryption and/or decryption.
  *
- * tweak AES key for tweak in XTS
+ * aes   buffer holding aes subkeys
- * aes   AES key for encrypt/decrypt process
+ * key   AES key for encrypt/decrypt and tweak process (concatenated)
- * key   buffer holding aes key | tweak key
  * len   length of key buffer in bytes. Should be twice that of key size. i.e.
  *       32 for a 16 byte key.
- * dir   direction, either AES_ENCRYPTION or AES_DECRYPTION
+ * dir   direction: AES_ENCRYPTION, AES_DECRYPTION, or
- * heap  heap hint to use for memory. Can be NULL
+ *       AES_ENCRYPTION_AND_DECRYPTION
- * devId id to use with async crypto. Can be 0
  *
  * return 0 on success
  */
@@ -12293,28 +12306,83 @@ int wc_AesXtsSetKeyNoInit(XtsAes* aes, const byte* key, word32 len, int dir)
         return BAD_FUNC_ARG;
     }
 
+    if ((dir != AES_ENCRYPTION) && (dir != AES_DECRYPTION)
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+        && (dir != AES_ENCRYPTION_AND_DECRYPTION)
+#endif
+        )
+    {
+        return BAD_FUNC_ARG;
+    }
+
     keySz = len/2;
-    if (keySz != 16 && keySz != 32) {
+    if (keySz != AES_128_KEY_SIZE && keySz != AES_256_KEY_SIZE) {
         WOLFSSL_MSG("Unsupported key size");
         return WC_KEY_SIZE_E;
     }
 
-    if ((ret = wc_AesSetKey(&aes->aes, key, keySz, NULL, dir)) == 0) {
+#ifdef HAVE_FIPS
-        ret = wc_AesSetKey(&aes->tweak, key + keySz, keySz, NULL,
+    if (XMEMCMP(key, key + keySz, keySz) == 0) {
-                AES_ENCRYPTION);
+        WOLFSSL_MSG("FIPS AES-XTS main and tweak keys must differ");
-        if (ret != 0) {
+        return BAD_FUNC_ARG;
-            wc_AesFree(&aes->aes);
-        }
-#ifdef WOLFSSL_AESNI
-        if (aes->aes.use_aesni != aes->tweak.use_aesni) {
-            if (aes->aes.use_aesni)
-                aes->aes.use_aesni = 0;
-            else
-                aes->tweak.use_aesni = 0;
     }
 #endif
+
+    if (dir == AES_ENCRYPTION
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+        || dir == AES_ENCRYPTION_AND_DECRYPTION
+#endif
+        )
+    {
+        ret = wc_AesSetKey(&aes->aes, key, keySz, NULL, AES_ENCRYPTION);
     }
 
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    if ((ret == 0) && ((dir == AES_DECRYPTION)
+                       || (dir == AES_ENCRYPTION_AND_DECRYPTION)))
+        ret = wc_AesSetKey(&aes->aes_decrypt, key, keySz, NULL, AES_DECRYPTION);
+#else
+    if (dir == AES_DECRYPTION)
+        ret = wc_AesSetKey(&aes->aes, key, keySz, NULL, AES_DECRYPTION);
+#endif
+
+    if (ret == 0)
+        ret = wc_AesSetKey(&aes->tweak, key + keySz, keySz, NULL,
+                AES_ENCRYPTION);
+
+#ifdef WOLFSSL_AESNI
+    if (ret == 0) {
+        /* With WC_AES_C_DYNAMIC_FALLBACK, the main and tweak keys could have
+         * conflicting _aesni status, but the AES-XTS asm implementations need
+         * them to all be AESNI.  If any aren't, disable AESNI on all.
+         */
+        if ((((dir == AES_ENCRYPTION)
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+              || (dir == AES_ENCRYPTION_AND_DECRYPTION)
+#endif
+             ) &&
+             (aes->aes.use_aesni != aes->tweak.use_aesni))
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+            ||
+            (((dir == AES_DECRYPTION)
+              || (dir == AES_ENCRYPTION_AND_DECRYPTION)) &&
+             (aes->aes_decrypt.use_aesni != aes->tweak.use_aesni))
+#endif
+            )
+        {
+#ifdef WC_AES_C_DYNAMIC_FALLBACK
+            aes->aes.use_aesni = 0;
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+            aes->aes_decrypt.use_aesni = 0;
+#endif
+            aes->tweak.use_aesni = 0;
+#else
+            ret = SYSLIB_FAILED_E;
+#endif
+        }
+    }
+#endif
+
     return ret;
 }
 
@@ -12356,6 +12424,9 @@ int wc_AesXtsFree(XtsAes* aes)
 {
     if (aes != NULL) {
         wc_AesFree(&aes->aes);
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+        wc_AesFree(&aes->aes_decrypt);
+#endif
         wc_AesFree(&aes->tweak);
     }
 
@@ -12615,10 +12686,19 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
 {
     int ret;
 
+    Aes *aes;
+
     if (xaes == NULL || out == NULL || in == NULL) {
         return BAD_FUNC_ARG;
     }
 
+    aes = &xaes->aes;
+
+    if (aes->keylen == 0) {
+        WOLFSSL_MSG("wc_AesXtsEncrypt called with unset encryption key.");
+        return BAD_FUNC_ARG;
+    }
+
     if (iSz < AES_BLOCK_SIZE) {
         return BAD_FUNC_ARG;
     }
@@ -12631,28 +12711,33 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
     {
 #ifdef WOLFSSL_AESNI
 #ifdef WC_AES_C_DYNAMIC_FALLBACK
-        int orig_use_aesni = xaes->aes.use_aesni;
+        int orig_use_aesni = aes->use_aesni;
 #endif
-        if (xaes->aes.use_aesni && ((ret = SAVE_VECTOR_REGISTERS2()) != 0)) {
+
+        if (aes->use_aesni && ((ret = SAVE_VECTOR_REGISTERS2()) != 0)) {
 #ifdef WC_AES_C_DYNAMIC_FALLBACK
-            xaes->aes.use_aesni = 0;
+            aes->use_aesni = 0;
             xaes->tweak.use_aesni = 0;
 #else
             return ret;
 #endif
         }
-        if (xaes->aes.use_aesni) {
+        if (aes->use_aesni) {
 #if defined(HAVE_INTEL_AVX1)
             if (IS_INTEL_AVX1(intel_flags)) {
-                AES_XTS_encrypt_avx1(in, out, sz, i, (const byte*)xaes->aes.key,
+                AES_XTS_encrypt_avx1(in, out, sz, i,
-                                     (const byte*)xaes->tweak.key, (int)xaes->aes.rounds);
+                                     (const byte*)aes->key,
+                                     (const byte*)xaes->tweak.key,
+                                     (int)aes->rounds);
                 ret = 0;
             }
             else
 #endif
             {
-                AES_XTS_encrypt_aesni(in, out, sz, i, (const byte*)xaes->aes.key,
+                AES_XTS_encrypt_aesni(in, out, sz, i,
-                                      (const byte*)xaes->tweak.key, (int)xaes->aes.rounds);
+                                      (const byte*)aes->key,
+                                      (const byte*)xaes->tweak.key,
+                                      (int)aes->rounds);
                 ret = 0;
             }
         }
@@ -12663,11 +12748,11 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
         }
 
 #ifdef WOLFSSL_AESNI
-        if (xaes->aes.use_aesni)
+        if (aes->use_aesni)
             RESTORE_VECTOR_REGISTERS();
 #ifdef WC_AES_C_DYNAMIC_FALLBACK
         else if (orig_use_aesni) {
-            xaes->aes.use_aesni = orig_use_aesni;
+            aes->use_aesni = orig_use_aesni;
             xaes->tweak.use_aesni = orig_use_aesni;
         }
 #endif
@@ -12677,7 +12762,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
     return ret;
 }
 
-/* Same process as encryption but Aes key is AES_DECRYPTION type.
+/* Same process as encryption but use aes_decrypt key.
  *
  * xaes  AES keys to use for block encrypt/decrypt
  * out   output buffer to hold plain text
@@ -12693,7 +12778,11 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz,
 {
     int ret = 0;
     word32 blocks = (sz / AES_BLOCK_SIZE);
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    Aes *aes = &xaes->aes_decrypt;
+#else
     Aes *aes = &xaes->aes;
+#endif
     Aes *tweak = &xaes->tweak;
     word32 j;
     byte carry = 0;
@@ -12821,11 +12910,23 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
         const byte* i, word32 iSz)
 {
     int ret;
+    Aes *aes;
 
     if (xaes == NULL || out == NULL || in == NULL) {
         return BAD_FUNC_ARG;
     }
 
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    aes = &xaes->aes_decrypt;
+#else
+    aes = &xaes->aes;
+#endif
+
+    if (aes->keylen == 0) {
+        WOLFSSL_MSG("wc_AesXtsDecrypt called with unset decryption key.");
+        return BAD_FUNC_ARG;
+    }
+
     if (iSz < AES_BLOCK_SIZE) {
         return BAD_FUNC_ARG;
     }
@@ -12838,25 +12939,33 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
     {
 #ifdef WOLFSSL_AESNI
 #ifdef WC_AES_C_DYNAMIC_FALLBACK
-        int orig_use_aesni = xaes->aes.use_aesni;
+        int orig_use_aesni = aes->use_aesni;
 #endif
 
-        if (xaes->aes.use_aesni && (SAVE_VECTOR_REGISTERS2() != 0)) {
+        if (aes->use_aesni && ((ret = SAVE_VECTOR_REGISTERS2() != 0))) {
-            xaes->aes.use_aesni = 0;
+#ifdef WC_AES_C_DYNAMIC_FALLBACK
+            aes->use_aesni = 0;
             xaes->tweak.use_aesni = 0;
+#else
+            return ret;
+#endif
         }
-        if (xaes->aes.use_aesni) {
+        if (aes->use_aesni) {
 #if defined(HAVE_INTEL_AVX1)
             if (IS_INTEL_AVX1(intel_flags)) {
-                AES_XTS_decrypt_avx1(in, out, sz, i, (const byte*)xaes->aes.key,
+                AES_XTS_decrypt_avx1(in, out, sz, i,
-                                     (const byte*)xaes->tweak.key, (int)xaes->aes.rounds);
+                                     (const byte*)aes->key,
+                                     (const byte*)xaes->tweak.key,
+                                     (int)aes->rounds);
                 ret = 0;
             }
             else
 #endif
             {
-                AES_XTS_decrypt_aesni(in, out, sz, i, (const byte*)xaes->aes.key,
+                AES_XTS_decrypt_aesni(in, out, sz, i,
-                                      (const byte*)xaes->tweak.key, (int)xaes->aes.rounds);
+                                      (const byte*)aes->key,
+                                      (const byte*)xaes->tweak.key,
+                                      (int)aes->rounds);
                 ret = 0;
             }
         }
@@ -12867,11 +12976,11 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
         }
 
 #ifdef WOLFSSL_AESNI
-        if (xaes->aes.use_aesni)
+        if (aes->use_aesni)
             RESTORE_VECTOR_REGISTERS();
 #ifdef WC_AES_C_DYNAMIC_FALLBACK
         else if (orig_use_aesni) {
-            xaes->aes.use_aesni = orig_use_aesni;
+            aes->use_aesni = orig_use_aesni;
             xaes->tweak.use_aesni = orig_use_aesni;
         }
 #endif

wolfcrypt/src/memory.c

@@ -1534,9 +1534,44 @@ WOLFSSL_LOCAL int SAVE_VECTOR_REGISTERS2_fuzzer(void) {
         return 0;
 }
 
-#endif
+#endif /* DEBUG_VECTOR_REGISTER_ACCESS_FUZZING */
 
-#endif
+#elif defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING)
+
+/* DEBUG_VECTOR_REGISTER_ACCESS is undefined but fuzzing requested --
+ * fuzz vector register access without the detailed debugging.
+ * this is useful for testing in the kernel module build, where glibc and
+ * thread-local storage are unavailable.
+ *
+ * note this is not a well-behaved PRNG, but is adequate for fuzzing purposes.
+ * the prn sequence is incompressible according to ent and xz, and does not
+ * cycle within 10M iterations with various seeds including zero, but the Chi
+ * square distribution is poor, and the unconditioned lsb bit balance is ~54%
+ * regardless of seed.
+ */
+
+WOLFSSL_LOCAL int SAVE_VECTOR_REGISTERS2_fuzzer(void) {
+    static unsigned long prn = WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED;
+    static int balance_bit = 0;
+    /* access to prn is racey, but it doesn't matter. */
+    unsigned long new_prn = prn ^ 0xba86943da66ee701ul; /* note this magic
+                                                         * random number is
+                                                         * bit-balanced.
+                                                         */
+    /* barrel-roll using the bottom 6 bits. */
+    if (new_prn & 0x3f)
+        new_prn = (new_prn << (new_prn & 0x3f)) |
+            (new_prn >> (0x40 - (new_prn & 0x3f)));
+    prn = new_prn;
+
+    balance_bit = !balance_bit;
+
+    return ((prn & 1) ^ balance_bit) ? IO_FAILED_E : 0;
+}
+
+#endif /* DEBUG_VECTOR_REGISTER_ACCESS ||
+        * DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
+        */
 
 #ifdef WOLFSSL_LINUXKM
     #include "../../linuxkm/linuxkm_memory.c"

wolfcrypt/test/test.c

@@ -9438,6 +9438,7 @@ static wc_test_ret_t aes_xts_128_test(void)
         0x77, 0x8a, 0xe8, 0xb4, 0x3c, 0xb9, 0x8d, 0x5a
     };
 
+#ifndef HAVE_FIPS /* FIPS requires different keys for main and tweak. */
     WOLFSSL_SMALL_STACK_STATIC unsigned char k3[] = {
         0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
         0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
@@ -9462,6 +9463,7 @@ static wc_test_ret_t aes_xts_128_test(void)
         0xA0, 0x85, 0xD2, 0x69, 0x6E, 0x87, 0x0A, 0xBF,
         0xB5, 0x5A, 0xDD, 0xCB, 0x80, 0xE0, 0xFC, 0xCD
     };
+#endif /* HAVE_FIPS */
 
 #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
     if ((aes = (XtsAes *)XMALLOC(sizeof *aes, HEAP_HINT, DYNAMIC_TYPE_AES)) == NULL)
@@ -9569,7 +9571,11 @@ static wc_test_ret_t aes_xts_128_test(void)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     ret = wc_AesXtsDecrypt(aes, buf, cipher, sizeof(pp), i1, sizeof(i1));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -9581,7 +9587,11 @@ static wc_test_ret_t aes_xts_128_test(void)
     XMEMSET(buf, 0, sizeof(buf));
     ret = wc_AesXtsDecrypt(aes, buf, cipher, sizeof(pp), i1, sizeof(i1));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0);
     if (ret != 0)
@@ -9594,7 +9604,11 @@ static wc_test_ret_t aes_xts_128_test(void)
     XMEMSET(buf, 0, sizeof(buf));
     ret = wc_AesXtsDecrypt(aes, buf, c1, sizeof(c1), i1, sizeof(i1));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -9606,7 +9620,11 @@ static wc_test_ret_t aes_xts_128_test(void)
     XMEMSET(buf, 0, sizeof(buf));
     ret = wc_AesXtsDecrypt(aes, buf, c1, sizeof(c1), i1, sizeof(i1));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0);
     if (ret != 0)
@@ -9619,7 +9637,11 @@ static wc_test_ret_t aes_xts_128_test(void)
     XMEMSET(buf, 0, sizeof(buf));
     ret = wc_AesXtsDecrypt(aes, buf, c2, sizeof(c2), i2, sizeof(i2));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -9633,13 +9655,19 @@ static wc_test_ret_t aes_xts_128_test(void)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     ret = wc_AesXtsDecrypt(aes, buf, c2, sizeof(c2), i2, sizeof(i2));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     if (XMEMCMP(p2, buf, sizeof(p2)))
         ERROR_OUT(WC_TEST_RET_ENC_NC, out);
 
+#ifndef HAVE_FIPS
+
     /* Test ciphertext stealing in-place. */
     XMEMCPY(buf, p3, sizeof(p3));
     ret = wc_AesXtsSetKeyNoInit(aes, k3, sizeof(k3), AES_ENCRYPTION);
@@ -9660,13 +9688,19 @@ static wc_test_ret_t aes_xts_128_test(void)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     ret = wc_AesXtsDecrypt(aes, buf, buf, sizeof(c3), i3, sizeof(i3));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     if (XMEMCMP(p3, buf, sizeof(p3)))
         ERROR_OUT(WC_TEST_RET_ENC_NC, out);
 
+#endif /* !HAVE_FIPS */
+
 #if !defined(BENCH_EMBEDDED) && !defined(HAVE_CAVIUM) && \
     !defined(WOLFSSL_AFALG)
     {
@@ -9705,8 +9739,13 @@ static wc_test_ret_t aes_xts_128_test(void)
             ret = wc_AesXtsDecrypt(aes, large_input, large_input, j, i1,
                 sizeof(i1));
         #if defined(WOLFSSL_ASYNC_CRYPT)
+            #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+            ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev,
+                               WC_ASYNC_FLAG_NONE);
+            #else
             ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
             #endif
+        #endif
             if (ret != 0)
                 ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
             for (i = 0; i < j; i++) {
@@ -9886,7 +9925,11 @@ static wc_test_ret_t aes_xts_256_test(void)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     ret = wc_AesXtsDecrypt(aes, buf, cipher, sizeof(pp), i1, sizeof(i1));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -9897,7 +9940,11 @@ static wc_test_ret_t aes_xts_256_test(void)
     XMEMSET(buf, 0, sizeof(buf));
     ret = wc_AesXtsDecrypt(aes, buf, c1, sizeof(c1), i1, sizeof(i1));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -9910,7 +9957,11 @@ static wc_test_ret_t aes_xts_256_test(void)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     ret = wc_AesXtsDecrypt(aes, buf, c2, sizeof(c2), i2, sizeof(i2));
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -10141,7 +10192,11 @@ static wc_test_ret_t aes_xts_sector_test(void)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     ret = wc_AesXtsDecryptSector(aes, buf, c1, sizeof(c1), s1);
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -10169,7 +10224,11 @@ static wc_test_ret_t aes_xts_sector_test(void)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     ret = wc_AesXtsDecryptSector(aes, buf, c2, sizeof(c2), s2);
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -10201,7 +10260,11 @@ static wc_test_ret_t aes_xts_sector_test(void)
     ret = wc_AesXtsDecryptConsecutiveSectors(aes, data, c3,
             sizeof(c3), s3, sectorSz);
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret != 0)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
@@ -10298,14 +10361,22 @@ static wc_test_ret_t aes_xts_args_test(void)
         ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out);
     ret = wc_AesXtsDecryptSector(NULL, buf, c1, sizeof(c1), s1);
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret == 0)
         ERROR_OUT(WC_TEST_RET_ENC_NC, out);
 
     ret = wc_AesXtsDecryptSector(aes, NULL, c1, sizeof(c1), s1);
 #if defined(WOLFSSL_ASYNC_CRYPT)
+    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, WC_ASYNC_FLAG_NONE);
+    #else
     ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE);
+    #endif
 #endif
     if (ret == 0)
         ERROR_OUT(WC_TEST_RET_ENC_NC, out);
@@ -30070,8 +30141,8 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ecc_test(void)
 #endif /* HAVE_ECC160 */
 #if (defined(HAVE_ECC192) || defined(HAVE_ALL_CURVES)) && ECC_MIN_KEY_SZ <= 192
     ret = ecc_test_curve(&rng, 24, ECC_CURVE_DEF);
-        printf("keySize=24, Default\n");
     if (ret < 0) {
+        printf("keySize=24, Default\n");
         goto done;
     }
 #endif /* HAVE_ECC192 */

wolfssl/wolfcrypt/aes.h

@@ -179,6 +179,9 @@ enum {
     AES_ENC_TYPE   = WC_CIPHER_AES,   /* cipher unique type */
     AES_ENCRYPTION = 0,
     AES_DECRYPTION = 1,
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    AES_ENCRYPTION_AND_DECRYPTION = 2,
+#endif
 
     AES_BLOCK_SIZE      = 16,
 
@@ -399,6 +402,9 @@ struct Aes {
 #ifdef WOLFSSL_AES_XTS
 typedef struct XtsAes {
     Aes aes;
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    Aes aes_decrypt;
+#endif
     Aes tweak;
 } XtsAes;
 #endif

wolfssl/wolfcrypt/memory.h

@@ -267,6 +267,13 @@ WOLFSSL_LOCAL int wc_debug_CipherLifecycleFree(void **CipherLifecycleTag,
         ((void)(CipherLifecycleTag), (void)(heap), (void)(abort_p), 0)
 #endif
 
+#ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
+    WOLFSSL_LOCAL int SAVE_VECTOR_REGISTERS2_fuzzer(void);
+    #ifndef WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED
+        #define WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED 0
+    #endif
+#endif
+
 #ifdef DEBUG_VECTOR_REGISTER_ACCESS
     WOLFSSL_API extern THREAD_LS_T int wc_svr_count;
     WOLFSSL_API extern THREAD_LS_T const char *wc_svr_last_file;
@@ -320,11 +327,6 @@ WOLFSSL_LOCAL int wc_debug_CipherLifecycleFree(void **CipherLifecycleTag,
     } while (0)
 
 #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
-    #ifndef WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED
-        #define WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED 0
-    #endif
-        WOLFSSL_LOCAL int SAVE_VECTOR_REGISTERS2_fuzzer(void);
-
     #define SAVE_VECTOR_REGISTERS2(...) ({                          \
         int _svr2_val = SAVE_VECTOR_REGISTERS2_fuzzer();            \
         if (_svr2_val == 0) {                                       \

wolfssl/wolfcrypt/settings.h

@@ -2771,6 +2771,7 @@ extern void uITRON4_free(void *p) ;
     #ifndef WOLFSSL_TEST_SUBROUTINE
         #define WOLFSSL_TEST_SUBROUTINE static
     #endif
+    #undef HAVE_PTHREAD
     #undef HAVE_STRINGS_H
     #undef HAVE_ERRNO_H
     #undef HAVE_THREAD_LS