wolfcrypt/src/aes.c and wolfssl/wolfcrypt/aes.h: add support for WC_FLAG_DONT_USE_AESNI in wc_AesSetKeyLocal(); add support for USE_INTEL_SPEEDUP_FOR_AES.

linuxkm/lkcapi_glue.c: finish implementation of WC_LINUXKM_C_FALLBACK_IN_SHIMS and add TEST_WC_LINUXKM_C_FALLBACK_IN_SHIMS.

use "WC_C_DYNAMIC_FALLBACK" consistently (remove/replace uses of "WC_AES_C_DYNAMIC_FALLBACK").

.github/workflows/intelasm-c-fallback.yml

@@ -18,7 +18,7 @@ jobs:
       matrix:
         config: [
           # Add new configs here
-          '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy CPPFLAGS="-DNO_WOLFSSL_CIPHER_SUITE_TEST -DWC_AES_C_DYNAMIC_FALLBACK -DWC_C_DYNAMIC_FALLBACK -DDEBUG_VECTOR_REGISTER_ACCESS -DDEBUG_VECTOR_REGISTER_ACCESS_FUZZING -DWC_DEBUG_CIPHER_LIFECYCLE"'
+          '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy CPPFLAGS="-DNO_WOLFSSL_CIPHER_SUITE_TEST -DWC_C_DYNAMIC_FALLBACK -DDEBUG_VECTOR_REGISTER_ACCESS -DDEBUG_VECTOR_REGISTER_ACCESS_FUZZING -DWC_DEBUG_CIPHER_LIFECYCLE"'
         ]
     name: make check
     if: github.repository_owner == 'wolfssl'

linuxkm/lkcapi_glue.c

@@ -221,7 +221,10 @@ static int disable_setkey_warnings = 0;
 
 #include <wolfssl/wolfcrypt/aes.h>
 
-#if !defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && defined(WOLFSSL_AESNI) && (!defined(WC_AES_C_DYNAMIC_FALLBACK) || (defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0)))
+#if defined(WOLFSSL_AESNI) &&                            \
+    (!defined(WC_C_DYNAMIC_FALLBACK) ||                  \
+     (defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0))) && \
+    !defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS)
     #define WC_LINUXKM_C_FALLBACK_IN_SHIMS
 #elif !defined(WOLFSSL_AESNI)
     #undef WC_LINUXKM_C_FALLBACK_IN_SHIMS
@@ -338,8 +341,6 @@ static int km_AesInitCommon(
         goto out;
     }
 
-    ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI;
-
     if (! need_decryption) {
         ctx->aes_decrypt_C = NULL;
     }
@@ -362,8 +363,6 @@ static int km_AesInitCommon(
             err = -EINVAL;
             goto out;
         }
-
-        ctx->aes_decrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI;
     }
 
 #endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */
@@ -380,14 +379,30 @@ static int km_AesGet(struct km_AesCtx *ctx, int decrypt_p, int copy_p, Aes **aes
     Aes *ret;
 
 #ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
-    if (! CAN_SAVE_VECTOR_REGISTERS()) {
+    /* First, check if AESNI was disabled in the main SetKey for the requested
+     * direction.  If so, use it (the fallback schedule won't even be inited).
+     */
+    if (((! decrypt_p) || (! ctx->aes_decrypt)) && (! ctx->aes_encrypt->use_aesni))
+        ret = ctx->aes_encrypt;
+    else if (decrypt_p && ctx->aes_decrypt && (! ctx->aes_decrypt->use_aesni))
+        ret = ctx->aes_decrypt;
+    else if (
+#ifdef TEST_WC_LINUXKM_C_FALLBACK_IN_SHIMS
+             1
+#else
+             ! CAN_SAVE_VECTOR_REGISTERS()
+#endif
+        )
+    {
         if (decrypt_p && ctx->aes_decrypt_C)
             ret = ctx->aes_decrypt_C;
         else
             ret = ctx->aes_encrypt_C;
+        if (ret->use_aesni)
+            return -EINVAL;
     }
     else
-#endif
+#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */
     {
         if (decrypt_p && ctx->aes_decrypt)
             ret = ctx->aes_decrypt;
@@ -483,6 +498,43 @@ static int km_AesSetKeyCommon(struct km_AesCtx * ctx, const u8 *in_key,
         }
     }
 
+#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
+
+    if (ctx->aes_encrypt->use_aesni) {
+        ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI;
+
+        err = wc_AesSetKey(ctx->aes_encrypt_C, in_key, key_len, NULL, AES_ENCRYPTION);
+
+        if (unlikely(err)) {
+            if (! disable_setkey_warnings)
+                pr_err("%s: wc_AesSetKey for encryption key failed: %d\n", name, err);
+            return -EINVAL;
+        }
+
+        if (ctx->aes_encrypt_C->use_aesni)
+            pr_err("%s: after wc_AesSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", name);
+
+    }
+
+    if (ctx->aes_decrypt_C && ctx->aes_decrypt->use_aesni) {
+        ctx->aes_decrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI;
+
+        err = wc_AesSetKey(ctx->aes_decrypt_C, in_key, key_len, NULL,
+                           AES_DECRYPTION);
+
+        if (unlikely(err)) {
+            if (! disable_setkey_warnings)
+                pr_err("%s: wc_AesSetKey for decryption key failed: %d\n",
+                       name, err);
+            return -EINVAL;
+        }
+
+        if (ctx->aes_decrypt_C->use_aesni)
+            pr_err("%s: after wc_AesSetKey, ctx->aes_decrypt_C has AES-NI asserted.\n", name);
+    }
+
+#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */
+
     return 0;
 }
 
@@ -850,6 +902,24 @@ static int km_AesGcmSetKey(struct crypto_aead *tfm, const u8 *in_key,
         return -EINVAL;
     }
 
+#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
+    if (ctx->aes_encrypt->use_aesni) {
+        ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI;
+
+        err = wc_AesGcmSetKey(ctx->aes_encrypt_C, in_key, key_len);
+
+        if (unlikely(err)) {
+            if (! disable_setkey_warnings)
+                pr_err("%s: wc_AesGcmSetKey failed: %d\n",
+                       crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
+            return -EINVAL;
+        }
+
+        if (ctx->aes_encrypt_C->use_aesni)
+            pr_err("%s: after wc_AesGcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESGCM_DRIVER);
+    }
+#endif
+
     return 0;
 }
 
@@ -877,6 +947,24 @@ static int km_AesGcmSetKey_Rfc4106(struct crypto_aead *tfm, const u8 *in_key,
         return -EINVAL;
     }
 
+#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
+    if (ctx->aes_encrypt->use_aesni) {
+        ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI;
+
+        err = wc_AesGcmSetKey(ctx->aes_encrypt_C, in_key, key_len);
+
+        if (unlikely(err)) {
+            if (! disable_setkey_warnings)
+                pr_err("%s: wc_AesGcmSetKey failed: %d\n",
+                       crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
+            return -EINVAL;
+        }
+
+        if (ctx->aes_encrypt_C->use_aesni)
+            pr_err("%s: after wc_AesGcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESGCM_DRIVER);
+    }
+#endif
+
     return 0;
 }
 
@@ -1375,8 +1463,8 @@ static int gcmAesAead_rfc4106_loaded = 0;
     #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream).
 #endif
 
-#ifndef WC_AES_C_DYNAMIC_FALLBACK
+#ifndef WC_C_DYNAMIC_FALLBACK
-    #error LKCAPI registration of AES-XTS requires WC_AES_C_DYNAMIC_FALLBACK.
+    #error LKCAPI registration of AES-XTS requires WC_C_DYNAMIC_FALLBACK.
 #endif
 
 struct km_AesXtsCtx {

wolfcrypt/src/aes.c

@@ -4575,12 +4575,21 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir)
 #endif /* WC_C_DYNAMIC_FALLBACK */
 
     #ifdef WOLFSSL_AESNI
-        aes->use_aesni = 0;
+
-        if (checkedAESNI == 0) {
+#if defined(WC_FLAG_DONT_USE_AESNI)
-            haveAESNI  = Check_CPU_support_AES();
+        if (aes->use_aesni == WC_FLAG_DONT_USE_AESNI) {
-            checkedAESNI = 1;
+            aes->use_aesni = 0;
         }
-        if (haveAESNI) {
+        else
+#endif
+        {
+            if (checkedAESNI == 0) {
+                haveAESNI  = Check_CPU_support_AES();
+                checkedAESNI = 1;
+            }
+            aes->use_aesni = haveAESNI;
+        }
+        if (aes->use_aesni) {
             #ifdef WOLFSSL_LINUXKM
             /* runtime alignment check */
             if ((wc_ptr_t)&aes->key & (wc_ptr_t)0xf) {
@@ -12993,6 +13002,10 @@ int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz,
 
 #ifdef WOLFSSL_AESNI
 
+#if defined(USE_INTEL_SPEEDUP_FOR_AES) && !defined(USE_INTEL_SPEEDUP)
+    #define USE_INTEL_SPEEDUP
+#endif
+
 #if defined(USE_INTEL_SPEEDUP)
     #define HAVE_INTEL_AVX1
     #define HAVE_INTEL_AVX2

wolfssl/wolfcrypt/aes.h

@@ -303,6 +303,13 @@ struct Aes {
 #endif
 #ifdef WOLFSSL_AESNI
     byte use_aesni;
+    #if defined(WOLFSSL_LINUXKM) || defined(WC_WANT_FLAG_DONT_USE_AESNI)
+        /* Note, we can't support WC_FLAG_DONT_USE_AESNI by default because we
+         * need to support legacy applications that call wc_AesSetKey() on
+         * uninited struct Aes.
+         */
+        #define WC_FLAG_DONT_USE_AESNI 2
+    #endif
 #endif /* WOLFSSL_AESNI */
 #if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
     !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)