From 9d931d45de2cd85cb2260876a338490b9b4c66c9 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Thu, 27 Mar 2025 21:36:15 -0500 Subject: [PATCH 1/6] LKCAPI checkpoint (all AES except CCM working). --- configure.ac | 26 +- linuxkm/lkcapi_glue.c | 1688 ++++++++++++++++++++++++++++++++++------- 2 files changed, 1427 insertions(+), 287 deletions(-) diff --git a/configure.ac b/configure.ac index 9fd819c42..7206b7218 100644 --- a/configure.ac +++ b/configure.ac @@ -498,7 +498,8 @@ AS_CASE([$ENABLED_FIPS], [v5-dev],[ FIPS_VERSION="v5-dev" HAVE_FIPS_VERSION_MAJOR=5 - HAVE_FIPS_VERSION_MINOR=3 + HAVE_FIPS_VERSION_MINOR=2 + HAVE_FIPS_VERSION_PATCH=1 ENABLED_FIPS="yes" # for dev, DEF_SP_MATH and DEF_FAST_MATH follow non-FIPS defaults (currently sp-math-all) ], @@ -1243,7 +1244,6 @@ then test "$enable_aesgcm" = "" && enable_aesgcm=yes test "$enable_aesccm" = "" && enable_aesccm=yes test "$enable_aesctr" = "" && enable_aesctr=yes - test "$enable_aeseax" = "" && enable_aeseax=yes test "$enable_aesofb" = "" && enable_aesofb=yes test "$enable_aescfb" = "" && enable_aescfb=yes test "$enable_aescbc_length_checks" = "" && enable_aescbc_length_checks=yes @@ -1318,6 +1318,8 @@ then test "$enable_ed25519_stream" = "" && test "$enable_ed25519" != "no" && enable_ed25519_stream=yes test "$enable_ed448" = "" && enable_ed448=yes test "$enable_ed448_stream" = "" && test "$enable_ed448" != "no" && enable_ed448_stream=yes + test "$enable_aessiv" = "" && enable_aessiv=yes + test "$enable_aeseax" = "" && enable_aeseax=yes if test "$ENABLED_LINUXKM_DEFAULTS" != "yes" then @@ -1331,7 +1333,6 @@ then test "$enable_aesgcm_stream" = "" && test "$enable_aesgcm" = "yes" && enable_aesgcm_stream=yes test "$enable_aesxts" = "" && enable_aesxts=yes test "$enable_aesxts_stream" = "" && test "$enable_aesxts" = "yes" && enable_aesxts_stream=yes - test "$enable_aessiv" = "" && enable_aessiv=yes test "$enable_shake128" = "" && enable_shake128=yes test "$enable_shake256" = "" && enable_shake256=yes test "$enable_compkey" = "" && test "$ENABLED_LINUXKM_DEFAULTS" != "yes" && enable_compkey=yes @@ -9320,7 +9321,7 @@ if test "$ENABLED_LINUXKM_LKCAPI_REGISTER" != "none" then AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER" - if test "$ENABLED_AESGCM" != "no" && test "$ENABLED_AESGCM_STREAM" = "no" && test "$ENABLED_FIPS" = "no"; then + if test "$ENABLED_AESGCM" != "no" && test "$ENABLED_AESGCM_STREAM" = "no" && test "$enable_aesgcm_stream" != "no" && (test "$ENABLED_FIPS" = "no" || test $HAVE_FIPS_VERSION -ge 6); then ENABLED_AESGCM_STREAM=yes fi @@ -9338,11 +9339,26 @@ then 'cfb(aes)') test "$ENABLED_AESCFB" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-CFB implementation not enabled.]) AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESCFB" ;; 'gcm(aes)') test "$ENABLED_AESGCM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-GCM implementation not enabled.]) - test "$ENABLED_AESGCM_STREAM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: --enable-aesgcm-stream is required for LKCAPI.]) AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESGCM" ;; + 'rfc4106(gcm(aes))') test "$ENABLED_AESGCM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-GCM implementation not enabled.]) + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106" ;; 'xts(aes)') test "$ENABLED_AESXTS" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-XTS implementation not enabled.]) test "$ENABLED_AESXTS_STREAM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: --enable-aesxts-stream is required for LKCAPI.]) AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESXTS" ;; + 'ctr(aes)') test "$ENABLED_AESCTR" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-CTR implementation not enabled.]) + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESCTR" ;; + 'ofb(aes)') test "$ENABLED_AESOFB" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-OFB implementation not enabled.]) + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESOFB" ;; + 'ecb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESECB -DHAVE_AES_ECB" ;; + '-cbc(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESCBC" ;; + '-cfb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESCFB" ;; + '-gcm(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESGCM" ;; + '-rfc4106(gcm(aes))') + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESGCM_RFC4106" ;; + '-xts(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESXTS" ;; + '-ctr(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESCTR" ;; + '-ofb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESOFB" ;; + '-ecb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESECB" ;; *) AC_MSG_ERROR([Unsupported LKCAPI algorithm "$lkcapi_alg".]) ;; esac done diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index a98e0ee17..ef567c14b 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -30,8 +30,11 @@ * results FIPS vs non-FIPS. */ #if defined(CONFIG_CRYPTO_MANAGER) && \ - (defined(CONFIG_CRYPTO_FIPS) != defined(HAVE_FIPS)) -#error CONFIG_CRYPTO_MANAGER requires that CONFIG_CRYPTO_FIPS match HAVE_FIPS. + !defined(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) + #if defined(CONFIG_CRYPTO_FIPS) != defined(HAVE_FIPS) + #error CONFIG_CRYPTO_MANAGER requires that CONFIG_CRYPTO_FIPS match HAVE_FIPS. + #endif + #include #endif #ifndef WOLFSSL_LINUXKM_LKCAPI_PRIORITY @@ -56,7 +59,12 @@ static int disable_setkey_warnings = 0; #define WOLFKM_AESCBC_NAME "cbc(aes)" #define WOLFKM_AESCFB_NAME "cfb(aes)" #define WOLFKM_AESGCM_NAME "gcm(aes)" +#define WOLFKM_AESGCM_RFC4106_NAME "rfc4106(gcm(aes))" #define WOLFKM_AESXTS_NAME "xts(aes)" +#define WOLFKM_AESCTR_NAME "ctr(aes)" +#define WOLFKM_AESOFB_NAME "ofb(aes)" +#define WOLFKM_AESECB_NAME "ecb(aes)" +#define WOLFKM_AESCCM_NAME "ccm(aes)" #ifdef WOLFSSL_AESNI #define WOLFKM_DRIVER_ISA_EXT "-aesni" @@ -84,7 +92,12 @@ static int disable_setkey_warnings = 0; #define WOLFKM_AESCBC_DRIVER ("cbc-aes" WOLFKM_DRIVER_SUFFIX) #define WOLFKM_AESCFB_DRIVER ("cfb-aes" WOLFKM_DRIVER_SUFFIX) #define WOLFKM_AESGCM_DRIVER ("gcm-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESGCM_RFC4106_DRIVER ("rfc4106-gcm-aes" WOLFKM_DRIVER_SUFFIX) #define WOLFKM_AESXTS_DRIVER ("xts-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESCTR_DRIVER ("ctr-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESOFB_DRIVER ("ofb-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESECB_DRIVER ("ecb-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESCCM_DRIVER ("ccm-aes" WOLFKM_DRIVER_SUFFIX) #ifdef WOLFSSL_DEBUG_TRACE_ERROR_CODES enum linux_errcodes { @@ -102,25 +115,105 @@ static int disable_setkey_warnings = 0; #define EBADMSG WC_ERR_TRACE(my_EBADMSG) #endif -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) -static int linuxkm_test_aescbc(void); +#ifdef HAVE_AES_CBC + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCBC)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESCBC) + #define LINUXKM_LKCAPI_REGISTER_AESCBC + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESCBC #endif -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) -static int linuxkm_test_aescfb(void); +#ifdef WOLFSSL_AES_CFB + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCFB)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESCFB) + #define LINUXKM_LKCAPI_REGISTER_AESCFB + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESCFB #endif -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) -static int linuxkm_test_aesgcm(void); +#ifdef HAVE_AESGCM + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESGCM)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESGCM) + #define LINUXKM_LKCAPI_REGISTER_AESGCM + #endif + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESGCM_RFC4106)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) + #define LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESGCM + #undef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 #endif -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) -static int linuxkm_test_aesxts(void); +#ifdef WOLFSSL_AES_XTS + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESXTS)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESXTS) + #define LINUXKM_LKCAPI_REGISTER_AESXTS + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESXTS +#endif +#ifdef WOLFSSL_AES_COUNTER + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCTR)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESCTR) + #define LINUXKM_LKCAPI_REGISTER_AESCTR + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESCTR +#endif +#ifdef WOLFSSL_AES_OFB + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESOFB)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESOFB) + #define LINUXKM_LKCAPI_REGISTER_AESOFB + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESOFB +#endif +#ifdef HAVE_AES_ECB + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESECB)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESECB) + #define LINUXKM_LKCAPI_REGISTER_AESECB + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESECB +#endif + +#ifdef notyet +#ifdef HAVE_AESCCM + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCCM)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESCCM) + #define LINUXKM_LKCAPI_REGISTER_AESCCM + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESCCM +#endif +#endif /* notyet */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC + static int linuxkm_test_aescbc(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB + static int linuxkm_test_aescfb(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM + static int linuxkm_test_aesgcm(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + static int linuxkm_test_aesgcm_rfc4106(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS + static int linuxkm_test_aesxts(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + static int linuxkm_test_aesctr(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + static int linuxkm_test_aesofb(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + static int linuxkm_test_aesecb(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM + static int linuxkm_test_aesccm(void); #endif /* km_AesX(): wrappers to wolfcrypt wc_AesX functions and @@ -133,12 +226,26 @@ struct km_AesCtx { * alignment, needed for AESNI. */ Aes *aes_decrypt; /* same. */ +#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309) + union { +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + byte rfc4106_nonce[4]; +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 + byte rfc4309_nonce[3]; +#endif + }; +#endif }; -#if defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ +#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM) + defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESECB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309) static void km_AesExitCommon(struct km_AesCtx * ctx); @@ -207,9 +314,12 @@ static void km_AesExitCommon(struct km_AesCtx * ctx) } } -#if defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB) +#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESECB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309) static int km_AesSetKeyCommon(struct km_AesCtx * ctx, const u8 *in_key, unsigned int key_len, const char * name) @@ -245,18 +355,22 @@ static void km_AesExit(struct crypto_skcipher *tfm) km_AesExitCommon(ctx); } -#endif /* LINUXKM_LKCAPI_REGISTER_ALL || - * LINUXKM_LKCAPI_REGISTER_AESCBC || - * LINUXKM_LKCAPI_REGISTER_AESCFB +#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC || + * LINUXKM_LKCAPI_REGISTER_AESCFB || + * LINUXKM_LKCAPI_REGISTER_AESCTR || + * LINUXKM_LKCAPI_REGISTER_AESOFB || + * LINUXKM_LKCAPI_REGISTER_AESECB || + * LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */ -#endif /* LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCBC || - * LINUXKM_LKCAPI_REGISTER_AESCFB || LINUXKM_LKCAPI_REGISTER_AESGCM +#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC || + * LINUXKM_LKCAPI_REGISTER_AESCFB || LINUXKM_LKCAPI_REGISTER_AESGCM || + * LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 || + * LINUXKM_LKCAPI_REGISTER_AESCTR || LINUXKM_LKCAPI_REGISTER_AESOFB || + * LINUXKM_LKCAPI_REGISTER_AESECB || LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */ -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC static int km_AesCbcInit(struct crypto_skcipher *tfm) { @@ -406,13 +520,9 @@ static struct skcipher_alg cbcAesAlg = { }; static int cbcAesAlg_loaded = 0; -#endif /* HAVE_AES_CBC && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCBC) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */ -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB static int km_AesCfbInit(struct crypto_skcipher *tfm) { @@ -577,17 +687,10 @@ static struct skcipher_alg cfbAesAlg = { }; static int cfbAesAlg_loaded = 0; -#endif /* WOLFSSL_AES_CFB && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCBC) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */ -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) - -#ifndef WOLFSSL_AESGCM_STREAM - #error LKCAPI registration of AES-GCM requires WOLFSSL_AESGCM_STREAM (--enable-aesgcm-stream). -#endif +#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) static int km_AesGcmInit(struct crypto_aead * tfm) { @@ -601,6 +704,8 @@ static void km_AesGcmExit(struct crypto_aead * tfm) km_AesExitCommon(ctx); } +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM + static int km_AesGcmSetKey(struct crypto_aead *tfm, const u8 *in_key, unsigned int key_len) { @@ -619,6 +724,37 @@ static int km_AesGcmSetKey(struct crypto_aead *tfm, const u8 *in_key, return 0; } +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + +static int km_AesGcmSetKey_Rfc4106(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + int err; + struct km_AesCtx * ctx = crypto_aead_ctx(tfm); + + if (key_len < 4) + return -EINVAL; + key_len -= 4; + memcpy(ctx->rfc4106_nonce, in_key + key_len, 4); + + err = wc_AesGcmSetKey(ctx->aes_encrypt, in_key, key_len); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesGcmSetKey failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return -EINVAL; + } + + return 0; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM + static int km_AesGcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize) { (void)tfm; @@ -641,6 +777,30 @@ static int km_AesGcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize) return -EINVAL; } +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + +static int km_AesGcmSetAuthsize_Rfc4106(struct crypto_aead *tfm, unsigned int authsize) +{ + (void)tfm; + + switch (authsize) { + case 8: + case 12: + case 16: + return 0; + } + +#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG + pr_err("%s: invalid authsize: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), authsize); +#endif + return -EINVAL; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + /* * aead ciphers receive data in scatterlists in following order: * encrypt @@ -651,7 +811,9 @@ static int km_AesGcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize) * req->dst: aad||plaintext, return 0 or -EBADMSG */ -static int km_AesGcmEncrypt(struct aead_request *req) +#ifdef WOLFSSL_AESGCM_STREAM + +static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) { struct crypto_aead * tfm = NULL; struct km_AesCtx * ctx = NULL; @@ -659,6 +821,7 @@ static int km_AesGcmEncrypt(struct aead_request *req) struct scatter_walk assocSgWalk; u8 authTag[WC_AES_BLOCK_SIZE]; int err; + unsigned int assoclen = req->assoclen; u8 * assoc = NULL; u8 * assocmem = NULL; Aes *aes_copy; @@ -666,142 +829,22 @@ static int km_AesGcmEncrypt(struct aead_request *req) tfm = crypto_aead_reqtfm(req); ctx = crypto_aead_ctx(tfm); - err = skcipher_walk_aead_encrypt(&walk, req, false); - if (unlikely(err)) { - pr_err("%s: skcipher_walk_aead_encrypt failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - return -EINVAL; - } - - /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and - * aes->streamData. - */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - aes_copy->streamData = NULL; -#endif - - err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv, - GCM_NONCE_MID_SZ); - if (unlikely(err)) { - pr_err("%s: wc_AesGcmInit failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - if (req->src->length >= req->assoclen && req->src->length) { - scatterwalk_start(&assocSgWalk, req->src); - assoc = scatterwalk_map(&assocSgWalk); - if (unlikely(IS_ERR(assoc))) { - pr_err("%s: scatterwalk_map failed: %ld\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), - PTR_ERR(assoc)); - goto out; - } + if (decrypt_p) { + /* Copy out original auth tag from req->src. */ + scatterwalk_map_and_copy(authTag, req->src, + req->assoclen + req->cryptlen - tfm->authsize, + tfm->authsize, 0); + err = skcipher_walk_aead_decrypt(&walk, req, false); } else { - /* assoc can be any length, so if it's noncontiguous, we have to copy it - * to a contiguous heap allocation. - */ - assocmem = malloc(req->assoclen); - if (unlikely(assocmem == NULL)) { - err = -ENOMEM; - goto out; - } - assoc = assocmem; - scatterwalk_map_and_copy(assoc, req->src, 0, req->assoclen, 0); + err = skcipher_walk_aead_encrypt(&walk, req, false); } - err = wc_AesGcmEncryptUpdate(aes_copy, NULL, NULL, 0, - assoc, req->assoclen); - - if (assocmem) - free(assocmem); - else - scatterwalk_unmap(assoc); - if (unlikely(err)) { - pr_err("%s: wc_AesGcmEncryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - while (walk.nbytes) { - err = wc_AesGcmEncryptUpdate( - aes_copy, - walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes, - NULL, 0); - - if (unlikely(err)) { - pr_err("%s: wc_AesGcmEncryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - err = skcipher_walk_done(&walk, 0); - - if (unlikely(err)) { - pr_err("%s: skcipher_walk_done failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - goto out; - } - } - - err = wc_AesGcmEncryptFinal(aes_copy, authTag, tfm->authsize); - if (unlikely(err)) { - pr_err("%s: wc_AesGcmEncryptFinal failed with return code %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - /* Now copy the auth tag into request scatterlist. */ - scatterwalk_map_and_copy(authTag, req->dst, - req->assoclen + req->cryptlen, - tfm->authsize, 1); - -out: - -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - free(aes_copy->streamData); -#endif - free(aes_copy); - - return err; -} - -static int km_AesGcmDecrypt(struct aead_request *req) -{ - struct crypto_aead * tfm = NULL; - struct km_AesCtx * ctx = NULL; - struct skcipher_walk walk; - struct scatter_walk assocSgWalk; - u8 origAuthTag[WC_AES_BLOCK_SIZE]; - int err; - u8 * assoc = NULL; - u8 * assocmem = NULL; - Aes *aes_copy; - - tfm = crypto_aead_reqtfm(req); - ctx = crypto_aead_ctx(tfm); - - /* Copy out original auth tag from req->src. */ - scatterwalk_map_and_copy(origAuthTag, req->src, - req->assoclen + req->cryptlen - tfm->authsize, - tfm->authsize, 0); - - err = skcipher_walk_aead_decrypt(&walk, req, false); - if (unlikely(err)) { - pr_err("%s: skcipher_walk_aead_decrypt failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + pr_err("%s: %s failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + decrypt_p ? "skcipher_walk_aead_decrypt" : "skcipher_walk_aead_encrypt", + err); return err; } @@ -819,8 +862,27 @@ static int km_AesGcmDecrypt(struct aead_request *req) aes_copy->streamData = NULL; #endif - err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv, - GCM_NONCE_MID_SZ); +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + if (rfc4106_p) { + byte rfc4106_iv[12]; + + if (unlikely(assoclen != 16 && assoclen != 20)) + return -EINVAL; + assoclen -= 8; + + memcpy(rfc4106_iv, ctx->rfc4106_nonce, 4); + memcpy(rfc4106_iv + 4, walk.iv, 8); + err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, rfc4106_iv, + GCM_NONCE_MID_SZ); + } + else +#else + (void)rfc4106_p; +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + { + err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv, + GCM_NONCE_MID_SZ); + } if (unlikely(err)) { pr_err("%s: wc_AesGcmInit failed: %d\n", crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); @@ -828,7 +890,7 @@ static int km_AesGcmDecrypt(struct aead_request *req) goto out; } - if (req->src->length >= req->assoclen && req->src->length) { + if (req->src->length >= assoclen && req->src->length) { scatterwalk_start(&assocSgWalk, req->src); assoc = scatterwalk_map(&assocSgWalk); if (unlikely(IS_ERR(assoc))) { @@ -842,17 +904,23 @@ static int km_AesGcmDecrypt(struct aead_request *req) /* assoc can be any length, so if it's noncontiguous, we have to copy it * to a contiguous heap allocation. */ - assocmem = malloc(req->assoclen); + assocmem = malloc(assoclen); if (unlikely(assocmem == NULL)) { err = -ENOMEM; goto out; } assoc = assocmem; - scatterwalk_map_and_copy(assoc, req->src, 0, req->assoclen, 0); + scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); } - err = wc_AesGcmDecryptUpdate(aes_copy, NULL, NULL, 0, - assoc, req->assoclen); + if (decrypt_p) { + err = wc_AesGcmDecryptUpdate(aes_copy, NULL, NULL, 0, + assoc, assoclen); + } + else { + err = wc_AesGcmEncryptUpdate(aes_copy, NULL, NULL, 0, + assoc, assoclen); + } if (assocmem) free(assocmem); @@ -860,23 +928,37 @@ static int km_AesGcmDecrypt(struct aead_request *req) scatterwalk_unmap(assoc); if (unlikely(err)) { - pr_err("%s: wc_AesGcmDecryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + pr_err("%s: %s failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + decrypt_p ? "wc_AesGcmDecryptUpdate" : "wc_AesGcmEncryptUpdate", + err); err = -EINVAL; goto out; } while (walk.nbytes) { - err = wc_AesGcmDecryptUpdate( - aes_copy, - walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes, - NULL, 0); + if (decrypt_p) { + err = wc_AesGcmDecryptUpdate( + aes_copy, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes, + NULL, 0); + } + else { + err = wc_AesGcmEncryptUpdate( + aes_copy, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes, + NULL, 0); + } if (unlikely(err)) { - pr_err("%s: wc_AesGcmDecryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + pr_err("%s: %s failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + decrypt_p ? "wc_AesGcmDecryptUpdate" : "wc_AesGcmEncryptUpdate", + err); err = -EINVAL; goto out; } @@ -890,20 +972,35 @@ static int km_AesGcmDecrypt(struct aead_request *req) } } - err = wc_AesGcmDecryptFinal(aes_copy, origAuthTag, tfm->authsize); - if (unlikely(err)) { + if (decrypt_p) { + err = wc_AesGcmDecryptFinal(aes_copy, authTag, tfm->authsize); + if (unlikely(err)) { #ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG - pr_err("%s: wc_AesGcmDecryptFinal failed with return code %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + pr_err("%s: wc_AesGcmDecryptFinal failed with return code %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); #endif - if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) { - err = -EBADMSG; - goto out; + if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) { + err = -EBADMSG; + goto out; + } + else { + err = -EINVAL; + goto out; + } } - else { + } + else { + err = wc_AesGcmEncryptFinal(aes_copy, authTag, tfm->authsize); + if (unlikely(err)) { + pr_err("%s: wc_AesGcmEncryptFinal failed with return code %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); err = -EINVAL; goto out; } + /* Now copy the auth tag into request scatterlist. */ + scatterwalk_map_and_copy(authTag, req->dst, + req->assoclen + req->cryptlen, + tfm->authsize, 1); } out: @@ -916,6 +1013,194 @@ out: return err; } +#else /* !WOLFSSL_AESGCM_STREAM */ + +static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) +{ + struct crypto_aead * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk sk_walk; + struct scatter_walk in_walk, out_walk; + u8 *in_map = NULL, *out_map = NULL; + u8 authTag[WC_AES_BLOCK_SIZE]; + int err; + unsigned int assoclen = req->assoclen; + u8 * assoc = NULL; + u8 * sg_buf = NULL; + Aes *aes_copy; + u8 * in_text = NULL; + u8 * out_text = NULL; +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + byte rfc4106_iv[12]; +#endif + + tfm = crypto_aead_reqtfm(req); + ctx = crypto_aead_ctx(tfm); + + if (decrypt_p) { + /* Copy out original auth tag from req->src. */ + scatterwalk_map_and_copy(authTag, req->src, + req->assoclen + req->cryptlen - tfm->authsize, + tfm->authsize, 0); + err = skcipher_walk_aead_decrypt(&sk_walk, req, false); + } + else { + err = skcipher_walk_aead_encrypt(&sk_walk, req, false); + } + + if (unlikely(err)) { + pr_err("%s: %s failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + decrypt_p ? "skcipher_walk_aead_decrypt" : "skcipher_walk_aead_encrypt", + err); + return -EINVAL; + } + + /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and + * aes->streamData. + */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + if (rfc4106_p) { + if (unlikely(assoclen != 16 && assoclen != 20)) + return -EINVAL; + assoclen -= 8; + + memcpy(rfc4106_iv, ctx->rfc4106_nonce, 4); + memcpy(rfc4106_iv + 4, sk_walk.iv, 8); + } +#else + (void)rfc4106_p; +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + + if ((req->src->length >= req->assoclen + req->cryptlen) && + (req->dst->length >= req->assoclen + req->cryptlen)) + { + scatterwalk_start(&in_walk, req->src); + in_map = scatterwalk_map(&in_walk); + if (unlikely(IS_ERR(in_map))) { + pr_err("%s: scatterwalk_map failed: %ld\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + PTR_ERR(assoc)); + goto out; + } + assoc = in_map; + in_text = in_map + req->assoclen; + + scatterwalk_start(&out_walk, req->dst); + out_map = scatterwalk_map(&out_walk); + if (unlikely(IS_ERR(out_map))) { + pr_err("%s: scatterwalk_map failed: %ld\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + PTR_ERR(assoc)); + goto out; + } + out_text = out_map + req->assoclen; + } + else { + sg_buf = malloc(req->assoclen + req->cryptlen); + if (unlikely(sg_buf == NULL)) { + err = -ENOMEM; + goto out; + } + if (decrypt_p) + scatterwalk_map_and_copy(sg_buf, req->src, 0, req->assoclen + req->cryptlen - tfm->authsize, 0); + else + scatterwalk_map_and_copy(sg_buf, req->src, 0, req->assoclen + req->cryptlen, 0); + assoc = sg_buf; + in_text = out_text = sg_buf + req->assoclen; + } + + if (decrypt_p) { + err = wc_AesGcmDecrypt(aes_copy, out_text, in_text, req->cryptlen - tfm->authsize, +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + rfc4106_p ? rfc4106_iv : +#endif + sk_walk.iv, GCM_NONCE_MID_SZ, + authTag, tfm->authsize, + assoc, assoclen); + + if (unlikely(err)) { +#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG + pr_err("%s: wc_AesGcmDecrypt failed with return code %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); +#endif + + if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) { + err = -EBADMSG; + goto out; + } + else { + err = -EINVAL; + goto out; + } + } + } + else { + err = wc_AesGcmEncrypt(aes_copy, out_text, in_text, req->cryptlen, +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + rfc4106_p ? rfc4106_iv : +#endif + sk_walk.iv, GCM_NONCE_MID_SZ, + authTag, tfm->authsize, + assoc, assoclen); + + if (unlikely(err)) { + pr_err("%s: wc_AesGcmEncrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + } + + if (sg_buf) { + if (decrypt_p) + scatterwalk_map_and_copy(sg_buf, req->dst, 0, req->assoclen + req->cryptlen - tfm->authsize, 1); + else + scatterwalk_map_and_copy(sg_buf, req->dst, 0, req->assoclen + req->cryptlen, 1); + } + + if (! decrypt_p) { + /* Now copy the auth tag into request scatterlist. */ + scatterwalk_map_and_copy(authTag, req->dst, + req->assoclen + req->cryptlen, + tfm->authsize, 1); + } + +out: + + if (sg_buf) { + free(sg_buf); + } + else { + if (in_map) + scatterwalk_unmap(in_map); + if (out_map) + scatterwalk_unmap(out_map); + } + + free(aes_copy); + + return err; +} + +#endif /* !WOLFSSL_AESGCM_STREAM */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM + +static int km_AesGcmEncrypt(struct aead_request *req) { + return AesGcmCrypt_1(req, 0 /* decrypt_p */, 0 /* rfc4106_p */); +} + +static int km_AesGcmDecrypt(struct aead_request *req) { + return AesGcmCrypt_1(req, 1 /* decrypt_p */, 0 /* rfc4106_p */); +} + static struct aead_alg gcmAesAead = { .base.cra_name = WOLFKM_AESGCM_NAME, .base.cra_driver_name = WOLFKM_AESGCM_DRIVER, @@ -935,13 +1220,42 @@ static struct aead_alg gcmAesAead = { }; static int gcmAesAead_loaded = 0; -#endif /* HAVE_AESGCM && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESGCM) && - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */ -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + +static int km_AesGcmEncrypt_Rfc4106(struct aead_request *req) { + return AesGcmCrypt_1(req, 0 /* decrypt_p */, 1 /* rfc4106 */); +} + +static int km_AesGcmDecrypt_Rfc4106(struct aead_request *req) { + return AesGcmCrypt_1(req, 1 /* decrypt_p */, 1 /* rfc4106 */); +} + +static struct aead_alg gcmAesAead_rfc4106 = { + .base.cra_name = WOLFKM_AESGCM_RFC4106_NAME, + .base.cra_driver_name = WOLFKM_AESGCM_RFC4106_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesGcmInit, + .exit = km_AesGcmExit, + .setkey = km_AesGcmSetKey_Rfc4106, + .setauthsize = km_AesGcmSetAuthsize_Rfc4106, + .encrypt = km_AesGcmEncrypt_Rfc4106, + .decrypt = km_AesGcmDecrypt_Rfc4106, + .ivsize = 8, + .maxauthsize = WC_AES_BLOCK_SIZE, + .chunksize = WC_AES_BLOCK_SIZE, +}; +static int gcmAesAead_rfc4106_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM || LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS #ifndef WOLFSSL_AESXTS_STREAM #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream). @@ -1292,15 +1606,794 @@ static struct skcipher_alg xtsAesAlg = { }; static int xtsAesAlg_loaded = 0; -#endif /* WOLFSSL_AES_XTS && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESXTS) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESXTS */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + +static int km_AesCtrInit(struct crypto_skcipher *tfm) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesInitCommon(ctx, WOLFKM_AESCTR_DRIVER, 0); +} + +static int km_AesCtrSetKey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESCTR_DRIVER); +} + +static int km_AesCtrEncrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + int err; + Aes *aes_copy; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_virt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + + err = wc_AesSetIV(aes_copy, walk.iv); + + if (unlikely(err)) { + pr_err("%s: wc_AesSetIV failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes != 0) { + err = wc_AesCtrEncrypt(aes_copy, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + + if (unlikely(err)) { + pr_err("%s: wc_AesCtrEncrypt failed %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + } + + /* copy iv from wolfCrypt back to walk.iv */ + XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE); + +out: + + free(aes_copy); + + return err; +} + +static int km_AesCtrDecrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + int err; + Aes *aes_copy; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_virt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* CTR uses the same + * schedule for encrypt + * and decrypt. + */ + + err = wc_AesSetIV(aes_copy, walk.iv); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesSetIV failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes != 0) { + /* CTR uses the same function for encrypt and decrypt. */ + err = wc_AesCtrEncrypt(aes_copy, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + + if (unlikely(err)) { + pr_err("%s: wc_AesCtrDecrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + goto out; + } + } + + /* copy iv from wolfCrypt back to walk.iv */ + XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE); + +out: + + free(aes_copy); + + return err; +} + +static struct skcipher_alg ctrAesAlg = { + .base.cra_name = WOLFKM_AESCTR_NAME, + .base.cra_driver_name = WOLFKM_AESCTR_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesCtrInit, + .exit = km_AesExit, + .min_keysize = AES_128_KEY_SIZE, + .max_keysize = AES_256_KEY_SIZE, + .ivsize = WC_AES_BLOCK_SIZE, + .setkey = km_AesCtrSetKey, + .encrypt = km_AesCtrEncrypt, + .decrypt = km_AesCtrDecrypt, +}; +static int ctrAesAlg_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESCTR */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + +static int km_AesOfbInit(struct crypto_skcipher *tfm) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesInitCommon(ctx, WOLFKM_AESOFB_DRIVER, 0); +} + +static int km_AesOfbSetKey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESOFB_DRIVER); +} + +static int km_AesOfbEncrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + int err; + Aes *aes_copy; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_virt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + + err = wc_AesSetIV(aes_copy, walk.iv); + + if (unlikely(err)) { + pr_err("%s: wc_AesSetIV failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes != 0) { + err = wc_AesOfbEncrypt(aes_copy, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + + if (unlikely(err)) { + pr_err("%s: wc_AesOfbEncrypt failed %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + } + + /* copy iv from wolfCrypt back to walk.iv */ + XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE); + +out: + + free(aes_copy); + + return err; +} + +static int km_AesOfbDecrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + int err; + Aes *aes_copy; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_virt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* OFB uses the same + * schedule for encrypt + * and decrypt. + */ + + err = wc_AesSetIV(aes_copy, walk.iv); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesSetIV failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes != 0) { + err = wc_AesOfbDecrypt(aes_copy, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + + if (unlikely(err)) { + pr_err("%s: wc_AesOfbDecrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + goto out; + } + } + + /* copy iv from wolfCrypt back to walk.iv */ + XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE); + +out: + + free(aes_copy); + + return err; +} + +static struct skcipher_alg ofbAesAlg = { + .base.cra_name = WOLFKM_AESOFB_NAME, + .base.cra_driver_name = WOLFKM_AESOFB_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesOfbInit, + .exit = km_AesExit, + .min_keysize = AES_128_KEY_SIZE, + .max_keysize = AES_256_KEY_SIZE, + .ivsize = WC_AES_BLOCK_SIZE, + .setkey = km_AesOfbSetKey, + .encrypt = km_AesOfbEncrypt, + .decrypt = km_AesOfbDecrypt, +}; +static int ofbAesAlg_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESOFB */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + +static int km_AesEcbInit(struct crypto_skcipher *tfm) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesInitCommon(ctx, WOLFKM_AESECB_DRIVER, 1); +} + +static int km_AesEcbSetKey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESECB_DRIVER); +} + +static int km_AesEcbEncrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + unsigned int nbytes = 0; + int err; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + return err; + } + + while ((nbytes = walk.nbytes) != 0) { + err = wc_AesEcbEncrypt(ctx->aes_encrypt, walk.dst.virt.addr, + walk.src.virt.addr, nbytes & (~(WC_AES_BLOCK_SIZE - 1))); + + if (unlikely(err)) { + pr_err("%s: wc_AesEcbEncrypt failed for %u bytes: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), nbytes, err); + err = -EINVAL; + goto out; + } + + nbytes &= WC_AES_BLOCK_SIZE - 1; + err = skcipher_walk_done(&walk, nbytes); + } + +out: + + return err; +} + +static int km_AesEcbDecrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + unsigned int nbytes = 0; + int err; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + return err; + } + + while ((nbytes = walk.nbytes) != 0) { + err = wc_AesEcbDecrypt(ctx->aes_decrypt, walk.dst.virt.addr, + walk.src.virt.addr, nbytes & (~(WC_AES_BLOCK_SIZE - 1))); + + if (unlikely(err)) { + pr_err("%s: wc_AesEcbDecrypt failed for %u bytes: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), nbytes, err); + err = -EINVAL; + goto out; + } + + nbytes &= WC_AES_BLOCK_SIZE - 1; + err = skcipher_walk_done(&walk, nbytes); + } + +out: + + return err; +} + +static struct skcipher_alg ecbAesAlg = { + .base.cra_name = WOLFKM_AESECB_NAME, + .base.cra_driver_name = WOLFKM_AESECB_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = WC_AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesEcbInit, + .exit = km_AesExit, + .min_keysize = AES_128_KEY_SIZE, + .max_keysize = AES_256_KEY_SIZE, + .ivsize = 0, + .setkey = km_AesEcbSetKey, + .encrypt = km_AesEcbEncrypt, + .decrypt = km_AesEcbDecrypt, +}; +static int ecbAesAlg_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESECB */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 + +static int km_AesCcmInit(struct crypto_aead * tfm) +{ + struct km_AesCtx * ctx = crypto_aead_ctx(tfm); + return km_AesInitCommon(ctx, WOLFKM_AESCCM_DRIVER, 0); +} + +static void km_AesCcmExit(struct crypto_aead * tfm) +{ + struct km_AesCtx * ctx = crypto_aead_ctx(tfm); + km_AesExitCommon(ctx); +} + +static int km_AesCcmSetKey_rfc4309(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + int err; + struct km_AesCtx * ctx = crypto_aead_ctx(tfm); + + + + err = wc_AesCcmSetKey(ctx->aes_encrypt, in_key, key_len); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesCcmSetKey failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return -EINVAL; + } + + return 0; +} + +static int km_AesCcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + (void)tfm; + + if (wc_AesCcmCheckTagSize((int)authsize) == 0) + return 0; + +#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG + pr_err("%s: invalid authsize: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), authsize); +#endif + return -EINVAL; +} + +/* + * aead ciphers receive data in scatterlists in following order: + * encrypt + * req->src: aad||plaintext + * req->dst: aad||ciphertext||tag + * decrypt + * req->src: aad||ciphertext||tag + * req->dst: aad||plaintext, return 0 or -EBADMSG + */ + +static int km_AesCcmEncrypt(struct aead_request *req) +{ + struct crypto_aead * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + struct scatter_walk copy_walk; + u8 * copy_mem = NULL; + u8 authTag[WC_AES_BLOCK_SIZE]; + int err; + u8 * assoc = NULL; + Aes *aes_copy = NULL; + u8 * plaintext = NULL; + u8 * plaintext_copy = NULL; + u8 * ciphertext = NULL; + int iv_size; + + tfm = crypto_aead_reqtfm(req); + ctx = crypto_aead_ctx(tfm); + + err = skcipher_walk_aead_encrypt(&walk, req, false); + if (unlikely(err)) { + pr_err("%s: skcipher_walk_aead_encrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return -EINVAL; + } + + if (req->src->length >= req->assoclen + req->cryptlen && req->src->length) { + scatterwalk_start(©_walk, req->src); + assoc = scatterwalk_map(©_walk); + if (unlikely(IS_ERR(assoc))) { + pr_err("%s: scatterwalk_map failed: %ld\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + PTR_ERR(assoc)); + goto out; + } + plaintext = assoc + req->assoclen; + ciphertext = ?; + } + else { + copy_mem = malloc(req->assoclen + req->cryptlen); + if (unlikely(copy_mem == NULL)) { + err = -ENOMEM; + goto out; + } + scatterwalk_map_and_copy(copy_mem, req->src, 0, req->assoclen + req->cryptlen, 0); + assoc = copy_mem; + plaintext = assoc + req->assoclen; + ciphertext = plaintext; + } + + + /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and + * aes->streamData. + */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) { + err = -ENOMEM; + goto out; + } + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); +#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) + aes_copy->streamData = NULL; +#endif + + + +err = wc_AesCcmEncrypt(aes_copy, ciphertext, plaintext, req->cryptlen, + walk.iv, crypto_aead_ivsize(crypto_aead_tfm(tfm)), + authTag, sizeof authTag, + assoc, req->assoclen); + + if (unlikely(err)) { + pr_err("%s: wc_AesCcmEncrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + goto out; + } + } + + err = wc_AesCcmEncryptFinal(aes_copy, authTag, tfm->authsize); + if (unlikely(err)) { + pr_err("%s: wc_AesCcmEncryptFinal failed with return code %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + /* Now copy the auth tag into request scatterlist. */ + scatterwalk_map_and_copy(authTag, req->dst, + req->assoclen + req->cryptlen, + tfm->authsize, 1); + +out: + + if (assocmem) + free(assocmem); + else if (assoc) + scatterwalk_unmap(assoc); + + if (plaintext_copy) + free(plaintext_copy); + else if (plaintext) + scatterwalk_unmap(plaintext); + + if (aes_copy) { +#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) + free(aes_copy->streamData); +#endif + free(aes_copy); + } + + return err; +} + +static int km_AesCcmDecrypt(struct aead_request *req) +{ + struct crypto_aead * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + struct scatter_walk assocSgWalk; + u8 origAuthTag[WC_AES_BLOCK_SIZE]; + int err; + u8 * assoc = NULL; + u8 * assocmem = NULL; + Aes *aes_copy; + + tfm = crypto_aead_reqtfm(req); + ctx = crypto_aead_ctx(tfm); + + /* Copy out original auth tag from req->src. */ + scatterwalk_map_and_copy(origAuthTag, req->src, + req->assoclen + req->cryptlen - tfm->authsize, + tfm->authsize, 0); + + err = skcipher_walk_aead_decrypt(&walk, req, false); + if (unlikely(err)) { + pr_err("%s: skcipher_walk_aead_decrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and + * aes->streamData. + */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* GCM uses the same + * schedule for encrypt + * and decrypt. + */ +#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) + aes_copy->streamData = NULL; +#endif + + err = wc_AesCcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv, + GCM_NONCE_MID_SZ); + if (unlikely(err)) { + pr_err("%s: wc_AesCcmInit failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + if (req->src->length >= req->assoclen && req->src->length) { + scatterwalk_start(&assocSgWalk, req->src); + assoc = scatterwalk_map(&assocSgWalk); + if (unlikely(IS_ERR(assoc))) { + pr_err("%s: scatterwalk_map failed: %ld\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + PTR_ERR(assoc)); + goto out; + } + } + else { + /* assoc can be any length, so if it's noncontiguous, we have to copy it + * to a contiguous heap allocation. + */ + assocmem = malloc(req->assoclen); + if (unlikely(assocmem == NULL)) { + err = -ENOMEM; + goto out; + } + assoc = assocmem; + scatterwalk_map_and_copy(assoc, req->src, 0, req->assoclen, 0); + } + + err = wc_AesCcmDecryptUpdate(aes_copy, NULL, NULL, 0, + assoc, req->assoclen); + + if (assocmem) + free(assocmem); + else + scatterwalk_unmap(assoc); + + if (unlikely(err)) { + pr_err("%s: wc_AesCcmDecryptUpdate failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes) { + err = wc_AesCcmDecryptUpdate( + aes_copy, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes, + NULL, 0); + + if (unlikely(err)) { + pr_err("%s: wc_AesCcmDecryptUpdate failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + goto out; + } + } + + err = wc_AesCcmDecryptFinal(aes_copy, origAuthTag, tfm->authsize); + if (unlikely(err)) { +#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG + pr_err("%s: wc_AesCcmDecryptFinal failed with return code %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); +#endif + if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) { + err = -EBADMSG; + goto out; + } + else { + err = -EINVAL; + goto out; + } + } + +out: + +#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) + free(aes_copy->streamData); +#endif + free(aes_copy); + + return err; +} + +static struct aead_alg ccmAesAead = { + .base.cra_name = WOLFKM_AESCCM_NAME, + .base.cra_driver_name = WOLFKM_AESCCM_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesCcmInit, + .exit = km_AesCcmExit, + .setkey = km_AesCcmSetKey, + .setauthsize = km_AesCcmSetAuthsize, + .encrypt = km_AesCcmEncrypt, + .decrypt = km_AesCcmDecrypt, + .ivsize = CCM_NONCE_MID_SZ, + .maxauthsize = WC_AES_BLOCK_SIZE, + .chunksize = WC_AES_BLOCK_SIZE, +}; +static int ccmAesAead_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM */ /* cipher tests, cribbed from test.c, with supplementary LKCAPI tests: */ -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC static int linuxkm_test_aescbc(void) { @@ -1502,13 +2595,9 @@ test_cbc_end: return ret; } -#endif /* HAVE_AES_CBC && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCBC) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */ -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB static int linuxkm_test_aescfb(void) { @@ -1708,16 +2797,15 @@ test_cfb_end: return ret; } -#endif /* WOLFSSL_AES_CFB && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCFB) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESCFB */ -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM static int linuxkm_test_aesgcm(void) { +#ifndef WOLFSSL_AESGCM_STREAM + return 0; +#else int ret = 0; struct crypto_aead * tfm = NULL; struct aead_request * req = NULL; @@ -1998,15 +3086,21 @@ test_gcm_end: free(aes); return ret; +#endif /* WOLFSSL_AESGCM_STREAM */ } -#endif /* HAVE_AESGCM && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESGCM) && - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */ -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + +static int linuxkm_test_aesgcm_rfc4106(void) +{ + return 0; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS /* test vectors from * http://csrc.nist.gov/groups/STM/cavp/block-cipher-modes.html @@ -3114,21 +4208,42 @@ out: return ret; } -#endif /* WOLFSSL_AES_XTS && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESXTS) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESXTS */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + +static int linuxkm_test_aesctr(void) { + return 0; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESCTR */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + +static int linuxkm_test_aesofb(void) { + return 0; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESOFB */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + +static int linuxkm_test_aesecb(void) { + return 0; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESECB */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM + +static int linuxkm_test_aesccm(void) { + return 0; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM */ #endif /* !NO_AES */ -#if defined(HAVE_FIPS) && defined(CONFIG_CRYPTO_MANAGER) && \ - !defined(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) - #ifdef CONFIG_CRYPTO_FIPS - #include - #else - #error wolfCrypt FIPS with LINUXKM_LKCAPI_REGISTER and CONFIG_CRYPTO_MANAGER requires CONFIG_CRYPTO_FIPS - #endif -#endif - static int linuxkm_lkcapi_register(void) { int ret = 0; @@ -3188,33 +4303,36 @@ static int linuxkm_lkcapi_register(void) (alg).base.cra_priority); \ } while (0) -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC REGISTER_ALG(cbcAesAlg, crypto_register_skcipher, linuxkm_test_aescbc); #endif - -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB REGISTER_ALG(cfbAesAlg, crypto_register_skcipher, linuxkm_test_aescfb); #endif - -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM REGISTER_ALG(gcmAesAead, crypto_register_aead, linuxkm_test_aesgcm); #endif - -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + REGISTER_ALG(gcmAesAead_rfc4106, crypto_register_aead, linuxkm_test_aesgcm_rfc4106); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS REGISTER_ALG(xtsAesAlg, crypto_register_skcipher, linuxkm_test_aesxts); #endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + REGISTER_ALG(ctrAesAlg, crypto_register_skcipher, linuxkm_test_aesctr); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + REGISTER_ALG(ofbAesAlg, crypto_register_skcipher, linuxkm_test_aesofb); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + REGISTER_ALG(ecbAesAlg, crypto_register_skcipher, linuxkm_test_aesecb); +#endif + +#ifdef notyet +#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM + REGISTER_ALG(ccmAesAead, crypto_register_aead, linuxkm_test_aesccm); +#endif +#endif /* notyet */ #undef REGISTER_ALG @@ -3241,30 +4359,36 @@ static void linuxkm_lkcapi_unregister(void) } \ } while (0) -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC UNREGISTER_ALG(cbcAesAlg, crypto_unregister_skcipher); #endif -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB UNREGISTER_ALG(cfbAesAlg, crypto_unregister_skcipher); #endif -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM UNREGISTER_ALG(gcmAesAead, crypto_unregister_aead); #endif -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + UNREGISTER_ALG(gcmAesAead_rfc4106, crypto_unregister_aead); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS UNREGISTER_ALG(xtsAesAlg, crypto_unregister_skcipher); #endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + UNREGISTER_ALG(ctrAesAlg, crypto_unregister_skcipher); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + UNREGISTER_ALG(ofbAesAlg, crypto_unregister_skcipher); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + UNREGISTER_ALG(ecbAesAlg, crypto_unregister_skcipher); +#endif + +#ifdef notyet +#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM + UNREGISTER_ALG(ccmAesAlg, crypto_unregister_aead); +#endif +#endif /* notyet */ #undef UNREGISTER_ALG } From 6d92dae632592d1288c06068f85c46a0bf2ce556 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Fri, 28 Mar 2025 15:48:06 -0500 Subject: [PATCH 2/6] configure.ac: add support for --enable-aesni-with-avx/USE_INTEL_SPEEDUP_FOR_AES (AESNI+AVX, but only for AES modes). linuxkm/lkcapi_glue.c: implement WC_LINUXKM_C_FALLBACK_IN_SHIMS, km_AesGet(), and km_AesFree(). src/include.am: add missing gates for AES-GCM and AES-XTS asm. wolfcrypt/src/aes_xts_asm.S and wolfssl/wolfcrypt/sp_int.h: don't redefine HAVE_INTEL_AVX2. --- configure.ac | 25 ++- linuxkm/lkcapi_glue.c | 297 +++++++++++++++++++++++++----------- src/include.am | 16 ++ wolfcrypt/src/aes_xts_asm.S | 2 +- wolfssl/wolfcrypt/sp_int.h | 6 +- 5 files changed, 252 insertions(+), 94 deletions(-) diff --git a/configure.ac b/configure.ac index 7206b7218..5b45e3cc2 100644 --- a/configure.ac +++ b/configure.ac @@ -3508,6 +3508,13 @@ AC_ARG_ENABLE([aesni], [ ENABLED_AESNI=no ] ) +# INTEL AES-NI with AVX +AC_ARG_ENABLE([aesni-with-avx], + [AS_HELP_STRING([--enable-aesni-with-avx],[Enable AES-NI with additional AVX acceleration for AES (default: disabled)])], + [ ENABLED_AESNI_WITH_AVX=$enableval ], + [ ENABLED_AESNI_WITH_AVX=no ] + ) + # INTEL ASM AC_ARG_ENABLE([intelasm], [AS_HELP_STRING([--enable-intelasm],[Enable All Intel ASM speedups (default: disabled)])], @@ -3523,6 +3530,17 @@ then ENABLED_AESNI=yes fi + if test "$ENABLED_INTELASM" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DUSE_INTEL_SPEEDUP" + ENABLED_AESNI=yes + ENABLED_AESNI_WITH_AVX=yes + elif test "$ENABLED_AESNI_WITH_AVX" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DUSE_INTEL_SPEEDUP_FOR_AES" + ENABLED_AESNI=yes + fi + if test "$ENABLED_AESNI" = "yes" || test "$ENABLED_INTELASM" = "yes" then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_AESNI" @@ -3548,12 +3566,6 @@ then AS_IF([test "x$ENABLED_SM3" != "xno"],[AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SM3"]) fi - if test "$ENABLED_INTELASM" = "yes" - then - AM_CFLAGS="$AM_CFLAGS -DUSE_INTEL_SPEEDUP" - ENABLED_AESNI=yes - fi - if test "$host_cpu" = "x86_64" || test "$host_cpu" = "amd64" then AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_X86_64_BUILD" @@ -10837,6 +10849,7 @@ echo " * snifftest: $ENABLED_SNIFFTEST" echo " * ARC4: $ENABLED_ARC4" echo " * AES: $ENABLED_AES" echo " * AES-NI: $ENABLED_AESNI" +echo " * AVX for AES: $ENABLED_AESNI_WITH_AVX" echo " * AES-CBC: $ENABLED_AESCBC" echo " * AES-CBC length checks: $ENABLED_AESCBC_LENGTH_CHECKS" echo " * AES-GCM: $ENABLED_AESGCM" diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index ef567c14b..e778920e1 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -221,11 +221,29 @@ static int disable_setkey_warnings = 0; #include +#if !defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && defined(WOLFSSL_AESNI) && (!defined(WC_AES_C_DYNAMIC_FALLBACK) || (defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0))) + #define WC_LINUXKM_C_FALLBACK_IN_SHIMS +#elif !defined(WOLFSSL_AESNI) + #undef WC_LINUXKM_C_FALLBACK_IN_SHIMS +#endif + +#if defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && !defined(WC_FLAG_DONT_USE_AESNI) + #error WC_LINUXKM_C_FALLBACK_IN_SHIMS is defined but WC_FLAG_DONT_USE_AESNI is missing. +#endif + +#if defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && !defined(CAN_SAVE_VECTOR_REGISTERS) + #error WC_LINUXKM_C_FALLBACK_IN_SHIMS is defined but CAN_SAVE_VECTOR_REGISTERS is missing. +#endif + struct km_AesCtx { Aes *aes_encrypt; /* allocated in km_AesInitCommon() to assure * alignment, needed for AESNI. */ Aes *aes_decrypt; /* same. */ +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + Aes *aes_encrypt_C; /* fallback if vector registers aren't available. */ + Aes *aes_decrypt_C; +#endif #if defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309) union { #ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 @@ -261,7 +279,8 @@ static int km_AesInitCommon( if (! ctx->aes_encrypt) { pr_err("%s: allocation of %zu bytes for encryption key failed.\n", name, sizeof(*ctx->aes_encrypt)); - return MEMORY_E; + err = -MEMORY_E; + goto out; } err = wc_AesInit(ctx->aes_encrypt, NULL, INVALID_DEVID); @@ -270,48 +289,166 @@ static int km_AesInitCommon( pr_err("%s: wc_AesInit failed: %d\n", name, err); free(ctx->aes_encrypt); ctx->aes_encrypt = NULL; - return -EINVAL; + err = -EINVAL; + goto out; } if (! need_decryption) { ctx->aes_decrypt = NULL; - return 0; + } + else { + ctx->aes_decrypt = (Aes *)malloc(sizeof(*ctx->aes_decrypt)); + + if (! ctx->aes_decrypt) { + pr_err("%s: allocation of %zu bytes for decryption key failed.\n", + name, sizeof(*ctx->aes_decrypt)); + err = -MEMORY_E; + goto out; + } + + err = wc_AesInit(ctx->aes_decrypt, NULL, INVALID_DEVID); + + if (unlikely(err)) { + pr_err("%s: wc_AesInit failed: %d\n", name, err); + free(ctx->aes_decrypt); + ctx->aes_decrypt = NULL; + err = -EINVAL; + goto out; + } } - ctx->aes_decrypt = (Aes *)malloc(sizeof(*ctx->aes_decrypt)); +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS - if (! ctx->aes_decrypt) { - pr_err("%s: allocation of %zu bytes for decryption key failed.\n", - name, sizeof(*ctx->aes_decrypt)); - km_AesExitCommon(ctx); - return MEMORY_E; + ctx->aes_encrypt_C = (Aes *)malloc(sizeof(*ctx->aes_encrypt_C)); + + if (! ctx->aes_encrypt_C) { + pr_err("%s: allocation of %zu bytes for encryption key failed.\n", + name, sizeof(*ctx->aes_encrypt_C)); + err = -MEMORY_E; + goto out; } - err = wc_AesInit(ctx->aes_decrypt, NULL, INVALID_DEVID); + err = wc_AesInit(ctx->aes_encrypt_C, NULL, INVALID_DEVID); if (unlikely(err)) { pr_err("%s: wc_AesInit failed: %d\n", name, err); - free(ctx->aes_decrypt); - ctx->aes_decrypt = NULL; + free(ctx->aes_encrypt_C); + ctx->aes_encrypt_C = NULL; + err = -EINVAL; + goto out; + } + + ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + if (! need_decryption) { + ctx->aes_decrypt_C = NULL; + } + else { + ctx->aes_decrypt_C = (Aes *)malloc(sizeof(*ctx->aes_decrypt_C)); + + if (! ctx->aes_decrypt_C) { + pr_err("%s: allocation of %zu bytes for decryption key failed.\n", + name, sizeof(*ctx->aes_decrypt_C)); + err = -MEMORY_E; + goto out; + } + + err = wc_AesInit(ctx->aes_decrypt_C, NULL, INVALID_DEVID); + + if (unlikely(err)) { + pr_err("%s: wc_AesInit failed: %d\n", name, err); + free(ctx->aes_decrypt_C); + ctx->aes_decrypt_C = NULL; + err = -EINVAL; + goto out; + } + + ctx->aes_decrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + } + +#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */ + +out: + + if (err != 0) km_AesExitCommon(ctx); - return -EINVAL; + + return err; +} + +static int km_AesGet(struct km_AesCtx *ctx, int decrypt_p, int copy_p, Aes **aes) { + Aes *ret; + +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + if (! CAN_SAVE_VECTOR_REGISTERS()) { + if (decrypt_p && ctx->aes_decrypt_C) + ret = ctx->aes_decrypt_C; + else + ret = ctx->aes_encrypt_C; + } + else +#endif + { + if (decrypt_p && ctx->aes_decrypt) + ret = ctx->aes_decrypt; + else + ret = ctx->aes_encrypt; + } + + if (copy_p) { + /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and dynamic Aes.use_aesni. */ + Aes *aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ret, sizeof(Aes)); +#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) + aes_copy->streamData = NULL; +#endif + *aes = aes_copy; + } + else { + *aes = ret; } return 0; } +static void km_AesFree(Aes **aes) { + if ((! aes) || (! *aes)) + return; +#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0) + #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) + if ((*aes)->streamData) { + ForceZero((*aes)->streamData, 5 * AES_BLOCK_SIZE); + free((*aes)->streamData); + } + #endif +#endif + wc_AesFree(*aes); +#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0) + ForceZero(*aes, sizeof **aes); +#endif + free(*aes); + *aes = NULL; +} + static void km_AesExitCommon(struct km_AesCtx * ctx) { if (ctx->aes_encrypt) { - wc_AesFree(ctx->aes_encrypt); - free(ctx->aes_encrypt); - ctx->aes_encrypt = NULL; + km_AesFree(&ctx->aes_encrypt); } if (ctx->aes_decrypt) { - wc_AesFree(ctx->aes_decrypt); - free(ctx->aes_decrypt); - ctx->aes_decrypt = NULL; + km_AesFree(&ctx->aes_decrypt); } + +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + if (ctx->aes_encrypt_C) { + km_AesFree(&ctx->aes_encrypt_C); + } + if (ctx->aes_decrypt_C) { + km_AesFree(&ctx->aes_decrypt_C); + } +#endif } #if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ @@ -398,16 +535,14 @@ static int km_AesCbcEncrypt(struct skcipher_request *req) ctx = crypto_skcipher_ctx(tfm); err = skcipher_walk_virt(&walk, req, false); - if (unlikely(err)) { return err; } - /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + err = km_AesGet(ctx, 0 /* decrypt_p */, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } err = wc_AesSetIV(aes_copy, walk.iv); @@ -438,7 +573,7 @@ static int km_AesCbcEncrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -461,11 +596,10 @@ static int km_AesCbcDecrypt(struct skcipher_request *req) return err; } - /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_decrypt, sizeof(Aes)); + err = km_AesGet(ctx, 1 /* decrypt_p */, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } err = wc_AesSetIV(aes_copy, walk.iv); @@ -497,7 +631,7 @@ static int km_AesCbcDecrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -556,11 +690,10 @@ static int km_AesCfbEncrypt(struct skcipher_request *req) return err; } - /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + err = km_AesGet(ctx, 0 /* decrypt_p */, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } err = wc_AesSetIV(aes_copy, walk.iv); @@ -596,7 +729,7 @@ static int km_AesCfbEncrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -620,14 +753,10 @@ static int km_AesCfbDecrypt(struct skcipher_request *req) return err; } - /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* CFB uses the same - * schedule for encrypt - * and decrypt. - */ + err = km_AesGet(ctx, 1 /* decrypt_p */, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } err = wc_AesSetIV(aes_copy, walk.iv); @@ -664,7 +793,7 @@ static int km_AesCfbDecrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -848,19 +977,10 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) return err; } - /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and - * aes->streamData. - */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* GCM uses the same - * schedule for encrypt - * and decrypt. - */ -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - aes_copy->streamData = NULL; -#endif + err = km_AesGet(ctx, decrypt_p, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } #ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 if (rfc4106_p) { @@ -1005,10 +1125,7 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) out: -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - free(aes_copy->streamData); -#endif - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -1056,13 +1173,10 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) return -EINVAL; } - /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and - * aes->streamData. - */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + err = km_AesGet(ctx, decrypt_p, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } #ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 if (rfc4106_p) { @@ -1184,7 +1298,7 @@ out: scatterwalk_unmap(out_map); } - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -1261,6 +1375,10 @@ static int gcmAesAead_rfc4106_loaded = 0; #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream). #endif +#ifndef WC_AES_C_DYNAMIC_FALLBACK + #error LKCAPI registration of AES-XTS requires WC_AES_C_DYNAMIC_FALLBACK. +#endif + struct km_AesXtsCtx { XtsAes *aesXts; /* allocated in km_AesXtsInitCommon() to assure alignment * for AESNI. @@ -1682,7 +1800,7 @@ static int km_AesCtrEncrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -1751,7 +1869,7 @@ static int km_AesCtrDecrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -1850,7 +1968,7 @@ static int km_AesOfbEncrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -1918,7 +2036,7 @@ static int km_AesOfbDecrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -1965,6 +2083,7 @@ static int km_AesEcbEncrypt(struct skcipher_request *req) struct skcipher_walk walk; unsigned int nbytes = 0; int err; + Aes *aes; tfm = crypto_skcipher_reqtfm(req); ctx = crypto_skcipher_ctx(tfm); @@ -1975,9 +2094,14 @@ static int km_AesEcbEncrypt(struct skcipher_request *req) return err; } + err = km_AesGet(ctx, 0 /* decrypt_p */, 0 /* copy_p */, &aes); + if (unlikely(err)) { + return err; + } + while ((nbytes = walk.nbytes) != 0) { - err = wc_AesEcbEncrypt(ctx->aes_encrypt, walk.dst.virt.addr, - walk.src.virt.addr, nbytes & (~(WC_AES_BLOCK_SIZE - 1))); + err = wc_AesEcbEncrypt(aes, walk.dst.virt.addr, walk.src.virt.addr, + nbytes & (~(WC_AES_BLOCK_SIZE - 1))); if (unlikely(err)) { pr_err("%s: wc_AesEcbEncrypt failed for %u bytes: %d\n", @@ -2002,6 +2126,7 @@ static int km_AesEcbDecrypt(struct skcipher_request *req) struct skcipher_walk walk; unsigned int nbytes = 0; int err; + Aes *aes; tfm = crypto_skcipher_reqtfm(req); ctx = crypto_skcipher_ctx(tfm); @@ -2012,9 +2137,14 @@ static int km_AesEcbDecrypt(struct skcipher_request *req) return err; } + err = km_AesGet(ctx, 1 /* decrypt_p */, 0 /* copy_p */, &aes); + if (unlikely(err)) { + return err; + } + while ((nbytes = walk.nbytes) != 0) { - err = wc_AesEcbDecrypt(ctx->aes_decrypt, walk.dst.virt.addr, - walk.src.virt.addr, nbytes & (~(WC_AES_BLOCK_SIZE - 1))); + err = wc_AesEcbDecrypt(aes, walk.dst.virt.addr, walk.src.virt.addr, + nbytes & (~(WC_AES_BLOCK_SIZE - 1))); if (unlikely(err)) { pr_err("%s: wc_AesEcbDecrypt failed for %u bytes: %d\n", @@ -2226,7 +2356,7 @@ out: #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) free(aes_copy->streamData); #endif - free(aes_copy); + km_AesFree(&aes_copy); } return err; @@ -2362,10 +2492,7 @@ static int km_AesCcmDecrypt(struct aead_request *req) out: -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - free(aes_copy->streamData); -#endif - free(aes_copy); + km_AesFree(&aes_copy); return err; } diff --git a/src/include.am b/src/include.am index 9be3060fc..876ba7c65 100644 --- a/src/include.am +++ b/src/include.am @@ -72,10 +72,14 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else +if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif +if BUILD_AESXTS src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S endif endif +endif if BUILD_DES3 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/des3.c @@ -194,10 +198,14 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else +if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif +if BUILD_AESXTS src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S endif endif +endif if BUILD_RISCV_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/riscv/riscv-64-aes.c @@ -397,10 +405,14 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else +if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif +if BUILD_AESXTS src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S endif endif +endif if BUILD_SHA src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha.c @@ -1100,11 +1112,15 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else +if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif +if BUILD_AESXTS src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S endif endif endif +endif if BUILD_CAMELLIA src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/camellia.c diff --git a/wolfcrypt/src/aes_xts_asm.S b/wolfcrypt/src/aes_xts_asm.S index 3d90cec33..7d9f0b989 100644 --- a/wolfcrypt/src/aes_xts_asm.S +++ b/wolfcrypt/src/aes_xts_asm.S @@ -41,7 +41,7 @@ #ifndef HAVE_INTEL_AVX1 #define HAVE_INTEL_AVX1 #endif /* HAVE_INTEL_AVX1 */ -#ifndef NO_AVX2_SUPPORT +#if !defined(NO_AVX2_SUPPORT) && !defined(HAVE_INTEL_AVX2) #define HAVE_INTEL_AVX2 #endif /* NO_AVX2_SUPPORT */ diff --git a/wolfssl/wolfcrypt/sp_int.h b/wolfssl/wolfcrypt/sp_int.h index 7c5fdd1ab..fb202d4c5 100644 --- a/wolfssl/wolfcrypt/sp_int.h +++ b/wolfssl/wolfcrypt/sp_int.h @@ -210,8 +210,10 @@ extern "C" { #elif defined(WOLFSSL_SP_X86_64_ASM) || defined(WOLFSSL_SP_X86_64) #if SP_ULONG_BITS == 64 || SP_ULLONG_BITS == 64 #define SP_WORD_SIZE 64 - #define HAVE_INTEL_AVX1 - #ifndef NO_AVX2_SUPPORT + #ifndef HAVE_INTEL_AVX1 + #define HAVE_INTEL_AVX1 + #endif + #if !defined(NO_AVX2_SUPPORT) && !defined(HAVE_INTEL_AVX2) #define HAVE_INTEL_AVX2 #endif #elif SP_ULONG_BITS == 32 From 3c1672253860444ace423d98e9d5baca802a0e92 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Fri, 28 Mar 2025 22:58:53 -0500 Subject: [PATCH 3/6] wolfcrypt/src/aes.c and wolfssl/wolfcrypt/aes.h: add support for WC_FLAG_DONT_USE_AESNI in wc_AesSetKeyLocal(); add support for USE_INTEL_SPEEDUP_FOR_AES. linuxkm/lkcapi_glue.c: finish implementation of WC_LINUXKM_C_FALLBACK_IN_SHIMS and add TEST_WC_LINUXKM_C_FALLBACK_IN_SHIMS. use "WC_C_DYNAMIC_FALLBACK" consistently (remove/replace uses of "WC_AES_C_DYNAMIC_FALLBACK"). --- .github/workflows/intelasm-c-fallback.yml | 2 +- linuxkm/lkcapi_glue.c | 106 ++++++++++++++++++++-- wolfcrypt/src/aes.c | 23 ++++- wolfssl/wolfcrypt/aes.h | 7 ++ 4 files changed, 123 insertions(+), 15 deletions(-) diff --git a/.github/workflows/intelasm-c-fallback.yml b/.github/workflows/intelasm-c-fallback.yml index 33fb20d4e..49d3639bc 100644 --- a/.github/workflows/intelasm-c-fallback.yml +++ b/.github/workflows/intelasm-c-fallback.yml @@ -18,7 +18,7 @@ jobs: matrix: config: [ # Add new configs here - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy CPPFLAGS="-DNO_WOLFSSL_CIPHER_SUITE_TEST -DWC_AES_C_DYNAMIC_FALLBACK -DWC_C_DYNAMIC_FALLBACK -DDEBUG_VECTOR_REGISTER_ACCESS -DDEBUG_VECTOR_REGISTER_ACCESS_FUZZING -DWC_DEBUG_CIPHER_LIFECYCLE"' + '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy CPPFLAGS="-DNO_WOLFSSL_CIPHER_SUITE_TEST -DWC_C_DYNAMIC_FALLBACK -DDEBUG_VECTOR_REGISTER_ACCESS -DDEBUG_VECTOR_REGISTER_ACCESS_FUZZING -DWC_DEBUG_CIPHER_LIFECYCLE"' ] name: make check if: github.repository_owner == 'wolfssl' diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index e778920e1..9d971571f 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -221,7 +221,10 @@ static int disable_setkey_warnings = 0; #include -#if !defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && defined(WOLFSSL_AESNI) && (!defined(WC_AES_C_DYNAMIC_FALLBACK) || (defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0))) +#if defined(WOLFSSL_AESNI) && \ + (!defined(WC_C_DYNAMIC_FALLBACK) || \ + (defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0))) && \ + !defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) #define WC_LINUXKM_C_FALLBACK_IN_SHIMS #elif !defined(WOLFSSL_AESNI) #undef WC_LINUXKM_C_FALLBACK_IN_SHIMS @@ -338,8 +341,6 @@ static int km_AesInitCommon( goto out; } - ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; - if (! need_decryption) { ctx->aes_decrypt_C = NULL; } @@ -362,8 +363,6 @@ static int km_AesInitCommon( err = -EINVAL; goto out; } - - ctx->aes_decrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; } #endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */ @@ -380,14 +379,30 @@ static int km_AesGet(struct km_AesCtx *ctx, int decrypt_p, int copy_p, Aes **aes Aes *ret; #ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS - if (! CAN_SAVE_VECTOR_REGISTERS()) { + /* First, check if AESNI was disabled in the main SetKey for the requested + * direction. If so, use it (the fallback schedule won't even be inited). + */ + if (((! decrypt_p) || (! ctx->aes_decrypt)) && (! ctx->aes_encrypt->use_aesni)) + ret = ctx->aes_encrypt; + else if (decrypt_p && ctx->aes_decrypt && (! ctx->aes_decrypt->use_aesni)) + ret = ctx->aes_decrypt; + else if ( +#ifdef TEST_WC_LINUXKM_C_FALLBACK_IN_SHIMS + 1 +#else + ! CAN_SAVE_VECTOR_REGISTERS() +#endif + ) + { if (decrypt_p && ctx->aes_decrypt_C) ret = ctx->aes_decrypt_C; else ret = ctx->aes_encrypt_C; + if (ret->use_aesni) + return -EINVAL; } else -#endif +#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */ { if (decrypt_p && ctx->aes_decrypt) ret = ctx->aes_decrypt; @@ -483,6 +498,43 @@ static int km_AesSetKeyCommon(struct km_AesCtx * ctx, const u8 *in_key, } } +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + + if (ctx->aes_encrypt->use_aesni) { + ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + err = wc_AesSetKey(ctx->aes_encrypt_C, in_key, key_len, NULL, AES_ENCRYPTION); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesSetKey for encryption key failed: %d\n", name, err); + return -EINVAL; + } + + if (ctx->aes_encrypt_C->use_aesni) + pr_err("%s: after wc_AesSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", name); + + } + + if (ctx->aes_decrypt_C && ctx->aes_decrypt->use_aesni) { + ctx->aes_decrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + err = wc_AesSetKey(ctx->aes_decrypt_C, in_key, key_len, NULL, + AES_DECRYPTION); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesSetKey for decryption key failed: %d\n", + name, err); + return -EINVAL; + } + + if (ctx->aes_decrypt_C->use_aesni) + pr_err("%s: after wc_AesSetKey, ctx->aes_decrypt_C has AES-NI asserted.\n", name); + } + +#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */ + return 0; } @@ -850,6 +902,24 @@ static int km_AesGcmSetKey(struct crypto_aead *tfm, const u8 *in_key, return -EINVAL; } +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + if (ctx->aes_encrypt->use_aesni) { + ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + err = wc_AesGcmSetKey(ctx->aes_encrypt_C, in_key, key_len); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesGcmSetKey failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return -EINVAL; + } + + if (ctx->aes_encrypt_C->use_aesni) + pr_err("%s: after wc_AesGcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESGCM_DRIVER); + } +#endif + return 0; } @@ -877,6 +947,24 @@ static int km_AesGcmSetKey_Rfc4106(struct crypto_aead *tfm, const u8 *in_key, return -EINVAL; } +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + if (ctx->aes_encrypt->use_aesni) { + ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + err = wc_AesGcmSetKey(ctx->aes_encrypt_C, in_key, key_len); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesGcmSetKey failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return -EINVAL; + } + + if (ctx->aes_encrypt_C->use_aesni) + pr_err("%s: after wc_AesGcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESGCM_DRIVER); + } +#endif + return 0; } @@ -1375,8 +1463,8 @@ static int gcmAesAead_rfc4106_loaded = 0; #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream). #endif -#ifndef WC_AES_C_DYNAMIC_FALLBACK - #error LKCAPI registration of AES-XTS requires WC_AES_C_DYNAMIC_FALLBACK. +#ifndef WC_C_DYNAMIC_FALLBACK + #error LKCAPI registration of AES-XTS requires WC_C_DYNAMIC_FALLBACK. #endif struct km_AesXtsCtx { diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index b5cc2c918..17273b3fb 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -4575,12 +4575,21 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #endif /* WC_C_DYNAMIC_FALLBACK */ #ifdef WOLFSSL_AESNI - aes->use_aesni = 0; - if (checkedAESNI == 0) { - haveAESNI = Check_CPU_support_AES(); - checkedAESNI = 1; + +#if defined(WC_FLAG_DONT_USE_AESNI) + if (aes->use_aesni == WC_FLAG_DONT_USE_AESNI) { + aes->use_aesni = 0; } - if (haveAESNI) { + else +#endif + { + if (checkedAESNI == 0) { + haveAESNI = Check_CPU_support_AES(); + checkedAESNI = 1; + } + aes->use_aesni = haveAESNI; + } + if (aes->use_aesni) { #ifdef WOLFSSL_LINUXKM /* runtime alignment check */ if ((wc_ptr_t)&aes->key & (wc_ptr_t)0xf) { @@ -12993,6 +13002,10 @@ int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz, #ifdef WOLFSSL_AESNI +#if defined(USE_INTEL_SPEEDUP_FOR_AES) && !defined(USE_INTEL_SPEEDUP) + #define USE_INTEL_SPEEDUP +#endif + #if defined(USE_INTEL_SPEEDUP) #define HAVE_INTEL_AVX1 #define HAVE_INTEL_AVX2 diff --git a/wolfssl/wolfcrypt/aes.h b/wolfssl/wolfcrypt/aes.h index 127c2f97a..5fd0e63aa 100644 --- a/wolfssl/wolfcrypt/aes.h +++ b/wolfssl/wolfcrypt/aes.h @@ -303,6 +303,13 @@ struct Aes { #endif #ifdef WOLFSSL_AESNI byte use_aesni; + #if defined(WOLFSSL_LINUXKM) || defined(WC_WANT_FLAG_DONT_USE_AESNI) + /* Note, we can't support WC_FLAG_DONT_USE_AESNI by default because we + * need to support legacy applications that call wc_AesSetKey() on + * uninited struct Aes. + */ + #define WC_FLAG_DONT_USE_AESNI 2 + #endif #endif /* WOLFSSL_AESNI */ #if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) From 8092ff915ce348bb6652e90990d6c4b7836579a4 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Sat, 29 Mar 2025 00:26:31 -0500 Subject: [PATCH 4/6] linuxkm/lkcapi_glue.c: bring in wolfcrypt/src/misc.c for ForceZero if FIPS_VERSION3_LT(6,0,0). --- linuxkm/lkcapi_glue.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index 9d971571f..b48a26cd2 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -37,6 +37,16 @@ #include #endif +#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0) + /* need misc.c for ForceZero(). */ + #ifdef NO_INLINE + #include + #else + #define WOLFSSL_MISC_INCLUDED + #include + #endif +#endif + #ifndef WOLFSSL_LINUXKM_LKCAPI_PRIORITY /* Larger number means higher priority. The highest in-tree priority is 4001, * in the Cavium driver. From 8705d28d489237488e2f5322cb117e9b0203af01 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Tue, 1 Apr 2025 01:17:10 -0500 Subject: [PATCH 5/6] wolfcrypt/src/aes.c: in wc_AesSetKeyLocal(), rework support for WC_FLAG_DONT_USE_AESNI (fixes WC_C_DYNAMIC_FALLBACK). wolfssl/wolfcrypt/settings.h: in WOLFSSL_LINUXKM section, #ifdef LINUXKM_LKCAPI_REGISTER, #define WOLFSSL_TEST_SUBROUTINE to nothing, and #define WC_TEST_EXPORT_SUBTESTS. linuxkm/lkcapi_glue.c: * add check_skcipher_driver_masking() and check_aead_driver_masking(), * use _masking() checks in all linuxkm_test_*(). * add !WOLFSSL_AESGCM_STREAM implementation of linuxkm_test_aesgcm(). * add implementations of linuxkm_test_aesctr(), linuxkm_test_aesofb(), and linuxkm_test_aesecb() * remove incomplete+disabled AES-CCM shim implementation. linuxkm/module_hooks.c: pull in wolfcrypt/test/test.h if LINUXKM_LKCAPI_REGISTER. linuxkm/Makefile: build wolfcrypt/test/test.o if ENABLED_LINUXKM_LKCAPI_REGISTER. Makefile.am: add ENABLED_LINUXKM_LKCAPI_REGISTER to exports in BUILD_LINUXKM section. configure.ac: add AC_SUBST([ENABLED_LINUXKM_LKCAPI_REGISTER]); in ENABLED_LINUXKM_DEFAULTS set up, remove `-DWOLFSSL_TEST_SUBROUTINE=static` from AM_CFLAGS adds; fix whitespace. .wolfssl_known_macro_extras: add WC_WANT_FLAG_DONT_USE_AESNI. wolfcrypt/test/test.c: add `|| defined(WC_TEST_EXPORT_SUBTESTS)` to outermost gate, add wc_test_ prefix to render_error_message() and export it, wolfcrypt/test/test.h: add prototype for wc_test_render_error_message(), and #ifdef WC_TEST_EXPORT_SUBTESTS, add prototypes for all the subtests. --- .wolfssl_known_macro_extras | 1 + Makefile.am | 3 +- configure.ac | 7 +- linuxkm/Makefile | 2 + linuxkm/lkcapi_glue.c | 548 +++++++++-------------------------- linuxkm/module_hooks.c | 2 +- wolfcrypt/src/aes.c | 27 +- wolfcrypt/test/test.c | 12 +- wolfcrypt/test/test.h | 289 ++++++++++++++++++ wolfssl/wolfcrypt/settings.h | 9 +- 10 files changed, 458 insertions(+), 442 deletions(-) diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 33547245f..a0ab08c6b 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -547,6 +547,7 @@ WC_SHA384_DIGEST_SIZE WC_SHA512 WC_SSIZE_TYPE WC_STRICT_SIG +WC_WANT_FLAG_DONT_USE_AESNI WC_XMSS_FULL_HASH WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE WOLFSENTRY_H diff --git a/Makefile.am b/Makefile.am index d8e4b6ddf..ba768ec35 100644 --- a/Makefile.am +++ b/Makefile.am @@ -213,7 +213,8 @@ if BUILD_LINUXKM EXTRA_CFLAGS EXTRA_CPPFLAGS EXTRA_CCASFLAGS EXTRA_LDFLAGS \ AM_CPPFLAGS CPPFLAGS AM_CFLAGS CFLAGS \ AM_CCASFLAGS CCASFLAGS \ - src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_LINUXKM_PIE ENABLED_ASM \ + src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_LINUXKM_LKCAPI_REGISTER \ + ENABLED_LINUXKM_PIE ENABLED_ASM \ CFLAGS_FPU_DISABLE CFLAGS_FPU_ENABLE CFLAGS_SIMD_DISABLE CFLAGS_SIMD_ENABLE \ CFLAGS_AUTO_VECTORIZE_DISABLE CFLAGS_AUTO_VECTORIZE_ENABLE \ ASFLAGS_FPU_DISABLE_SIMD_ENABLE ASFLAGS_FPU_ENABLE_SIMD_DISABLE \ diff --git a/configure.ac b/configure.ac index 5b45e3cc2..b99c8ffb0 100644 --- a/configure.ac +++ b/configure.ac @@ -679,7 +679,7 @@ AC_SUBST([ENABLED_LINUXKM_BENCHMARKS]) if test "$ENABLED_LINUXKM_DEFAULTS" = "yes" then - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_DH_CONST -DWOLFSSL_SP_MOD_WORD_RP -DWOLFSSL_SP_DIV_64 -DWOLFSSL_SP_DIV_WORD_HALF -DWOLFSSL_SMALL_STACK_STATIC -DWOLFSSL_TEST_SUBROUTINE=static" + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_DH_CONST -DWOLFSSL_SP_MOD_WORD_RP -DWOLFSSL_SP_DIV_64 -DWOLFSSL_SP_DIV_WORD_HALF -DWOLFSSL_SMALL_STACK_STATIC" if test "$ENABLED_LINUXKM_PIE" = "yes"; then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_NO_OCSP_ISSUER_CHECK" fi @@ -3534,7 +3534,7 @@ then then AM_CFLAGS="$AM_CFLAGS -DUSE_INTEL_SPEEDUP" ENABLED_AESNI=yes - ENABLED_AESNI_WITH_AVX=yes + ENABLED_AESNI_WITH_AVX=yes elif test "$ENABLED_AESNI_WITH_AVX" = "yes" then AM_CFLAGS="$AM_CFLAGS -DUSE_INTEL_SPEEDUP_FOR_AES" @@ -9366,7 +9366,7 @@ then '-cfb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESCFB" ;; '-gcm(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESGCM" ;; '-rfc4106(gcm(aes))') - AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESGCM_RFC4106" ;; + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESGCM_RFC4106" ;; '-xts(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESXTS" ;; '-ctr(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESCTR" ;; '-ofb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESOFB" ;; @@ -9375,6 +9375,7 @@ then esac done fi +AC_SUBST([ENABLED_LINUXKM_LKCAPI_REGISTER]) # Library Suffix LIBSUFFIX="" diff --git a/linuxkm/Makefile b/linuxkm/Makefile index d673da6c3..ba6629c56 100644 --- a/linuxkm/Makefile +++ b/linuxkm/Makefile @@ -43,6 +43,8 @@ WOLFSSL_OBJ_FILES=$(patsubst %.lo, %.o, $(patsubst src/src_libwolfssl_la-%, src/ ifeq "$(ENABLED_CRYPT_TESTS)" "yes" WOLFSSL_OBJ_FILES+=wolfcrypt/test/test.o +else ifneq "$(ENABLED_LINUXKM_LKCAPI_REGISTER)" "none" + WOLFSSL_OBJ_FILES+=wolfcrypt/test/test.o else WOLFSSL_CFLAGS+=-DNO_CRYPT_TEST endif diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index b48a26cd2..c7ae88124 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -74,7 +74,6 @@ static int disable_setkey_warnings = 0; #define WOLFKM_AESCTR_NAME "ctr(aes)" #define WOLFKM_AESOFB_NAME "ofb(aes)" #define WOLFKM_AESECB_NAME "ecb(aes)" -#define WOLFKM_AESCCM_NAME "ccm(aes)" #ifdef WOLFSSL_AESNI #define WOLFKM_DRIVER_ISA_EXT "-aesni" @@ -107,7 +106,6 @@ static int disable_setkey_warnings = 0; #define WOLFKM_AESCTR_DRIVER ("ctr-aes" WOLFKM_DRIVER_SUFFIX) #define WOLFKM_AESOFB_DRIVER ("ofb-aes" WOLFKM_DRIVER_SUFFIX) #define WOLFKM_AESECB_DRIVER ("ecb-aes" WOLFKM_DRIVER_SUFFIX) -#define WOLFKM_AESCCM_DRIVER ("ccm-aes" WOLFKM_DRIVER_SUFFIX) #ifdef WOLFSSL_DEBUG_TRACE_ERROR_CODES enum linux_errcodes { @@ -187,17 +185,6 @@ static int disable_setkey_warnings = 0; #undef LINUXKM_LKCAPI_REGISTER_AESECB #endif -#ifdef notyet -#ifdef HAVE_AESCCM - #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCCM)) && \ - !defined(LINUXKM_LKCAPI_REGISTER_AESCCM) - #define LINUXKM_LKCAPI_REGISTER_AESCCM - #endif -#else - #undef LINUXKM_LKCAPI_REGISTER_AESCCM -#endif -#endif /* notyet */ - #ifdef LINUXKM_LKCAPI_REGISTER_AESCBC static int linuxkm_test_aescbc(void); #endif @@ -222,9 +209,6 @@ static int disable_setkey_warnings = 0; #ifdef LINUXKM_LKCAPI_REGISTER_AESECB static int linuxkm_test_aesecb(void); #endif -#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM - static int linuxkm_test_aesccm(void); -#endif /* km_AesX(): wrappers to wolfcrypt wc_AesX functions and * structures. */ @@ -248,6 +232,72 @@ static int disable_setkey_warnings = 0; #error WC_LINUXKM_C_FALLBACK_IN_SHIMS is defined but CAN_SAVE_VECTOR_REGISTERS is missing. #endif +WC_MAYBE_UNUSED static int check_skcipher_driver_masking(struct crypto_skcipher *tfm, const char *alg_name, const char *expected_driver_name) { +#ifdef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING + (void)tfm; (void)alg_name; (void)expected_driver_name; + return 0; +#else + const char *actual_driver_name; + int ret; + int alloced_tfm = 0; + + if (! tfm) { + alloced_tfm = 1; + tfm = crypto_alloc_skcipher(alg_name, 0, 0); + } + if (IS_ERR(tfm)) { + pr_err("error: allocating AES skcipher algorithm %s failed: %ld\n", + alg_name, PTR_ERR(tfm)); + return -EINVAL; + } + actual_driver_name = crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)); + if (strcmp(actual_driver_name, expected_driver_name)) { + pr_err("error: unexpected implementation for %s: %s (expected %s)\n", + alg_name, actual_driver_name, expected_driver_name); + ret = -ENOENT; + } else + ret = 0; + + if (alloced_tfm) + crypto_free_skcipher(tfm); + + return ret; +#endif +} + +WC_MAYBE_UNUSED static int check_aead_driver_masking(struct crypto_aead *tfm, const char *alg_name, const char *expected_driver_name) { +#ifdef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING + (void)tfm; (void)alg_name; (void)expected_driver_name; + return 0; +#else + const char *actual_driver_name; + int ret; + int alloced_tfm = 0; + + if (! tfm) { + alloced_tfm = 1; + tfm = crypto_alloc_aead(alg_name, 0, 0); + } + if (IS_ERR(tfm)) { + pr_err("error: allocating AES AEAD algorithm %s failed: %ld\n", + alg_name, PTR_ERR(tfm)); + return -EINVAL; + } + actual_driver_name = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); + if (strcmp(actual_driver_name, expected_driver_name)) { + pr_err("error: unexpected implementation for %s: %s (expected %s)\n", + alg_name, actual_driver_name, expected_driver_name); + ret = -ENOENT; + } else + ret = 0; + + if (alloced_tfm) + crypto_free_aead(tfm); + + return ret; +#endif +} + struct km_AesCtx { Aes *aes_encrypt; /* allocated in km_AesInitCommon() to assure * alignment, needed for AESNI. @@ -257,15 +307,8 @@ struct km_AesCtx { Aes *aes_encrypt_C; /* fallback if vector registers aren't available. */ Aes *aes_decrypt_C; #endif -#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309) - union { #ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 - byte rfc4106_nonce[4]; -#endif -#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 - byte rfc4309_nonce[3]; -#endif - }; + byte rfc4106_nonce[4]; #endif }; @@ -275,8 +318,7 @@ struct km_AesCtx { defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || \ defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESECB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309) + defined(LINUXKM_LKCAPI_REGISTER_AESECB) static void km_AesExitCommon(struct km_AesCtx * ctx); @@ -480,8 +522,7 @@ static void km_AesExitCommon(struct km_AesCtx * ctx) defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESECB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309) + defined(LINUXKM_LKCAPI_REGISTER_AESECB) static int km_AesSetKeyCommon(struct km_AesCtx * ctx, const u8 *in_key, unsigned int key_len, const char * name) @@ -558,15 +599,14 @@ static void km_AesExit(struct crypto_skcipher *tfm) * LINUXKM_LKCAPI_REGISTER_AESCFB || * LINUXKM_LKCAPI_REGISTER_AESCTR || * LINUXKM_LKCAPI_REGISTER_AESOFB || - * LINUXKM_LKCAPI_REGISTER_AESECB || - * LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 + * LINUXKM_LKCAPI_REGISTER_AESECB */ #endif /* LINUXKM_LKCAPI_REGISTER_AESCBC || * LINUXKM_LKCAPI_REGISTER_AESCFB || LINUXKM_LKCAPI_REGISTER_AESGCM || * LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 || * LINUXKM_LKCAPI_REGISTER_AESCTR || LINUXKM_LKCAPI_REGISTER_AESOFB || - * LINUXKM_LKCAPI_REGISTER_AESECB || LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 + * LINUXKM_LKCAPI_REGISTER_AESECB */ #ifdef LINUXKM_LKCAPI_REGISTER_AESCBC @@ -1330,7 +1370,7 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) if (decrypt_p) { err = wc_AesGcmDecrypt(aes_copy, out_text, in_text, req->cryptlen - tfm->authsize, #ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 - rfc4106_p ? rfc4106_iv : + rfc4106_p ? rfc4106_iv : #endif sk_walk.iv, GCM_NONCE_MID_SZ, authTag, tfm->authsize, @@ -1355,7 +1395,7 @@ static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) else { err = wc_AesGcmEncrypt(aes_copy, out_text, in_text, req->cryptlen, #ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 - rfc4106_p ? rfc4106_iv : + rfc4106_p ? rfc4106_iv : #endif sk_walk.iv, GCM_NONCE_MID_SZ, authTag, tfm->authsize, @@ -1473,8 +1513,8 @@ static int gcmAesAead_rfc4106_loaded = 0; #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream). #endif -#ifndef WC_C_DYNAMIC_FALLBACK - #error LKCAPI registration of AES-XTS requires WC_C_DYNAMIC_FALLBACK. +#if defined(WOLFSSL_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK) + #error LKCAPI registration of AES-XTS with AESNI requires WC_C_DYNAMIC_FALLBACK. #endif struct km_AesXtsCtx { @@ -1534,6 +1574,12 @@ static int km_AesXtsSetKey(struct crypto_skcipher *tfm, const u8 *in_key, return -EINVAL; } + /* It's possible to set ctx->aesXts->{tweak,aes,aes_decrypt}.use_aesni to + * WC_FLAG_DONT_USE_AESNI here, for WC_LINUXKM_C_FALLBACK_IN_SHIMS in + * AES-XTS, but we can use the WC_C_DYNAMIC_FALLBACK mechanism + * unconditionally because there's no AES-XTS in Cert 4718. + */ + return 0; } @@ -2280,342 +2326,6 @@ static int ecbAesAlg_loaded = 0; #endif /* LINUXKM_LKCAPI_REGISTER_AESECB */ -#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 - -static int km_AesCcmInit(struct crypto_aead * tfm) -{ - struct km_AesCtx * ctx = crypto_aead_ctx(tfm); - return km_AesInitCommon(ctx, WOLFKM_AESCCM_DRIVER, 0); -} - -static void km_AesCcmExit(struct crypto_aead * tfm) -{ - struct km_AesCtx * ctx = crypto_aead_ctx(tfm); - km_AesExitCommon(ctx); -} - -static int km_AesCcmSetKey_rfc4309(struct crypto_aead *tfm, const u8 *in_key, - unsigned int key_len) -{ - int err; - struct km_AesCtx * ctx = crypto_aead_ctx(tfm); - - - - err = wc_AesCcmSetKey(ctx->aes_encrypt, in_key, key_len); - - if (unlikely(err)) { - if (! disable_setkey_warnings) - pr_err("%s: wc_AesCcmSetKey failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - return -EINVAL; - } - - return 0; -} - -static int km_AesCcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize) -{ - (void)tfm; - - if (wc_AesCcmCheckTagSize((int)authsize) == 0) - return 0; - -#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG - pr_err("%s: invalid authsize: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), authsize); -#endif - return -EINVAL; -} - -/* - * aead ciphers receive data in scatterlists in following order: - * encrypt - * req->src: aad||plaintext - * req->dst: aad||ciphertext||tag - * decrypt - * req->src: aad||ciphertext||tag - * req->dst: aad||plaintext, return 0 or -EBADMSG - */ - -static int km_AesCcmEncrypt(struct aead_request *req) -{ - struct crypto_aead * tfm = NULL; - struct km_AesCtx * ctx = NULL; - struct skcipher_walk walk; - struct scatter_walk copy_walk; - u8 * copy_mem = NULL; - u8 authTag[WC_AES_BLOCK_SIZE]; - int err; - u8 * assoc = NULL; - Aes *aes_copy = NULL; - u8 * plaintext = NULL; - u8 * plaintext_copy = NULL; - u8 * ciphertext = NULL; - int iv_size; - - tfm = crypto_aead_reqtfm(req); - ctx = crypto_aead_ctx(tfm); - - err = skcipher_walk_aead_encrypt(&walk, req, false); - if (unlikely(err)) { - pr_err("%s: skcipher_walk_aead_encrypt failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - return -EINVAL; - } - - if (req->src->length >= req->assoclen + req->cryptlen && req->src->length) { - scatterwalk_start(©_walk, req->src); - assoc = scatterwalk_map(©_walk); - if (unlikely(IS_ERR(assoc))) { - pr_err("%s: scatterwalk_map failed: %ld\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), - PTR_ERR(assoc)); - goto out; - } - plaintext = assoc + req->assoclen; - ciphertext = ?; - } - else { - copy_mem = malloc(req->assoclen + req->cryptlen); - if (unlikely(copy_mem == NULL)) { - err = -ENOMEM; - goto out; - } - scatterwalk_map_and_copy(copy_mem, req->src, 0, req->assoclen + req->cryptlen, 0); - assoc = copy_mem; - plaintext = assoc + req->assoclen; - ciphertext = plaintext; - } - - - /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and - * aes->streamData. - */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) { - err = -ENOMEM; - goto out; - } - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - aes_copy->streamData = NULL; -#endif - - - -err = wc_AesCcmEncrypt(aes_copy, ciphertext, plaintext, req->cryptlen, - walk.iv, crypto_aead_ivsize(crypto_aead_tfm(tfm)), - authTag, sizeof authTag, - assoc, req->assoclen); - - if (unlikely(err)) { - pr_err("%s: wc_AesCcmEncrypt failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - err = skcipher_walk_done(&walk, 0); - - if (unlikely(err)) { - pr_err("%s: skcipher_walk_done failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - goto out; - } - } - - err = wc_AesCcmEncryptFinal(aes_copy, authTag, tfm->authsize); - if (unlikely(err)) { - pr_err("%s: wc_AesCcmEncryptFinal failed with return code %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - /* Now copy the auth tag into request scatterlist. */ - scatterwalk_map_and_copy(authTag, req->dst, - req->assoclen + req->cryptlen, - tfm->authsize, 1); - -out: - - if (assocmem) - free(assocmem); - else if (assoc) - scatterwalk_unmap(assoc); - - if (plaintext_copy) - free(plaintext_copy); - else if (plaintext) - scatterwalk_unmap(plaintext); - - if (aes_copy) { -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - free(aes_copy->streamData); -#endif - km_AesFree(&aes_copy); - } - - return err; -} - -static int km_AesCcmDecrypt(struct aead_request *req) -{ - struct crypto_aead * tfm = NULL; - struct km_AesCtx * ctx = NULL; - struct skcipher_walk walk; - struct scatter_walk assocSgWalk; - u8 origAuthTag[WC_AES_BLOCK_SIZE]; - int err; - u8 * assoc = NULL; - u8 * assocmem = NULL; - Aes *aes_copy; - - tfm = crypto_aead_reqtfm(req); - ctx = crypto_aead_ctx(tfm); - - /* Copy out original auth tag from req->src. */ - scatterwalk_map_and_copy(origAuthTag, req->src, - req->assoclen + req->cryptlen - tfm->authsize, - tfm->authsize, 0); - - err = skcipher_walk_aead_decrypt(&walk, req, false); - if (unlikely(err)) { - pr_err("%s: skcipher_walk_aead_decrypt failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - return err; - } - - /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and - * aes->streamData. - */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* GCM uses the same - * schedule for encrypt - * and decrypt. - */ -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - aes_copy->streamData = NULL; -#endif - - err = wc_AesCcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv, - GCM_NONCE_MID_SZ); - if (unlikely(err)) { - pr_err("%s: wc_AesCcmInit failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - if (req->src->length >= req->assoclen && req->src->length) { - scatterwalk_start(&assocSgWalk, req->src); - assoc = scatterwalk_map(&assocSgWalk); - if (unlikely(IS_ERR(assoc))) { - pr_err("%s: scatterwalk_map failed: %ld\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), - PTR_ERR(assoc)); - goto out; - } - } - else { - /* assoc can be any length, so if it's noncontiguous, we have to copy it - * to a contiguous heap allocation. - */ - assocmem = malloc(req->assoclen); - if (unlikely(assocmem == NULL)) { - err = -ENOMEM; - goto out; - } - assoc = assocmem; - scatterwalk_map_and_copy(assoc, req->src, 0, req->assoclen, 0); - } - - err = wc_AesCcmDecryptUpdate(aes_copy, NULL, NULL, 0, - assoc, req->assoclen); - - if (assocmem) - free(assocmem); - else - scatterwalk_unmap(assoc); - - if (unlikely(err)) { - pr_err("%s: wc_AesCcmDecryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - while (walk.nbytes) { - err = wc_AesCcmDecryptUpdate( - aes_copy, - walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes, - NULL, 0); - - if (unlikely(err)) { - pr_err("%s: wc_AesCcmDecryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - err = skcipher_walk_done(&walk, 0); - - if (unlikely(err)) { - pr_err("%s: skcipher_walk_done failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - goto out; - } - } - - err = wc_AesCcmDecryptFinal(aes_copy, origAuthTag, tfm->authsize); - if (unlikely(err)) { -#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG - pr_err("%s: wc_AesCcmDecryptFinal failed with return code %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); -#endif - if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) { - err = -EBADMSG; - goto out; - } - else { - err = -EINVAL; - goto out; - } - } - -out: - - km_AesFree(&aes_copy); - - return err; -} - -static struct aead_alg ccmAesAead = { - .base.cra_name = WOLFKM_AESCCM_NAME, - .base.cra_driver_name = WOLFKM_AESCCM_DRIVER, - .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct km_AesCtx), - .base.cra_module = THIS_MODULE, - .init = km_AesCcmInit, - .exit = km_AesCcmExit, - .setkey = km_AesCcmSetKey, - .setauthsize = km_AesCcmSetAuthsize, - .encrypt = km_AesCcmEncrypt, - .decrypt = km_AesCcmDecrypt, - .ivsize = CCM_NONCE_MID_SZ, - .maxauthsize = WC_AES_BLOCK_SIZE, - .chunksize = WC_AES_BLOCK_SIZE, -}; -static int ccmAesAead_loaded = 0; - -#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM */ - /* cipher tests, cribbed from test.c, with supplementary LKCAPI tests: */ #ifdef LINUXKM_LKCAPI_REGISTER_AESCBC @@ -2865,6 +2575,13 @@ static int linuxkm_test_aescfb(void) if (aes == NULL) return -ENOMEM; + ret = aesofb_test(); + if (ret) { + wc_test_render_error_message("aesgcm_test failed: ", ret); + ret = -EINVAL; + goto test_cfb_end; + } + XMEMSET(enc, 0, sizeof(enc)); XMEMSET(dec, 0, sizeof(enc)); @@ -2943,18 +2660,9 @@ static int linuxkm_test_aescfb(void) goto test_cfb_end; } -#ifndef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING - { - const char *driver_name = - crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)); - if (strcmp(driver_name, WOLFKM_AESCFB_DRIVER)) { - pr_err("error: unexpected implementation for %s: %s (expected %s)\n", - WOLFKM_AESCFB_NAME, driver_name, WOLFKM_AESCFB_DRIVER); - ret = -ENOENT; - goto test_cfb_end; - } - } -#endif + ret = check_skcipher_driver_masking(tfm, WOLFKM_AESCFB_NAME, WOLFKM_AESCFB_DRIVER); + if (ret) + goto test_cfb_end; ret = crypto_skcipher_setkey(tfm, key32, WC_AES_BLOCK_SIZE * 2); if (ret) { @@ -3029,7 +2737,13 @@ test_cfb_end: static int linuxkm_test_aesgcm(void) { #ifndef WOLFSSL_AESGCM_STREAM - return 0; + wc_test_ret_t ret = aesgcm_test(); + if (ret >= 0) + return check_aead_driver_masking(NULL /* tfm */, WOLFKM_AESGCM_NAME, WOLFKM_AESGCM_DRIVER); + else { + wc_test_render_error_message("aesgcm_test failed: ", ret); + return -EINVAL; + } #else int ret = 0; struct crypto_aead * tfm = NULL; @@ -3204,17 +2918,9 @@ static int linuxkm_test_aesgcm(void) goto test_gcm_end; } -#ifndef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING - { - const char *driver_name = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); - if (strcmp(driver_name, WOLFKM_AESGCM_DRIVER)) { - pr_err("error: unexpected implementation for %s: %s (expected %s)\n", - WOLFKM_AESGCM_NAME, driver_name, WOLFKM_AESGCM_DRIVER); - ret = -ENOENT; - goto test_gcm_end; - } - } -#endif + ret = check_aead_driver_masking(tfm, WOLFKM_AESGCM_NAME, WOLFKM_AESGCM_DRIVER); + if (ret) + goto test_gcm_end; ret = crypto_aead_setkey(tfm, key32, WC_AES_BLOCK_SIZE * 2); if (ret) { @@ -3320,7 +3026,13 @@ test_gcm_end: static int linuxkm_test_aesgcm_rfc4106(void) { - return 0; + wc_test_ret_t ret = aesgcm_test(); + if (ret >= 0) + return check_aead_driver_masking(NULL /* tfm */, WOLFKM_AESGCM_RFC4106_NAME, WOLFKM_AESGCM_RFC4106_DRIVER); + else { + wc_test_render_error_message("aesgcm_test failed: ", ret); + return -EINVAL; + } } #endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ @@ -4438,7 +4150,13 @@ out: #ifdef LINUXKM_LKCAPI_REGISTER_AESCTR static int linuxkm_test_aesctr(void) { - return 0; + wc_test_ret_t ret = aes_ctr_test(); + if (ret >= 0) + return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESCTR_NAME, WOLFKM_AESCTR_DRIVER); + else { + wc_test_render_error_message("aes_ctr_test failed: ", ret); + return -EINVAL; + } } #endif /* LINUXKM_LKCAPI_REGISTER_AESCTR */ @@ -4446,7 +4164,13 @@ static int linuxkm_test_aesctr(void) { #ifdef LINUXKM_LKCAPI_REGISTER_AESOFB static int linuxkm_test_aesofb(void) { - return 0; + wc_test_ret_t ret = aesofb_test(); + if (ret >= 0) + return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESOFB_NAME, WOLFKM_AESOFB_DRIVER); + else { + wc_test_render_error_message("aesofb_test failed: ", ret); + return -EINVAL; + } } #endif /* LINUXKM_LKCAPI_REGISTER_AESOFB */ @@ -4454,19 +4178,17 @@ static int linuxkm_test_aesofb(void) { #ifdef LINUXKM_LKCAPI_REGISTER_AESECB static int linuxkm_test_aesecb(void) { - return 0; + wc_test_ret_t ret = aes_test(); + if (ret >= 0) + return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESECB_NAME, WOLFKM_AESECB_DRIVER); + else { + wc_test_render_error_message("aes_test failed: ", ret); + return -EINVAL; + } } #endif /* LINUXKM_LKCAPI_REGISTER_AESECB */ -#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM - -static int linuxkm_test_aesccm(void) { - return 0; -} - -#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM */ - #endif /* !NO_AES */ static int linuxkm_lkcapi_register(void) @@ -4553,12 +4275,6 @@ static int linuxkm_lkcapi_register(void) REGISTER_ALG(ecbAesAlg, crypto_register_skcipher, linuxkm_test_aesecb); #endif -#ifdef notyet -#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM - REGISTER_ALG(ccmAesAead, crypto_register_aead, linuxkm_test_aesccm); -#endif -#endif /* notyet */ - #undef REGISTER_ALG out: @@ -4609,11 +4325,5 @@ static void linuxkm_lkcapi_unregister(void) UNREGISTER_ALG(ecbAesAlg, crypto_unregister_skcipher); #endif -#ifdef notyet -#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM - UNREGISTER_ALG(ccmAesAlg, crypto_unregister_aead); -#endif -#endif /* notyet */ - #undef UNREGISTER_ALG } diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index 8b431368e..32b1db9a2 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -45,7 +45,7 @@ #ifdef HAVE_FIPS #include #endif -#ifndef NO_CRYPT_TEST +#if !defined(NO_CRYPT_TEST) || defined(LINUXKM_LKCAPI_REGISTER) #include #endif #include diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 17273b3fb..c0e3c69b2 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -4575,21 +4575,23 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #endif /* WC_C_DYNAMIC_FALLBACK */ #ifdef WOLFSSL_AESNI - -#if defined(WC_FLAG_DONT_USE_AESNI) - if (aes->use_aesni == WC_FLAG_DONT_USE_AESNI) { - aes->use_aesni = 0; + if (checkedAESNI == 0) { + haveAESNI = Check_CPU_support_AES(); + checkedAESNI = 1; } - else + if (haveAESNI +#if defined(WC_FLAG_DONT_USE_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK) + && (aes->use_aesni != WC_FLAG_DONT_USE_AESNI) #endif + ) { - if (checkedAESNI == 0) { - haveAESNI = Check_CPU_support_AES(); - checkedAESNI = 1; +#if defined(WC_FLAG_DONT_USE_AESNI) + if (aes->use_aesni == WC_FLAG_DONT_USE_AESNI) { + aes->use_aesni = 0; + return 0; } - aes->use_aesni = haveAESNI; - } - if (aes->use_aesni) { +#endif + aes->use_aesni = 0; #ifdef WOLFSSL_LINUXKM /* runtime alignment check */ if ((wc_ptr_t)&aes->key & (wc_ptr_t)0xf) { @@ -4623,6 +4625,9 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #endif } } + else { + aes->use_aesni = 0; + } #endif /* WOLFSSL_AESNI */ #if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 9639e4096..1b7fec3db 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -41,7 +41,7 @@ #define WOLFSSL_DEBUG_TRACE_ERROR_CODES_ALWAYS #endif -#ifndef NO_CRYPT_TEST +#if !defined(NO_CRYPT_TEST) || defined(WC_TEST_EXPORT_SUBTESTS) #include #include @@ -832,7 +832,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_eax_test(void); /* Not all unexpected conditions are actually errors .*/ #define WARNING_OUT(err, eLabel) do { ret = (err); goto eLabel; } while (0) -static void render_error_message(const char* msg, wc_test_ret_t es) +void wc_test_render_error_message(const char* msg, wc_test_ret_t es) { (void)msg; (void)es; @@ -917,7 +917,7 @@ static THREAD_RETURN err_sys(const char* msg, int es) static wc_test_ret_t err_sys(const char* msg, wc_test_ret_t es) #endif { - render_error_message(msg, es); + wc_test_render_error_message(msg, es); print_fiducials(); #ifdef WOLFSSL_LINUXKM EXIT_TEST(es); @@ -1433,7 +1433,7 @@ static WOLFSSL_TEST_SUBROUTINE wc_test_ret_t nist_sp80056c_kdf_test(void) #endif #ifdef TEST_ALWAYS_RUN_TO_END - #define TEST_FAIL(msg, retval) do { last_failed_test_ret = (retval); render_error_message(msg, retval); } while (0) + #define TEST_FAIL(msg, retval) do { last_failed_test_ret = (retval); wc_test_render_error_message(msg, retval); } while (0) #elif !defined(TEST_FAIL) #define TEST_FAIL(msg, retval) return err_sys(msg, retval) #endif @@ -60700,8 +60700,8 @@ static void print_fiducials(void) { fiducial1, fiducial2, fiducial3, fiducial4); } -#else +#else /* NO_CRYPT_TEST && !WC_TEST_EXPORT_SUBTESTS */ #ifndef NO_MAIN_DRIVER int main(void) { return 0; } #endif -#endif /* NO_CRYPT_TEST */ +#endif /* NO_CRYPT_TEST && !WC_TEST_EXPORT_SUBTESTS */ diff --git a/wolfcrypt/test/test.h b/wolfcrypt/test/test.h index 7b2ececc9..bd4125b64 100644 --- a/wolfcrypt/test/test.h +++ b/wolfcrypt/test/test.h @@ -45,6 +45,8 @@ THREAD_RETURN WOLFSSL_THREAD wolfcrypt_test(void* args); wc_test_ret_t wolfcrypt_test(void* args); #endif +void wc_test_render_error_message(const char* msg, wc_test_ret_t es); + #ifndef NO_MAIN_DRIVER wc_test_ret_t wolfcrypt_test_main(int argc, char** argv); #endif @@ -100,6 +102,293 @@ wc_static_assert(-(long)MIN_CODE_E < 0x7ffL); #endif /* !WC_TEST_RET_HAVE_CUSTOM_MACROS */ +#ifdef WC_TEST_EXPORT_SUBTESTS + +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t error_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t base64_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t base16_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t asn_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t md2_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t md5_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t md4_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha224_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha256_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha512_test(void); +#if !defined(WOLFSSL_NOSHA512_224) && \ + (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5, 3)) && !defined(HAVE_SELFTEST) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha512_224_test(void); +#endif +#if !defined(WOLFSSL_NOSHA512_256) && \ + (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5, 3)) && !defined(HAVE_SELFTEST) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha512_256_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha384_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha3_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t shake128_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t shake256_test(void); +#ifdef WOLFSSL_SM3 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sm3_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hash_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_md5_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha224_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha256_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha384_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha512_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha3_test(void); +#if defined(HAVE_HKDF) && !defined(NO_HMAC) +#if defined(WOLFSSL_AFALG_XILINX) || defined(WOLFSSL_AFALG_XILINX_AES) || \ + defined(WOLFSSL_AFALG_XILINX_SHA3) || defined(WOLFSSL_AFALG_HASH_KEEP) || \ + defined(WOLFSSL_AFALG_XILINX_RSA) +/* hkdf_test has issue with extern WOLFSSL_TEST_SUBROUTINE set on Xilinx with afalg */ +static wc_test_ret_t hkdf_test(void); +#else +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hkdf_test(void); +#endif +#endif /* HAVE_HKDF && ! NO_HMAC */ +#ifdef WOLFSSL_HAVE_PRF +#if defined(HAVE_HKDF) && !defined(NO_HMAC) +#ifdef WOLFSSL_BASE16 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t tls12_kdf_test(void); +#endif /* WOLFSSL_BASE16 */ +#endif /* WOLFSSL_HAVE_HKDF && !NO_HMAC */ +#endif /* WOLFSSL_HAVE_PRF */ +#if defined(WOLFSSL_HAVE_PRF) && !defined(NO_HMAC) && defined(WOLFSSL_SHA384) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t prf_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sshkdf_test(void); +#ifdef WOLFSSL_TLS13 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t tls13_kdf_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t x963kdf_test(void); +#if defined(HAVE_HPKE) && defined(HAVE_ECC) && defined(HAVE_AESGCM) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hpke_test(void); +#endif +#ifdef WC_SRTP_KDF +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t srtpkdf_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t arc4_test(void); +#ifdef WC_RC2 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t rc2_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t chacha_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t XChaCha_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t chacha20_poly1305_aead_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t XChaCha20Poly1305_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t des_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t des3_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_cbc_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_ctr_test(void); +#if defined(WOLFSSL_AES_CFB) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_cfb_test(void); +#endif +#ifdef WOLFSSL_AES_XTS +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_xts_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes192_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes256_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aesofb_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t cmac_test(void); +#ifdef HAVE_ASCON +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ascon_hash256_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ascon_aead128_test(void); +#endif +#if defined(WOLFSSL_SIPHASH) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t siphash_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t poly1305_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aesgcm_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aesgcm_default_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t gmac_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aesccm_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aeskeywrap_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t camellia_test(void); +#ifdef WOLFSSL_SM4 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sm4_test(void); +#endif +#ifdef WC_RSA_NO_PADDING +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t rsa_no_pad_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t rsa_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dh_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t srp_test(void); +#ifndef WC_NO_RNG +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t random_test(void); +#endif /* WC_NO_RNG */ +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pwdbased_test(void); +#if defined(USE_CERT_BUFFERS_2048) && \ + defined(HAVE_PKCS12) && \ + !defined(NO_ASN) && !defined(NO_PWDBASED) && !defined(NO_HMAC) && \ + !defined(NO_CERTS) && !defined(NO_DES3) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs12_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ripemd_test(void); +#if defined(OPENSSL_EXTRA) && !defined(WOLFCRYPT_ONLY) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_test(void); /* test mini api */ + +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_pkey0_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_pkey1_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openSSL_evpMD_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_evpSig_test(void); +#endif + +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pbkdf1_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs12_pbkdf_test(void); +#if defined(HAVE_PBKDF2) && !defined(NO_SHA256) && !defined(NO_HMAC) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pbkdf2_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t scrypt_test(void); +#ifdef HAVE_ECC + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ecc_test(void); + #if defined(HAVE_ECC_ENCRYPT) && defined(HAVE_AES_CBC) && \ + (defined(WOLFSSL_AES_128) || defined(WOLFSSL_AES_256)) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ecc_encrypt_test(void); + #endif + #if defined(USE_CERT_BUFFERS_256) && !defined(WOLFSSL_ATECC508A) && \ + !defined(WOLFSSL_ATECC608A) && !defined(NO_ECC256) && \ + defined(HAVE_ECC_VERIFY) && defined(HAVE_ECC_SIGN) && \ + !defined(WOLF_CRYPTO_CB_ONLY_ECC) && !defined(NO_ECC_SECP) + /* skip for ATECC508/608A, cannot import private key buffers */ + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ecc_test_buffers(void); + #endif +#endif +#ifdef HAVE_CURVE25519 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t curve25519_test(void); +#endif +#ifdef HAVE_ED25519 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ed25519_test(void); +#endif +#ifdef HAVE_CURVE448 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t curve448_test(void); +#endif +#ifdef HAVE_ED448 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ed448_test(void); +#endif +#ifdef WOLFSSL_HAVE_MLKEM + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t mlkem_test(void); +#endif +#ifdef HAVE_DILITHIUM + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dilithium_test(void); +#endif +#if defined(WOLFSSL_HAVE_XMSS) + #if !defined(WOLFSSL_SMALL_STACK) && WOLFSSL_XMSS_MIN_HEIGHT <= 10 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t xmss_test_verify_only(void); + #endif + #if !defined(WOLFSSL_XMSS_VERIFY_ONLY) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t xmss_test(void); + #endif +#endif +#if defined(WOLFSSL_HAVE_LMS) + #if !defined(WOLFSSL_SMALL_STACK) + #if (defined(WOLFSSL_WC_LMS) && (LMS_MAX_HEIGHT >= 10) && \ + !defined(WOLFSSL_NO_LMS_SHA256_256)) || defined(HAVE_LIBLMS) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t lms_test_verify_only(void); + #endif + #endif + #if !defined(WOLFSSL_LMS_VERIFY_ONLY) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t lms_test(void); + #endif +#endif +#ifdef WOLFCRYPT_HAVE_ECCSI + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t eccsi_test(void); +#endif +#ifdef WOLFCRYPT_HAVE_SAKKE + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sakke_test(void); +#endif +#ifdef HAVE_BLAKE2 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t blake2b_test(void); +#endif +#ifdef HAVE_BLAKE2S + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t blake2s_test(void); +#endif +#ifdef HAVE_LIBZ + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t compress_test(void); +#endif +#ifdef HAVE_PKCS7 + #ifndef NO_PKCS7_ENCRYPTED_DATA + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7encrypted_test(void); + #endif + #if defined(HAVE_LIBZ) && !defined(NO_PKCS7_COMPRESSED_DATA) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7compressed_test(void); + #endif + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7signed_test(void); + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7enveloped_test(void); + #if defined(HAVE_AESGCM) || defined(HAVE_AESCCM) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7authenveloped_test(void); + #endif + #if !defined(NO_AES) && defined(HAVE_AES_CBC) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7callback_test(byte* cert, word32 certSz, byte* key, + word32 keySz); + #endif +#endif +#if !defined(NO_ASN_TIME) && !defined(NO_RSA) && defined(WOLFSSL_TEST_CERT) && \ + !defined(NO_FILESYSTEM) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t cert_test(void); +#endif +#if defined(WOLFSSL_CERT_EXT) && defined(WOLFSSL_TEST_CERT) && \ + !defined(NO_FILESYSTEM) && !defined(NO_RSA) && defined(WOLFSSL_GEN_CERT) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t certext_test(void); +#endif +#if defined(WOLFSSL_CERT_GEN_CACHE) && defined(WOLFSSL_TEST_CERT) && \ + defined(WOLFSSL_CERT_EXT) && defined(WOLFSSL_CERT_GEN) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t decodedCertCache_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void); +#if defined(WOLFSSL_PUBLIC_MP) && \ + ((defined(WOLFSSL_SP_MATH_ALL) && !defined(WOLFSSL_RSA_VERIFY_ONLY)) || \ + defined(USE_FAST_MATH)) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t mp_test(void); +#endif +#if defined(WOLFSSL_PUBLIC_MP) && defined(WOLFSSL_KEY_GEN) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t prime_test(void); +#endif +#if defined(ASN_BER_TO_DER) && \ + (defined(WOLFSSL_TEST_CERT) || defined(OPENSSL_EXTRA) || \ + defined(OPENSSL_EXTRA_X509_SMALL)) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t berder_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t logging_test(void); +#if !defined(NO_ASN) && !defined(NO_ASN_TIME) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t time_test(void); +#endif +#if defined(__INCLUDE_NUTTX_CONFIG_H) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t wolfcrypt_mutex_test(void); +#else +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t mutex_test(void); +#endif +#if defined(USE_WOLFSSL_MEMORY) && !defined(FREERTOS) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memcb_test(void); +#endif +#ifdef WOLFSSL_CAAM_BLOB +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t blob_test(void); +#endif +#ifdef HAVE_ARIA +#include "wolfssl/wolfcrypt/port/aria/aria-crypt.h" +void printOutput(const char *strName, unsigned char *data, unsigned int dataSz); +extern WOLFSSL_TEST_SUBROUTINE int ariagcm_test(MC_ALGID); +#endif + +#if defined(WOLF_CRYPTO_CB) && !defined(WC_TEST_NO_CRYPTOCB_SW_TEST) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t cryptocb_test(void); +#endif +#ifdef WOLFSSL_CERT_PIV +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t certpiv_test(void); +#endif +#ifdef WOLFSSL_AES_SIV +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_siv_test(void); +#endif + +#if defined(WOLFSSL_AES_EAX) && \ + (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5, 3)) && !defined(HAVE_SELFTEST) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_eax_test(void); +#endif /* WOLFSSL_AES_EAX */ + +#endif /* WC_TEST_EXPORT_SUBTESTS */ + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/wolfssl/wolfcrypt/settings.h b/wolfssl/wolfcrypt/settings.h index 6cfd33d46..855cb8d5c 100644 --- a/wolfssl/wolfcrypt/settings.h +++ b/wolfssl/wolfcrypt/settings.h @@ -3607,7 +3607,14 @@ extern void uITRON4_free(void *p) ; #define WOLFSSL_OLD_PRIME_CHECK #endif #ifndef WOLFSSL_TEST_SUBROUTINE - #define WOLFSSL_TEST_SUBROUTINE static + #ifdef LINUXKM_LKCAPI_REGISTER + #define WOLFSSL_TEST_SUBROUTINE + #else + #define WOLFSSL_TEST_SUBROUTINE static + #endif + #endif + #ifdef LINUXKM_LKCAPI_REGISTER + #define WC_TEST_EXPORT_SUBTESTS #endif #undef HAVE_PTHREAD /* linuxkm uses linux/string.h, included by linuxkm_wc_port.h. */ From 13c73a9691418613465f92f65ca195ba256fc2d7 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Wed, 2 Apr 2025 17:30:19 -0500 Subject: [PATCH 6/6] linuxkm/lkcapi_glue.c: add LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS and LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS helper macros (peer review suggestion). wolfcrypt/src/aes.c: add lengthy comment in software wc_AesSetKeyLocal() explaining the dynamics of aes->use_aesni (peer review suggestion), and in the !haveAESNI && WC_C_DYNAMIC_FALLBACK case, return with immediate success rather than following through to the redundant AesSetKey_C(). --- linuxkm/lkcapi_glue.c | 51 ++++++++++++++++++++++------------------- wolfcrypt/src/aes.c | 36 +++++++++++++++++++++++++++++ wolfssl/wolfcrypt/aes.h | 3 ++- 3 files changed, 65 insertions(+), 25 deletions(-) diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index c7ae88124..9bd437d76 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -210,6 +210,29 @@ static int disable_setkey_warnings = 0; static int linuxkm_test_aesecb(void); #endif +#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESECB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) + #define LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS +#endif + +#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESECB) + #define LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS +#endif + +#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) + #define LINUXKM_LKCAPI_REGISTER_AEADS +#endif + /* km_AesX(): wrappers to wolfcrypt wc_AesX functions and * structures. */ @@ -312,13 +335,7 @@ struct km_AesCtx { #endif }; -#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESECB) +#ifdef LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS static void km_AesExitCommon(struct km_AesCtx * ctx); @@ -518,11 +535,7 @@ static void km_AesExitCommon(struct km_AesCtx * ctx) #endif } -#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESECB) +#ifdef LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS static int km_AesSetKeyCommon(struct km_AesCtx * ctx, const u8 *in_key, unsigned int key_len, const char * name) @@ -595,19 +608,9 @@ static void km_AesExit(struct crypto_skcipher *tfm) km_AesExitCommon(ctx); } -#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC || - * LINUXKM_LKCAPI_REGISTER_AESCFB || - * LINUXKM_LKCAPI_REGISTER_AESCTR || - * LINUXKM_LKCAPI_REGISTER_AESOFB || - * LINUXKM_LKCAPI_REGISTER_AESECB - */ +#endif /* LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS */ -#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC || - * LINUXKM_LKCAPI_REGISTER_AESCFB || LINUXKM_LKCAPI_REGISTER_AESGCM || - * LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 || - * LINUXKM_LKCAPI_REGISTER_AESCTR || LINUXKM_LKCAPI_REGISTER_AESOFB || - * LINUXKM_LKCAPI_REGISTER_AESECB - */ +#endif /* LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS */ #ifdef LINUXKM_LKCAPI_REGISTER_AESCBC diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index c0e3c69b2..bb2c4160c 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -4575,6 +4575,36 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #endif /* WC_C_DYNAMIC_FALLBACK */ #ifdef WOLFSSL_AESNI + + /* The dynamics for determining whether AES-NI will be used are tricky. + * + * First, we check for CPU support and cache the result -- if AES-NI is + * missing, we always shortcut to the AesSetKey_C() path. + * + * Second, if the CPU supports AES-NI, we confirm on a per-call basis + * that it's safe to use in the caller context, using + * SAVE_VECTOR_REGISTERS2(). This is an always-true no-op in user-space + * builds, but has substantive logic behind it in kernel module builds. + * + * The outcome when SAVE_VECTOR_REGISTERS2() fails depends on + * WC_C_DYNAMIC_FALLBACK -- if that's defined, we return immediately with + * success but with AES-NI disabled (the earlier AesSetKey_C() allows + * future encrypt/decrypt calls to succeed), otherwise we fail. + * + * Upon successful return, aes->use_aesni will have a zero value if + * AES-NI is disabled, and a nonzero value if it's enabled. + * + * An additional, optional semantic is available via + * WC_FLAG_DONT_USE_AESNI, and is used in some kernel module builds to + * let the caller inhibit AES-NI. When this macro is defined, + * wc_AesInit() before wc_AesSetKey() is imperative, to avoid a read of + * uninitialized data in aes->use_aesni. That's why support for + * WC_FLAG_DONT_USE_AESNI must remain optional -- wc_AesInit() was only + * added in release 3.11.0, so legacy applications inevitably call + * wc_AesSetKey() on uninitialized Aes contexts. This must continue to + * function correctly with default build settings. + */ + if (checkedAESNI == 0) { haveAESNI = Check_CPU_support_AES(); checkedAESNI = 1; @@ -4627,6 +4657,12 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) } else { aes->use_aesni = 0; +#ifdef WC_C_DYNAMIC_FALLBACK + /* If WC_C_DYNAMIC_FALLBACK, we already called AesSetKey_C() + * above. + */ + return 0; +#endif } #endif /* WOLFSSL_AESNI */ diff --git a/wolfssl/wolfcrypt/aes.h b/wolfssl/wolfcrypt/aes.h index 5fd0e63aa..0ce8acc02 100644 --- a/wolfssl/wolfcrypt/aes.h +++ b/wolfssl/wolfcrypt/aes.h @@ -306,7 +306,8 @@ struct Aes { #if defined(WOLFSSL_LINUXKM) || defined(WC_WANT_FLAG_DONT_USE_AESNI) /* Note, we can't support WC_FLAG_DONT_USE_AESNI by default because we * need to support legacy applications that call wc_AesSetKey() on - * uninited struct Aes. + * uninited struct Aes. For details see the software implementation of + * wc_AesSetKeyLocal() (aes.c). */ #define WC_FLAG_DONT_USE_AESNI 2 #endif