diff --git a/.github/workflows/intelasm-c-fallback.yml b/.github/workflows/intelasm-c-fallback.yml index 33fb20d4e..49d3639bc 100644 --- a/.github/workflows/intelasm-c-fallback.yml +++ b/.github/workflows/intelasm-c-fallback.yml @@ -18,7 +18,7 @@ jobs: matrix: config: [ # Add new configs here - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy CPPFLAGS="-DNO_WOLFSSL_CIPHER_SUITE_TEST -DWC_AES_C_DYNAMIC_FALLBACK -DWC_C_DYNAMIC_FALLBACK -DDEBUG_VECTOR_REGISTER_ACCESS -DDEBUG_VECTOR_REGISTER_ACCESS_FUZZING -DWC_DEBUG_CIPHER_LIFECYCLE"' + '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy CPPFLAGS="-DNO_WOLFSSL_CIPHER_SUITE_TEST -DWC_C_DYNAMIC_FALLBACK -DDEBUG_VECTOR_REGISTER_ACCESS -DDEBUG_VECTOR_REGISTER_ACCESS_FUZZING -DWC_DEBUG_CIPHER_LIFECYCLE"' ] name: make check if: github.repository_owner == 'wolfssl' diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 814612350..0dc46fcab 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -548,6 +548,7 @@ WC_SHA384_DIGEST_SIZE WC_SHA512 WC_SSIZE_TYPE WC_STRICT_SIG +WC_WANT_FLAG_DONT_USE_AESNI WC_XMSS_FULL_HASH WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE WOLFSENTRY_H diff --git a/Makefile.am b/Makefile.am index d8e4b6ddf..ba768ec35 100644 --- a/Makefile.am +++ b/Makefile.am @@ -213,7 +213,8 @@ if BUILD_LINUXKM EXTRA_CFLAGS EXTRA_CPPFLAGS EXTRA_CCASFLAGS EXTRA_LDFLAGS \ AM_CPPFLAGS CPPFLAGS AM_CFLAGS CFLAGS \ AM_CCASFLAGS CCASFLAGS \ - src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_LINUXKM_PIE ENABLED_ASM \ + src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_LINUXKM_LKCAPI_REGISTER \ + ENABLED_LINUXKM_PIE ENABLED_ASM \ CFLAGS_FPU_DISABLE CFLAGS_FPU_ENABLE CFLAGS_SIMD_DISABLE CFLAGS_SIMD_ENABLE \ CFLAGS_AUTO_VECTORIZE_DISABLE CFLAGS_AUTO_VECTORIZE_ENABLE \ ASFLAGS_FPU_DISABLE_SIMD_ENABLE ASFLAGS_FPU_ENABLE_SIMD_DISABLE \ diff --git a/configure.ac b/configure.ac index bd8f3609d..62e336ec6 100644 --- a/configure.ac +++ b/configure.ac @@ -498,7 +498,8 @@ AS_CASE([$ENABLED_FIPS], [v5-dev],[ FIPS_VERSION="v5-dev" HAVE_FIPS_VERSION_MAJOR=5 - HAVE_FIPS_VERSION_MINOR=3 + HAVE_FIPS_VERSION_MINOR=2 + HAVE_FIPS_VERSION_PATCH=1 ENABLED_FIPS="yes" # for dev, DEF_SP_MATH and DEF_FAST_MATH follow non-FIPS defaults (currently sp-math-all) ], @@ -678,7 +679,7 @@ AC_SUBST([ENABLED_LINUXKM_BENCHMARKS]) if test "$ENABLED_LINUXKM_DEFAULTS" = "yes" then - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_DH_CONST -DWOLFSSL_SP_MOD_WORD_RP -DWOLFSSL_SP_DIV_64 -DWOLFSSL_SP_DIV_WORD_HALF -DWOLFSSL_SMALL_STACK_STATIC -DWOLFSSL_TEST_SUBROUTINE=static" + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_DH_CONST -DWOLFSSL_SP_MOD_WORD_RP -DWOLFSSL_SP_DIV_64 -DWOLFSSL_SP_DIV_WORD_HALF -DWOLFSSL_SMALL_STACK_STATIC" if test "$ENABLED_LINUXKM_PIE" = "yes"; then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_NO_OCSP_ISSUER_CHECK" fi @@ -1243,7 +1244,6 @@ then test "$enable_aesgcm" = "" && enable_aesgcm=yes test "$enable_aesccm" = "" && enable_aesccm=yes test "$enable_aesctr" = "" && enable_aesctr=yes - test "$enable_aeseax" = "" && enable_aeseax=yes test "$enable_aesofb" = "" && enable_aesofb=yes test "$enable_aescfb" = "" && enable_aescfb=yes test "$enable_aescbc_length_checks" = "" && enable_aescbc_length_checks=yes @@ -1318,6 +1318,8 @@ then test "$enable_ed25519_stream" = "" && test "$enable_ed25519" != "no" && enable_ed25519_stream=yes test "$enable_ed448" = "" && enable_ed448=yes test "$enable_ed448_stream" = "" && test "$enable_ed448" != "no" && enable_ed448_stream=yes + test "$enable_aessiv" = "" && enable_aessiv=yes + test "$enable_aeseax" = "" && enable_aeseax=yes if test "$ENABLED_LINUXKM_DEFAULTS" != "yes" then @@ -1331,7 +1333,6 @@ then test "$enable_aesgcm_stream" = "" && test "$enable_aesgcm" = "yes" && enable_aesgcm_stream=yes test "$enable_aesxts" = "" && enable_aesxts=yes test "$enable_aesxts_stream" = "" && test "$enable_aesxts" = "yes" && enable_aesxts_stream=yes - test "$enable_aessiv" = "" && enable_aessiv=yes test "$enable_shake128" = "" && enable_shake128=yes test "$enable_shake256" = "" && enable_shake256=yes test "$enable_compkey" = "" && test "$ENABLED_LINUXKM_DEFAULTS" != "yes" && enable_compkey=yes @@ -3507,6 +3508,13 @@ AC_ARG_ENABLE([aesni], [ ENABLED_AESNI=no ] ) +# INTEL AES-NI with AVX +AC_ARG_ENABLE([aesni-with-avx], + [AS_HELP_STRING([--enable-aesni-with-avx],[Enable AES-NI with additional AVX acceleration for AES (default: disabled)])], + [ ENABLED_AESNI_WITH_AVX=$enableval ], + [ ENABLED_AESNI_WITH_AVX=no ] + ) + # INTEL ASM AC_ARG_ENABLE([intelasm], [AS_HELP_STRING([--enable-intelasm],[Enable All Intel ASM speedups (default: disabled)])], @@ -3522,6 +3530,17 @@ then ENABLED_AESNI=yes fi + if test "$ENABLED_INTELASM" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DUSE_INTEL_SPEEDUP" + ENABLED_AESNI=yes + ENABLED_AESNI_WITH_AVX=yes + elif test "$ENABLED_AESNI_WITH_AVX" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DUSE_INTEL_SPEEDUP_FOR_AES" + ENABLED_AESNI=yes + fi + if test "$ENABLED_AESNI" = "yes" || test "$ENABLED_INTELASM" = "yes" then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_AESNI" @@ -3547,12 +3566,6 @@ then AS_IF([test "x$ENABLED_SM3" != "xno"],[AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SM3"]) fi - if test "$ENABLED_INTELASM" = "yes" - then - AM_CFLAGS="$AM_CFLAGS -DUSE_INTEL_SPEEDUP" - ENABLED_AESNI=yes - fi - if test "$host_cpu" = "x86_64" || test "$host_cpu" = "amd64" then AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_X86_64_BUILD" @@ -9320,7 +9333,7 @@ if test "$ENABLED_LINUXKM_LKCAPI_REGISTER" != "none" then AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER" - if test "$ENABLED_AESGCM" != "no" && test "$ENABLED_AESGCM_STREAM" = "no" && test "$ENABLED_FIPS" = "no"; then + if test "$ENABLED_AESGCM" != "no" && test "$ENABLED_AESGCM_STREAM" = "no" && test "$enable_aesgcm_stream" != "no" && (test "$ENABLED_FIPS" = "no" || test $HAVE_FIPS_VERSION -ge 6); then ENABLED_AESGCM_STREAM=yes fi @@ -9338,15 +9351,31 @@ then 'cfb(aes)') test "$ENABLED_AESCFB" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-CFB implementation not enabled.]) AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESCFB" ;; 'gcm(aes)') test "$ENABLED_AESGCM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-GCM implementation not enabled.]) - test "$ENABLED_AESGCM_STREAM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: --enable-aesgcm-stream is required for LKCAPI.]) AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESGCM" ;; + 'rfc4106(gcm(aes))') test "$ENABLED_AESGCM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-GCM implementation not enabled.]) + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106" ;; 'xts(aes)') test "$ENABLED_AESXTS" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-XTS implementation not enabled.]) test "$ENABLED_AESXTS_STREAM" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: --enable-aesxts-stream is required for LKCAPI.]) AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESXTS" ;; + 'ctr(aes)') test "$ENABLED_AESCTR" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-CTR implementation not enabled.]) + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESCTR" ;; + 'ofb(aes)') test "$ENABLED_AESOFB" != "no" || AC_MSG_ERROR([linuxkm-lkcapi-register ${lkcapi_alg}: AES-OFB implementation not enabled.]) + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESOFB" ;; + 'ecb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_REGISTER_AESECB -DHAVE_AES_ECB" ;; + '-cbc(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESCBC" ;; + '-cfb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESCFB" ;; + '-gcm(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESGCM" ;; + '-rfc4106(gcm(aes))') + AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESGCM_RFC4106" ;; + '-xts(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESXTS" ;; + '-ctr(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESCTR" ;; + '-ofb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESOFB" ;; + '-ecb(aes)') AM_CFLAGS="$AM_CFLAGS -DLINUXKM_LKCAPI_DONT_REGISTER_AESECB" ;; *) AC_MSG_ERROR([Unsupported LKCAPI algorithm "$lkcapi_alg".]) ;; esac done fi +AC_SUBST([ENABLED_LINUXKM_LKCAPI_REGISTER]) # Library Suffix LIBSUFFIX="" @@ -10821,6 +10850,7 @@ echo " * snifftest: $ENABLED_SNIFFTEST" echo " * ARC4: $ENABLED_ARC4" echo " * AES: $ENABLED_AES" echo " * AES-NI: $ENABLED_AESNI" +echo " * AVX for AES: $ENABLED_AESNI_WITH_AVX" echo " * AES-CBC: $ENABLED_AESCBC" echo " * AES-CBC length checks: $ENABLED_AESCBC_LENGTH_CHECKS" echo " * AES-GCM: $ENABLED_AESGCM" diff --git a/linuxkm/Makefile b/linuxkm/Makefile index d673da6c3..ba6629c56 100644 --- a/linuxkm/Makefile +++ b/linuxkm/Makefile @@ -43,6 +43,8 @@ WOLFSSL_OBJ_FILES=$(patsubst %.lo, %.o, $(patsubst src/src_libwolfssl_la-%, src/ ifeq "$(ENABLED_CRYPT_TESTS)" "yes" WOLFSSL_OBJ_FILES+=wolfcrypt/test/test.o +else ifneq "$(ENABLED_LINUXKM_LKCAPI_REGISTER)" "none" + WOLFSSL_OBJ_FILES+=wolfcrypt/test/test.o else WOLFSSL_CFLAGS+=-DNO_CRYPT_TEST endif diff --git a/linuxkm/lkcapi_glue.c b/linuxkm/lkcapi_glue.c index a98e0ee17..9bd437d76 100644 --- a/linuxkm/lkcapi_glue.c +++ b/linuxkm/lkcapi_glue.c @@ -30,8 +30,21 @@ * results FIPS vs non-FIPS. */ #if defined(CONFIG_CRYPTO_MANAGER) && \ - (defined(CONFIG_CRYPTO_FIPS) != defined(HAVE_FIPS)) -#error CONFIG_CRYPTO_MANAGER requires that CONFIG_CRYPTO_FIPS match HAVE_FIPS. + !defined(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) + #if defined(CONFIG_CRYPTO_FIPS) != defined(HAVE_FIPS) + #error CONFIG_CRYPTO_MANAGER requires that CONFIG_CRYPTO_FIPS match HAVE_FIPS. + #endif + #include +#endif + +#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0) + /* need misc.c for ForceZero(). */ + #ifdef NO_INLINE + #include + #else + #define WOLFSSL_MISC_INCLUDED + #include + #endif #endif #ifndef WOLFSSL_LINUXKM_LKCAPI_PRIORITY @@ -56,7 +69,11 @@ static int disable_setkey_warnings = 0; #define WOLFKM_AESCBC_NAME "cbc(aes)" #define WOLFKM_AESCFB_NAME "cfb(aes)" #define WOLFKM_AESGCM_NAME "gcm(aes)" +#define WOLFKM_AESGCM_RFC4106_NAME "rfc4106(gcm(aes))" #define WOLFKM_AESXTS_NAME "xts(aes)" +#define WOLFKM_AESCTR_NAME "ctr(aes)" +#define WOLFKM_AESOFB_NAME "ofb(aes)" +#define WOLFKM_AESECB_NAME "ecb(aes)" #ifdef WOLFSSL_AESNI #define WOLFKM_DRIVER_ISA_EXT "-aesni" @@ -84,7 +101,11 @@ static int disable_setkey_warnings = 0; #define WOLFKM_AESCBC_DRIVER ("cbc-aes" WOLFKM_DRIVER_SUFFIX) #define WOLFKM_AESCFB_DRIVER ("cfb-aes" WOLFKM_DRIVER_SUFFIX) #define WOLFKM_AESGCM_DRIVER ("gcm-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESGCM_RFC4106_DRIVER ("rfc4106-gcm-aes" WOLFKM_DRIVER_SUFFIX) #define WOLFKM_AESXTS_DRIVER ("xts-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESCTR_DRIVER ("ctr-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESOFB_DRIVER ("ofb-aes" WOLFKM_DRIVER_SUFFIX) +#define WOLFKM_AESECB_DRIVER ("ecb-aes" WOLFKM_DRIVER_SUFFIX) #ifdef WOLFSSL_DEBUG_TRACE_ERROR_CODES enum linux_errcodes { @@ -102,25 +123,114 @@ static int disable_setkey_warnings = 0; #define EBADMSG WC_ERR_TRACE(my_EBADMSG) #endif -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) -static int linuxkm_test_aescbc(void); +#ifdef HAVE_AES_CBC + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCBC)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESCBC) + #define LINUXKM_LKCAPI_REGISTER_AESCBC + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESCBC #endif -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) -static int linuxkm_test_aescfb(void); +#ifdef WOLFSSL_AES_CFB + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCFB)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESCFB) + #define LINUXKM_LKCAPI_REGISTER_AESCFB + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESCFB #endif -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) -static int linuxkm_test_aesgcm(void); +#ifdef HAVE_AESGCM + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESGCM)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESGCM) + #define LINUXKM_LKCAPI_REGISTER_AESGCM + #endif + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESGCM_RFC4106)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) + #define LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESGCM + #undef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 #endif -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) -static int linuxkm_test_aesxts(void); +#ifdef WOLFSSL_AES_XTS + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESXTS)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESXTS) + #define LINUXKM_LKCAPI_REGISTER_AESXTS + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESXTS +#endif +#ifdef WOLFSSL_AES_COUNTER + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCTR)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESCTR) + #define LINUXKM_LKCAPI_REGISTER_AESCTR + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESCTR +#endif +#ifdef WOLFSSL_AES_OFB + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESOFB)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESOFB) + #define LINUXKM_LKCAPI_REGISTER_AESOFB + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESOFB +#endif +#ifdef HAVE_AES_ECB + #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESECB)) && \ + !defined(LINUXKM_LKCAPI_REGISTER_AESECB) + #define LINUXKM_LKCAPI_REGISTER_AESECB + #endif +#else + #undef LINUXKM_LKCAPI_REGISTER_AESECB +#endif + +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC + static int linuxkm_test_aescbc(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB + static int linuxkm_test_aescfb(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM + static int linuxkm_test_aesgcm(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + static int linuxkm_test_aesgcm_rfc4106(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS + static int linuxkm_test_aesxts(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + static int linuxkm_test_aesctr(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + static int linuxkm_test_aesofb(void); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + static int linuxkm_test_aesecb(void); +#endif + +#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESECB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) + #define LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS +#endif + +#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESECB) + #define LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS +#endif + +#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) + #define LINUXKM_LKCAPI_REGISTER_AEADS #endif /* km_AesX(): wrappers to wolfcrypt wc_AesX functions and @@ -128,17 +238,104 @@ static int linuxkm_test_aesxts(void); #include +#if defined(WOLFSSL_AESNI) && \ + (!defined(WC_C_DYNAMIC_FALLBACK) || \ + (defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0))) && \ + !defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) + #define WC_LINUXKM_C_FALLBACK_IN_SHIMS +#elif !defined(WOLFSSL_AESNI) + #undef WC_LINUXKM_C_FALLBACK_IN_SHIMS +#endif + +#if defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && !defined(WC_FLAG_DONT_USE_AESNI) + #error WC_LINUXKM_C_FALLBACK_IN_SHIMS is defined but WC_FLAG_DONT_USE_AESNI is missing. +#endif + +#if defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && !defined(CAN_SAVE_VECTOR_REGISTERS) + #error WC_LINUXKM_C_FALLBACK_IN_SHIMS is defined but CAN_SAVE_VECTOR_REGISTERS is missing. +#endif + +WC_MAYBE_UNUSED static int check_skcipher_driver_masking(struct crypto_skcipher *tfm, const char *alg_name, const char *expected_driver_name) { +#ifdef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING + (void)tfm; (void)alg_name; (void)expected_driver_name; + return 0; +#else + const char *actual_driver_name; + int ret; + int alloced_tfm = 0; + + if (! tfm) { + alloced_tfm = 1; + tfm = crypto_alloc_skcipher(alg_name, 0, 0); + } + if (IS_ERR(tfm)) { + pr_err("error: allocating AES skcipher algorithm %s failed: %ld\n", + alg_name, PTR_ERR(tfm)); + return -EINVAL; + } + actual_driver_name = crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)); + if (strcmp(actual_driver_name, expected_driver_name)) { + pr_err("error: unexpected implementation for %s: %s (expected %s)\n", + alg_name, actual_driver_name, expected_driver_name); + ret = -ENOENT; + } else + ret = 0; + + if (alloced_tfm) + crypto_free_skcipher(tfm); + + return ret; +#endif +} + +WC_MAYBE_UNUSED static int check_aead_driver_masking(struct crypto_aead *tfm, const char *alg_name, const char *expected_driver_name) { +#ifdef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING + (void)tfm; (void)alg_name; (void)expected_driver_name; + return 0; +#else + const char *actual_driver_name; + int ret; + int alloced_tfm = 0; + + if (! tfm) { + alloced_tfm = 1; + tfm = crypto_alloc_aead(alg_name, 0, 0); + } + if (IS_ERR(tfm)) { + pr_err("error: allocating AES AEAD algorithm %s failed: %ld\n", + alg_name, PTR_ERR(tfm)); + return -EINVAL; + } + actual_driver_name = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); + if (strcmp(actual_driver_name, expected_driver_name)) { + pr_err("error: unexpected implementation for %s: %s (expected %s)\n", + alg_name, actual_driver_name, expected_driver_name); + ret = -ENOENT; + } else + ret = 0; + + if (alloced_tfm) + crypto_free_aead(tfm); + + return ret; +#endif +} + struct km_AesCtx { Aes *aes_encrypt; /* allocated in km_AesInitCommon() to assure * alignment, needed for AESNI. */ Aes *aes_decrypt; /* same. */ +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + Aes *aes_encrypt_C; /* fallback if vector registers aren't available. */ + Aes *aes_decrypt_C; +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + byte rfc4106_nonce[4]; +#endif }; -#if defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM) +#ifdef LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS static void km_AesExitCommon(struct km_AesCtx * ctx); @@ -154,7 +351,8 @@ static int km_AesInitCommon( if (! ctx->aes_encrypt) { pr_err("%s: allocation of %zu bytes for encryption key failed.\n", name, sizeof(*ctx->aes_encrypt)); - return MEMORY_E; + err = -MEMORY_E; + goto out; } err = wc_AesInit(ctx->aes_encrypt, NULL, INVALID_DEVID); @@ -163,53 +361,181 @@ static int km_AesInitCommon( pr_err("%s: wc_AesInit failed: %d\n", name, err); free(ctx->aes_encrypt); ctx->aes_encrypt = NULL; - return -EINVAL; + err = -EINVAL; + goto out; } if (! need_decryption) { ctx->aes_decrypt = NULL; - return 0; + } + else { + ctx->aes_decrypt = (Aes *)malloc(sizeof(*ctx->aes_decrypt)); + + if (! ctx->aes_decrypt) { + pr_err("%s: allocation of %zu bytes for decryption key failed.\n", + name, sizeof(*ctx->aes_decrypt)); + err = -MEMORY_E; + goto out; + } + + err = wc_AesInit(ctx->aes_decrypt, NULL, INVALID_DEVID); + + if (unlikely(err)) { + pr_err("%s: wc_AesInit failed: %d\n", name, err); + free(ctx->aes_decrypt); + ctx->aes_decrypt = NULL; + err = -EINVAL; + goto out; + } } - ctx->aes_decrypt = (Aes *)malloc(sizeof(*ctx->aes_decrypt)); +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS - if (! ctx->aes_decrypt) { - pr_err("%s: allocation of %zu bytes for decryption key failed.\n", - name, sizeof(*ctx->aes_decrypt)); - km_AesExitCommon(ctx); - return MEMORY_E; + ctx->aes_encrypt_C = (Aes *)malloc(sizeof(*ctx->aes_encrypt_C)); + + if (! ctx->aes_encrypt_C) { + pr_err("%s: allocation of %zu bytes for encryption key failed.\n", + name, sizeof(*ctx->aes_encrypt_C)); + err = -MEMORY_E; + goto out; } - err = wc_AesInit(ctx->aes_decrypt, NULL, INVALID_DEVID); + err = wc_AesInit(ctx->aes_encrypt_C, NULL, INVALID_DEVID); if (unlikely(err)) { pr_err("%s: wc_AesInit failed: %d\n", name, err); - free(ctx->aes_decrypt); - ctx->aes_decrypt = NULL; + free(ctx->aes_encrypt_C); + ctx->aes_encrypt_C = NULL; + err = -EINVAL; + goto out; + } + + if (! need_decryption) { + ctx->aes_decrypt_C = NULL; + } + else { + ctx->aes_decrypt_C = (Aes *)malloc(sizeof(*ctx->aes_decrypt_C)); + + if (! ctx->aes_decrypt_C) { + pr_err("%s: allocation of %zu bytes for decryption key failed.\n", + name, sizeof(*ctx->aes_decrypt_C)); + err = -MEMORY_E; + goto out; + } + + err = wc_AesInit(ctx->aes_decrypt_C, NULL, INVALID_DEVID); + + if (unlikely(err)) { + pr_err("%s: wc_AesInit failed: %d\n", name, err); + free(ctx->aes_decrypt_C); + ctx->aes_decrypt_C = NULL; + err = -EINVAL; + goto out; + } + } + +#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */ + +out: + + if (err != 0) km_AesExitCommon(ctx); - return -EINVAL; + + return err; +} + +static int km_AesGet(struct km_AesCtx *ctx, int decrypt_p, int copy_p, Aes **aes) { + Aes *ret; + +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + /* First, check if AESNI was disabled in the main SetKey for the requested + * direction. If so, use it (the fallback schedule won't even be inited). + */ + if (((! decrypt_p) || (! ctx->aes_decrypt)) && (! ctx->aes_encrypt->use_aesni)) + ret = ctx->aes_encrypt; + else if (decrypt_p && ctx->aes_decrypt && (! ctx->aes_decrypt->use_aesni)) + ret = ctx->aes_decrypt; + else if ( +#ifdef TEST_WC_LINUXKM_C_FALLBACK_IN_SHIMS + 1 +#else + ! CAN_SAVE_VECTOR_REGISTERS() +#endif + ) + { + if (decrypt_p && ctx->aes_decrypt_C) + ret = ctx->aes_decrypt_C; + else + ret = ctx->aes_encrypt_C; + if (ret->use_aesni) + return -EINVAL; + } + else +#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */ + { + if (decrypt_p && ctx->aes_decrypt) + ret = ctx->aes_decrypt; + else + ret = ctx->aes_encrypt; + } + + if (copy_p) { + /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and dynamic Aes.use_aesni. */ + Aes *aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ret, sizeof(Aes)); +#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) + aes_copy->streamData = NULL; +#endif + *aes = aes_copy; + } + else { + *aes = ret; } return 0; } +static void km_AesFree(Aes **aes) { + if ((! aes) || (! *aes)) + return; +#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0) + #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) + if ((*aes)->streamData) { + ForceZero((*aes)->streamData, 5 * AES_BLOCK_SIZE); + free((*aes)->streamData); + } + #endif +#endif + wc_AesFree(*aes); +#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0) + ForceZero(*aes, sizeof **aes); +#endif + free(*aes); + *aes = NULL; +} + static void km_AesExitCommon(struct km_AesCtx * ctx) { if (ctx->aes_encrypt) { - wc_AesFree(ctx->aes_encrypt); - free(ctx->aes_encrypt); - ctx->aes_encrypt = NULL; + km_AesFree(&ctx->aes_encrypt); } if (ctx->aes_decrypt) { - wc_AesFree(ctx->aes_decrypt); - free(ctx->aes_decrypt); - ctx->aes_decrypt = NULL; + km_AesFree(&ctx->aes_decrypt); } + +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + if (ctx->aes_encrypt_C) { + km_AesFree(&ctx->aes_encrypt_C); + } + if (ctx->aes_decrypt_C) { + km_AesFree(&ctx->aes_decrypt_C); + } +#endif } -#if defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB) +#ifdef LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS static int km_AesSetKeyCommon(struct km_AesCtx * ctx, const u8 *in_key, unsigned int key_len, const char * name) @@ -236,6 +562,43 @@ static int km_AesSetKeyCommon(struct km_AesCtx * ctx, const u8 *in_key, } } +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + + if (ctx->aes_encrypt->use_aesni) { + ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + err = wc_AesSetKey(ctx->aes_encrypt_C, in_key, key_len, NULL, AES_ENCRYPTION); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesSetKey for encryption key failed: %d\n", name, err); + return -EINVAL; + } + + if (ctx->aes_encrypt_C->use_aesni) + pr_err("%s: after wc_AesSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", name); + + } + + if (ctx->aes_decrypt_C && ctx->aes_decrypt->use_aesni) { + ctx->aes_decrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + err = wc_AesSetKey(ctx->aes_decrypt_C, in_key, key_len, NULL, + AES_DECRYPTION); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesSetKey for decryption key failed: %d\n", + name, err); + return -EINVAL; + } + + if (ctx->aes_decrypt_C->use_aesni) + pr_err("%s: after wc_AesSetKey, ctx->aes_decrypt_C has AES-NI asserted.\n", name); + } + +#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */ + return 0; } @@ -245,18 +608,11 @@ static void km_AesExit(struct crypto_skcipher *tfm) km_AesExitCommon(ctx); } -#endif /* LINUXKM_LKCAPI_REGISTER_ALL || - * LINUXKM_LKCAPI_REGISTER_AESCBC || - * LINUXKM_LKCAPI_REGISTER_AESCFB - */ +#endif /* LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS */ -#endif /* LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCBC || - * LINUXKM_LKCAPI_REGISTER_AESCFB || LINUXKM_LKCAPI_REGISTER_AESGCM - */ +#endif /* LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS */ -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC static int km_AesCbcInit(struct crypto_skcipher *tfm) { @@ -284,16 +640,14 @@ static int km_AesCbcEncrypt(struct skcipher_request *req) ctx = crypto_skcipher_ctx(tfm); err = skcipher_walk_virt(&walk, req, false); - if (unlikely(err)) { return err; } - /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + err = km_AesGet(ctx, 0 /* decrypt_p */, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } err = wc_AesSetIV(aes_copy, walk.iv); @@ -324,7 +678,7 @@ static int km_AesCbcEncrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -347,11 +701,10 @@ static int km_AesCbcDecrypt(struct skcipher_request *req) return err; } - /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_decrypt, sizeof(Aes)); + err = km_AesGet(ctx, 1 /* decrypt_p */, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } err = wc_AesSetIV(aes_copy, walk.iv); @@ -383,7 +736,7 @@ static int km_AesCbcDecrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -406,13 +759,9 @@ static struct skcipher_alg cbcAesAlg = { }; static int cbcAesAlg_loaded = 0; -#endif /* HAVE_AES_CBC && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCBC) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */ -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB static int km_AesCfbInit(struct crypto_skcipher *tfm) { @@ -446,11 +795,10 @@ static int km_AesCfbEncrypt(struct skcipher_request *req) return err; } - /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + err = km_AesGet(ctx, 0 /* decrypt_p */, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } err = wc_AesSetIV(aes_copy, walk.iv); @@ -486,7 +834,7 @@ static int km_AesCfbEncrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -510,14 +858,10 @@ static int km_AesCfbDecrypt(struct skcipher_request *req) return err; } - /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* CFB uses the same - * schedule for encrypt - * and decrypt. - */ + err = km_AesGet(ctx, 1 /* decrypt_p */, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } err = wc_AesSetIV(aes_copy, walk.iv); @@ -554,7 +898,7 @@ static int km_AesCfbDecrypt(struct skcipher_request *req) out: - free(aes_copy); + km_AesFree(&aes_copy); return err; } @@ -577,17 +921,10 @@ static struct skcipher_alg cfbAesAlg = { }; static int cfbAesAlg_loaded = 0; -#endif /* WOLFSSL_AES_CFB && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCBC) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */ -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) - -#ifndef WOLFSSL_AESGCM_STREAM - #error LKCAPI registration of AES-GCM requires WOLFSSL_AESGCM_STREAM (--enable-aesgcm-stream). -#endif +#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \ + defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) static int km_AesGcmInit(struct crypto_aead * tfm) { @@ -601,6 +938,8 @@ static void km_AesGcmExit(struct crypto_aead * tfm) km_AesExitCommon(ctx); } +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM + static int km_AesGcmSetKey(struct crypto_aead *tfm, const u8 *in_key, unsigned int key_len) { @@ -616,9 +955,76 @@ static int km_AesGcmSetKey(struct crypto_aead *tfm, const u8 *in_key, return -EINVAL; } +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + if (ctx->aes_encrypt->use_aesni) { + ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + err = wc_AesGcmSetKey(ctx->aes_encrypt_C, in_key, key_len); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesGcmSetKey failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return -EINVAL; + } + + if (ctx->aes_encrypt_C->use_aesni) + pr_err("%s: after wc_AesGcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESGCM_DRIVER); + } +#endif + return 0; } +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + +static int km_AesGcmSetKey_Rfc4106(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + int err; + struct km_AesCtx * ctx = crypto_aead_ctx(tfm); + + if (key_len < 4) + return -EINVAL; + key_len -= 4; + memcpy(ctx->rfc4106_nonce, in_key + key_len, 4); + + err = wc_AesGcmSetKey(ctx->aes_encrypt, in_key, key_len); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesGcmSetKey failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return -EINVAL; + } + +#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS + if (ctx->aes_encrypt->use_aesni) { + ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_AESNI; + + err = wc_AesGcmSetKey(ctx->aes_encrypt_C, in_key, key_len); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesGcmSetKey failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + return -EINVAL; + } + + if (ctx->aes_encrypt_C->use_aesni) + pr_err("%s: after wc_AesGcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESGCM_DRIVER); + } +#endif + + return 0; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM + static int km_AesGcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize) { (void)tfm; @@ -641,6 +1047,30 @@ static int km_AesGcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize) return -EINVAL; } +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + +static int km_AesGcmSetAuthsize_Rfc4106(struct crypto_aead *tfm, unsigned int authsize) +{ + (void)tfm; + + switch (authsize) { + case 8: + case 12: + case 16: + return 0; + } + +#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG + pr_err("%s: invalid authsize: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), authsize); +#endif + return -EINVAL; +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + /* * aead ciphers receive data in scatterlists in following order: * encrypt @@ -651,7 +1081,9 @@ static int km_AesGcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize) * req->dst: aad||plaintext, return 0 or -EBADMSG */ -static int km_AesGcmEncrypt(struct aead_request *req) +#ifdef WOLFSSL_AESGCM_STREAM + +static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) { struct crypto_aead * tfm = NULL; struct km_AesCtx * ctx = NULL; @@ -659,6 +1091,7 @@ static int km_AesGcmEncrypt(struct aead_request *req) struct scatter_walk assocSgWalk; u8 authTag[WC_AES_BLOCK_SIZE]; int err; + unsigned int assoclen = req->assoclen; u8 * assoc = NULL; u8 * assocmem = NULL; Aes *aes_copy; @@ -666,26 +1099,51 @@ static int km_AesGcmEncrypt(struct aead_request *req) tfm = crypto_aead_reqtfm(req); ctx = crypto_aead_ctx(tfm); - err = skcipher_walk_aead_encrypt(&walk, req, false); - if (unlikely(err)) { - pr_err("%s: skcipher_walk_aead_encrypt failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - return -EINVAL; + if (decrypt_p) { + /* Copy out original auth tag from req->src. */ + scatterwalk_map_and_copy(authTag, req->src, + req->assoclen + req->cryptlen - tfm->authsize, + tfm->authsize, 0); + err = skcipher_walk_aead_decrypt(&walk, req, false); + } + else { + err = skcipher_walk_aead_encrypt(&walk, req, false); } - /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and - * aes->streamData. - */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - aes_copy->streamData = NULL; -#endif + if (unlikely(err)) { + pr_err("%s: %s failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + decrypt_p ? "skcipher_walk_aead_decrypt" : "skcipher_walk_aead_encrypt", + err); + return err; + } - err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv, - GCM_NONCE_MID_SZ); + err = km_AesGet(ctx, decrypt_p, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { + return err; + } + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + if (rfc4106_p) { + byte rfc4106_iv[12]; + + if (unlikely(assoclen != 16 && assoclen != 20)) + return -EINVAL; + assoclen -= 8; + + memcpy(rfc4106_iv, ctx->rfc4106_nonce, 4); + memcpy(rfc4106_iv + 4, walk.iv, 8); + err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, rfc4106_iv, + GCM_NONCE_MID_SZ); + } + else +#else + (void)rfc4106_p; +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + { + err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv, + GCM_NONCE_MID_SZ); + } if (unlikely(err)) { pr_err("%s: wc_AesGcmInit failed: %d\n", crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); @@ -693,7 +1151,7 @@ static int km_AesGcmEncrypt(struct aead_request *req) goto out; } - if (req->src->length >= req->assoclen && req->src->length) { + if (req->src->length >= assoclen && req->src->length) { scatterwalk_start(&assocSgWalk, req->src); assoc = scatterwalk_map(&assocSgWalk); if (unlikely(IS_ERR(assoc))) { @@ -707,17 +1165,23 @@ static int km_AesGcmEncrypt(struct aead_request *req) /* assoc can be any length, so if it's noncontiguous, we have to copy it * to a contiguous heap allocation. */ - assocmem = malloc(req->assoclen); + assocmem = malloc(assoclen); if (unlikely(assocmem == NULL)) { err = -ENOMEM; goto out; } assoc = assocmem; - scatterwalk_map_and_copy(assoc, req->src, 0, req->assoclen, 0); + scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); } - err = wc_AesGcmEncryptUpdate(aes_copy, NULL, NULL, 0, - assoc, req->assoclen); + if (decrypt_p) { + err = wc_AesGcmDecryptUpdate(aes_copy, NULL, NULL, 0, + assoc, assoclen); + } + else { + err = wc_AesGcmEncryptUpdate(aes_copy, NULL, NULL, 0, + assoc, assoclen); + } if (assocmem) free(assocmem); @@ -725,23 +1189,37 @@ static int km_AesGcmEncrypt(struct aead_request *req) scatterwalk_unmap(assoc); if (unlikely(err)) { - pr_err("%s: wc_AesGcmEncryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + pr_err("%s: %s failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + decrypt_p ? "wc_AesGcmDecryptUpdate" : "wc_AesGcmEncryptUpdate", + err); err = -EINVAL; goto out; } while (walk.nbytes) { - err = wc_AesGcmEncryptUpdate( - aes_copy, - walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes, - NULL, 0); + if (decrypt_p) { + err = wc_AesGcmDecryptUpdate( + aes_copy, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes, + NULL, 0); + } + else { + err = wc_AesGcmEncryptUpdate( + aes_copy, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes, + NULL, 0); + } if (unlikely(err)) { - pr_err("%s: wc_AesGcmEncryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + pr_err("%s: %s failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + decrypt_p ? "wc_AesGcmDecryptUpdate" : "wc_AesGcmEncryptUpdate", + err); err = -EINVAL; goto out; } @@ -755,167 +1233,229 @@ static int km_AesGcmEncrypt(struct aead_request *req) } } - err = wc_AesGcmEncryptFinal(aes_copy, authTag, tfm->authsize); - if (unlikely(err)) { - pr_err("%s: wc_AesGcmEncryptFinal failed with return code %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; + if (decrypt_p) { + err = wc_AesGcmDecryptFinal(aes_copy, authTag, tfm->authsize); + if (unlikely(err)) { +#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG + pr_err("%s: wc_AesGcmDecryptFinal failed with return code %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); +#endif + if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) { + err = -EBADMSG; + goto out; + } + else { + err = -EINVAL; + goto out; + } + } + } + else { + err = wc_AesGcmEncryptFinal(aes_copy, authTag, tfm->authsize); + if (unlikely(err)) { + pr_err("%s: wc_AesGcmEncryptFinal failed with return code %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + /* Now copy the auth tag into request scatterlist. */ + scatterwalk_map_and_copy(authTag, req->dst, + req->assoclen + req->cryptlen, + tfm->authsize, 1); } - - /* Now copy the auth tag into request scatterlist. */ - scatterwalk_map_and_copy(authTag, req->dst, - req->assoclen + req->cryptlen, - tfm->authsize, 1); out: -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - free(aes_copy->streamData); -#endif - free(aes_copy); + km_AesFree(&aes_copy); return err; } -static int km_AesGcmDecrypt(struct aead_request *req) +#else /* !WOLFSSL_AESGCM_STREAM */ + +static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p) { struct crypto_aead * tfm = NULL; struct km_AesCtx * ctx = NULL; - struct skcipher_walk walk; - struct scatter_walk assocSgWalk; - u8 origAuthTag[WC_AES_BLOCK_SIZE]; + struct skcipher_walk sk_walk; + struct scatter_walk in_walk, out_walk; + u8 *in_map = NULL, *out_map = NULL; + u8 authTag[WC_AES_BLOCK_SIZE]; int err; + unsigned int assoclen = req->assoclen; u8 * assoc = NULL; - u8 * assocmem = NULL; + u8 * sg_buf = NULL; Aes *aes_copy; + u8 * in_text = NULL; + u8 * out_text = NULL; +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + byte rfc4106_iv[12]; +#endif tfm = crypto_aead_reqtfm(req); ctx = crypto_aead_ctx(tfm); - /* Copy out original auth tag from req->src. */ - scatterwalk_map_and_copy(origAuthTag, req->src, - req->assoclen + req->cryptlen - tfm->authsize, - tfm->authsize, 0); + if (decrypt_p) { + /* Copy out original auth tag from req->src. */ + scatterwalk_map_and_copy(authTag, req->src, + req->assoclen + req->cryptlen - tfm->authsize, + tfm->authsize, 0); + err = skcipher_walk_aead_decrypt(&sk_walk, req, false); + } + else { + err = skcipher_walk_aead_encrypt(&sk_walk, req, false); + } - err = skcipher_walk_aead_decrypt(&walk, req, false); if (unlikely(err)) { - pr_err("%s: skcipher_walk_aead_decrypt failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + pr_err("%s: %s failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + decrypt_p ? "skcipher_walk_aead_decrypt" : "skcipher_walk_aead_encrypt", + err); + return -EINVAL; + } + + err = km_AesGet(ctx, decrypt_p, 1 /* copy_p */, &aes_copy); + if (unlikely(err)) { return err; } - /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and - * aes->streamData. - */ - aes_copy = (struct Aes *)malloc(sizeof(Aes)); - if (aes_copy == NULL) - return -ENOMEM; - XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* GCM uses the same - * schedule for encrypt - * and decrypt. - */ -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - aes_copy->streamData = NULL; -#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + if (rfc4106_p) { + if (unlikely(assoclen != 16 && assoclen != 20)) + return -EINVAL; + assoclen -= 8; - err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv, - GCM_NONCE_MID_SZ); - if (unlikely(err)) { - pr_err("%s: wc_AesGcmInit failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; + memcpy(rfc4106_iv, ctx->rfc4106_nonce, 4); + memcpy(rfc4106_iv + 4, sk_walk.iv, 8); } +#else + (void)rfc4106_p; +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ - if (req->src->length >= req->assoclen && req->src->length) { - scatterwalk_start(&assocSgWalk, req->src); - assoc = scatterwalk_map(&assocSgWalk); - if (unlikely(IS_ERR(assoc))) { + if ((req->src->length >= req->assoclen + req->cryptlen) && + (req->dst->length >= req->assoclen + req->cryptlen)) + { + scatterwalk_start(&in_walk, req->src); + in_map = scatterwalk_map(&in_walk); + if (unlikely(IS_ERR(in_map))) { pr_err("%s: scatterwalk_map failed: %ld\n", crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), PTR_ERR(assoc)); goto out; } + assoc = in_map; + in_text = in_map + req->assoclen; + + scatterwalk_start(&out_walk, req->dst); + out_map = scatterwalk_map(&out_walk); + if (unlikely(IS_ERR(out_map))) { + pr_err("%s: scatterwalk_map failed: %ld\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), + PTR_ERR(assoc)); + goto out; + } + out_text = out_map + req->assoclen; } else { - /* assoc can be any length, so if it's noncontiguous, we have to copy it - * to a contiguous heap allocation. - */ - assocmem = malloc(req->assoclen); - if (unlikely(assocmem == NULL)) { + sg_buf = malloc(req->assoclen + req->cryptlen); + if (unlikely(sg_buf == NULL)) { err = -ENOMEM; goto out; } - assoc = assocmem; - scatterwalk_map_and_copy(assoc, req->src, 0, req->assoclen, 0); + if (decrypt_p) + scatterwalk_map_and_copy(sg_buf, req->src, 0, req->assoclen + req->cryptlen - tfm->authsize, 0); + else + scatterwalk_map_and_copy(sg_buf, req->src, 0, req->assoclen + req->cryptlen, 0); + assoc = sg_buf; + in_text = out_text = sg_buf + req->assoclen; } - err = wc_AesGcmDecryptUpdate(aes_copy, NULL, NULL, 0, - assoc, req->assoclen); - - if (assocmem) - free(assocmem); - else - scatterwalk_unmap(assoc); - - if (unlikely(err)) { - pr_err("%s: wc_AesGcmDecryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - while (walk.nbytes) { - err = wc_AesGcmDecryptUpdate( - aes_copy, - walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes, - NULL, 0); - - if (unlikely(err)) { - pr_err("%s: wc_AesGcmDecryptUpdate failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - err = -EINVAL; - goto out; - } - - err = skcipher_walk_done(&walk, 0); - - if (unlikely(err)) { - pr_err("%s: skcipher_walk_done failed: %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); - goto out; - } - } - - err = wc_AesGcmDecryptFinal(aes_copy, origAuthTag, tfm->authsize); - if (unlikely(err)) { -#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG - pr_err("%s: wc_AesGcmDecryptFinal failed with return code %d\n", - crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); + if (decrypt_p) { + err = wc_AesGcmDecrypt(aes_copy, out_text, in_text, req->cryptlen - tfm->authsize, +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + rfc4106_p ? rfc4106_iv : #endif - if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) { - err = -EBADMSG; - goto out; + sk_walk.iv, GCM_NONCE_MID_SZ, + authTag, tfm->authsize, + assoc, assoclen); + + if (unlikely(err)) { +#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG + pr_err("%s: wc_AesGcmDecrypt failed with return code %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); +#endif + + if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) { + err = -EBADMSG; + goto out; + } + else { + err = -EINVAL; + goto out; + } } - else { + } + else { + err = wc_AesGcmEncrypt(aes_copy, out_text, in_text, req->cryptlen, +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + rfc4106_p ? rfc4106_iv : +#endif + sk_walk.iv, GCM_NONCE_MID_SZ, + authTag, tfm->authsize, + assoc, assoclen); + + if (unlikely(err)) { + pr_err("%s: wc_AesGcmEncrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err); err = -EINVAL; goto out; } + + } + + if (sg_buf) { + if (decrypt_p) + scatterwalk_map_and_copy(sg_buf, req->dst, 0, req->assoclen + req->cryptlen - tfm->authsize, 1); + else + scatterwalk_map_and_copy(sg_buf, req->dst, 0, req->assoclen + req->cryptlen, 1); + } + + if (! decrypt_p) { + /* Now copy the auth tag into request scatterlist. */ + scatterwalk_map_and_copy(authTag, req->dst, + req->assoclen + req->cryptlen, + tfm->authsize, 1); } out: -#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI) - free(aes_copy->streamData); -#endif - free(aes_copy); + if (sg_buf) { + free(sg_buf); + } + else { + if (in_map) + scatterwalk_unmap(in_map); + if (out_map) + scatterwalk_unmap(out_map); + } + + km_AesFree(&aes_copy); return err; } +#endif /* !WOLFSSL_AESGCM_STREAM */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM + +static int km_AesGcmEncrypt(struct aead_request *req) { + return AesGcmCrypt_1(req, 0 /* decrypt_p */, 0 /* rfc4106_p */); +} + +static int km_AesGcmDecrypt(struct aead_request *req) { + return AesGcmCrypt_1(req, 1 /* decrypt_p */, 0 /* rfc4106_p */); +} + static struct aead_alg gcmAesAead = { .base.cra_name = WOLFKM_AESGCM_NAME, .base.cra_driver_name = WOLFKM_AESGCM_DRIVER, @@ -935,18 +1475,51 @@ static struct aead_alg gcmAesAead = { }; static int gcmAesAead_loaded = 0; -#endif /* HAVE_AESGCM && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESGCM) && - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */ -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + +static int km_AesGcmEncrypt_Rfc4106(struct aead_request *req) { + return AesGcmCrypt_1(req, 0 /* decrypt_p */, 1 /* rfc4106 */); +} + +static int km_AesGcmDecrypt_Rfc4106(struct aead_request *req) { + return AesGcmCrypt_1(req, 1 /* decrypt_p */, 1 /* rfc4106 */); +} + +static struct aead_alg gcmAesAead_rfc4106 = { + .base.cra_name = WOLFKM_AESGCM_RFC4106_NAME, + .base.cra_driver_name = WOLFKM_AESGCM_RFC4106_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesGcmInit, + .exit = km_AesGcmExit, + .setkey = km_AesGcmSetKey_Rfc4106, + .setauthsize = km_AesGcmSetAuthsize_Rfc4106, + .encrypt = km_AesGcmEncrypt_Rfc4106, + .decrypt = km_AesGcmDecrypt_Rfc4106, + .ivsize = 8, + .maxauthsize = WC_AES_BLOCK_SIZE, + .chunksize = WC_AES_BLOCK_SIZE, +}; +static int gcmAesAead_rfc4106_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM || LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS #ifndef WOLFSSL_AESXTS_STREAM #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream). #endif +#if defined(WOLFSSL_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK) + #error LKCAPI registration of AES-XTS with AESNI requires WC_C_DYNAMIC_FALLBACK. +#endif + struct km_AesXtsCtx { XtsAes *aesXts; /* allocated in km_AesXtsInitCommon() to assure alignment * for AESNI. @@ -1004,6 +1577,12 @@ static int km_AesXtsSetKey(struct crypto_skcipher *tfm, const u8 *in_key, return -EINVAL; } + /* It's possible to set ctx->aesXts->{tweak,aes,aes_decrypt}.use_aesni to + * WC_FLAG_DONT_USE_AESNI here, for WC_LINUXKM_C_FALLBACK_IN_SHIMS in + * AES-XTS, but we can use the WC_C_DYNAMIC_FALLBACK mechanism + * unconditionally because there's no AES-XTS in Cert 4718. + */ + return 0; } @@ -1292,15 +1871,467 @@ static struct skcipher_alg xtsAesAlg = { }; static int xtsAesAlg_loaded = 0; -#endif /* WOLFSSL_AES_XTS && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESXTS) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESXTS */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + +static int km_AesCtrInit(struct crypto_skcipher *tfm) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesInitCommon(ctx, WOLFKM_AESCTR_DRIVER, 0); +} + +static int km_AesCtrSetKey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESCTR_DRIVER); +} + +static int km_AesCtrEncrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + int err; + Aes *aes_copy; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_virt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + + err = wc_AesSetIV(aes_copy, walk.iv); + + if (unlikely(err)) { + pr_err("%s: wc_AesSetIV failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes != 0) { + err = wc_AesCtrEncrypt(aes_copy, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + + if (unlikely(err)) { + pr_err("%s: wc_AesCtrEncrypt failed %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + } + + /* copy iv from wolfCrypt back to walk.iv */ + XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE); + +out: + + km_AesFree(&aes_copy); + + return err; +} + +static int km_AesCtrDecrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + int err; + Aes *aes_copy; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_virt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* CTR uses the same + * schedule for encrypt + * and decrypt. + */ + + err = wc_AesSetIV(aes_copy, walk.iv); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesSetIV failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes != 0) { + /* CTR uses the same function for encrypt and decrypt. */ + err = wc_AesCtrEncrypt(aes_copy, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + + if (unlikely(err)) { + pr_err("%s: wc_AesCtrDecrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + goto out; + } + } + + /* copy iv from wolfCrypt back to walk.iv */ + XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE); + +out: + + km_AesFree(&aes_copy); + + return err; +} + +static struct skcipher_alg ctrAesAlg = { + .base.cra_name = WOLFKM_AESCTR_NAME, + .base.cra_driver_name = WOLFKM_AESCTR_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesCtrInit, + .exit = km_AesExit, + .min_keysize = AES_128_KEY_SIZE, + .max_keysize = AES_256_KEY_SIZE, + .ivsize = WC_AES_BLOCK_SIZE, + .setkey = km_AesCtrSetKey, + .encrypt = km_AesCtrEncrypt, + .decrypt = km_AesCtrDecrypt, +}; +static int ctrAesAlg_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESCTR */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + +static int km_AesOfbInit(struct crypto_skcipher *tfm) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesInitCommon(ctx, WOLFKM_AESOFB_DRIVER, 0); +} + +static int km_AesOfbSetKey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESOFB_DRIVER); +} + +static int km_AesOfbEncrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + int err; + Aes *aes_copy; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_virt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); + + err = wc_AesSetIV(aes_copy, walk.iv); + + if (unlikely(err)) { + pr_err("%s: wc_AesSetIV failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes != 0) { + err = wc_AesOfbEncrypt(aes_copy, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + + if (unlikely(err)) { + pr_err("%s: wc_AesOfbEncrypt failed %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + } + + /* copy iv from wolfCrypt back to walk.iv */ + XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE); + +out: + + km_AesFree(&aes_copy); + + return err; +} + +static int km_AesOfbDecrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + int err; + Aes *aes_copy; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_virt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + return err; + } + + /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */ + aes_copy = (struct Aes *)malloc(sizeof(Aes)); + if (aes_copy == NULL) + return -ENOMEM; + XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* OFB uses the same + * schedule for encrypt + * and decrypt. + */ + + err = wc_AesSetIV(aes_copy, walk.iv); + + if (unlikely(err)) { + if (! disable_setkey_warnings) + pr_err("%s: wc_AesSetIV failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + while (walk.nbytes != 0) { + err = wc_AesOfbDecrypt(aes_copy, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + + if (unlikely(err)) { + pr_err("%s: wc_AesOfbDecrypt failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + err = -EINVAL; + goto out; + } + + err = skcipher_walk_done(&walk, 0); + + if (unlikely(err)) { + pr_err("%s: skcipher_walk_done failed: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err); + goto out; + } + } + + /* copy iv from wolfCrypt back to walk.iv */ + XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE); + +out: + + km_AesFree(&aes_copy); + + return err; +} + +static struct skcipher_alg ofbAesAlg = { + .base.cra_name = WOLFKM_AESOFB_NAME, + .base.cra_driver_name = WOLFKM_AESOFB_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesOfbInit, + .exit = km_AesExit, + .min_keysize = AES_128_KEY_SIZE, + .max_keysize = AES_256_KEY_SIZE, + .ivsize = WC_AES_BLOCK_SIZE, + .setkey = km_AesOfbSetKey, + .encrypt = km_AesOfbEncrypt, + .decrypt = km_AesOfbDecrypt, +}; +static int ofbAesAlg_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESOFB */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + +static int km_AesEcbInit(struct crypto_skcipher *tfm) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesInitCommon(ctx, WOLFKM_AESECB_DRIVER, 1); +} + +static int km_AesEcbSetKey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm); + return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESECB_DRIVER); +} + +static int km_AesEcbEncrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + unsigned int nbytes = 0; + int err; + Aes *aes; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + return err; + } + + err = km_AesGet(ctx, 0 /* decrypt_p */, 0 /* copy_p */, &aes); + if (unlikely(err)) { + return err; + } + + while ((nbytes = walk.nbytes) != 0) { + err = wc_AesEcbEncrypt(aes, walk.dst.virt.addr, walk.src.virt.addr, + nbytes & (~(WC_AES_BLOCK_SIZE - 1))); + + if (unlikely(err)) { + pr_err("%s: wc_AesEcbEncrypt failed for %u bytes: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), nbytes, err); + err = -EINVAL; + goto out; + } + + nbytes &= WC_AES_BLOCK_SIZE - 1; + err = skcipher_walk_done(&walk, nbytes); + } + +out: + + return err; +} + +static int km_AesEcbDecrypt(struct skcipher_request *req) +{ + struct crypto_skcipher * tfm = NULL; + struct km_AesCtx * ctx = NULL; + struct skcipher_walk walk; + unsigned int nbytes = 0; + int err; + Aes *aes; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(err)) { + return err; + } + + err = km_AesGet(ctx, 1 /* decrypt_p */, 0 /* copy_p */, &aes); + if (unlikely(err)) { + return err; + } + + while ((nbytes = walk.nbytes) != 0) { + err = wc_AesEcbDecrypt(aes, walk.dst.virt.addr, walk.src.virt.addr, + nbytes & (~(WC_AES_BLOCK_SIZE - 1))); + + if (unlikely(err)) { + pr_err("%s: wc_AesEcbDecrypt failed for %u bytes: %d\n", + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), nbytes, err); + err = -EINVAL; + goto out; + } + + nbytes &= WC_AES_BLOCK_SIZE - 1; + err = skcipher_walk_done(&walk, nbytes); + } + +out: + + return err; +} + +static struct skcipher_alg ecbAesAlg = { + .base.cra_name = WOLFKM_AESECB_NAME, + .base.cra_driver_name = WOLFKM_AESECB_DRIVER, + .base.cra_priority = WOLFSSL_LINUXKM_LKCAPI_PRIORITY, + .base.cra_blocksize = WC_AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct km_AesCtx), + .base.cra_module = THIS_MODULE, + .init = km_AesEcbInit, + .exit = km_AesExit, + .min_keysize = AES_128_KEY_SIZE, + .max_keysize = AES_256_KEY_SIZE, + .ivsize = 0, + .setkey = km_AesEcbSetKey, + .encrypt = km_AesEcbEncrypt, + .decrypt = km_AesEcbDecrypt, +}; +static int ecbAesAlg_loaded = 0; + +#endif /* LINUXKM_LKCAPI_REGISTER_AESECB */ /* cipher tests, cribbed from test.c, with supplementary LKCAPI tests: */ -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC static int linuxkm_test_aescbc(void) { @@ -1502,13 +2533,9 @@ test_cbc_end: return ret; } -#endif /* HAVE_AES_CBC && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCBC) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */ -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB static int linuxkm_test_aescfb(void) { @@ -1551,6 +2578,13 @@ static int linuxkm_test_aescfb(void) if (aes == NULL) return -ENOMEM; + ret = aesofb_test(); + if (ret) { + wc_test_render_error_message("aesgcm_test failed: ", ret); + ret = -EINVAL; + goto test_cfb_end; + } + XMEMSET(enc, 0, sizeof(enc)); XMEMSET(dec, 0, sizeof(enc)); @@ -1629,18 +2663,9 @@ static int linuxkm_test_aescfb(void) goto test_cfb_end; } -#ifndef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING - { - const char *driver_name = - crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)); - if (strcmp(driver_name, WOLFKM_AESCFB_DRIVER)) { - pr_err("error: unexpected implementation for %s: %s (expected %s)\n", - WOLFKM_AESCFB_NAME, driver_name, WOLFKM_AESCFB_DRIVER); - ret = -ENOENT; - goto test_cfb_end; - } - } -#endif + ret = check_skcipher_driver_masking(tfm, WOLFKM_AESCFB_NAME, WOLFKM_AESCFB_DRIVER); + if (ret) + goto test_cfb_end; ret = crypto_skcipher_setkey(tfm, key32, WC_AES_BLOCK_SIZE * 2); if (ret) { @@ -1708,16 +2733,21 @@ test_cfb_end: return ret; } -#endif /* WOLFSSL_AES_CFB && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESCFB) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESCFB */ -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM static int linuxkm_test_aesgcm(void) { +#ifndef WOLFSSL_AESGCM_STREAM + wc_test_ret_t ret = aesgcm_test(); + if (ret >= 0) + return check_aead_driver_masking(NULL /* tfm */, WOLFKM_AESGCM_NAME, WOLFKM_AESGCM_DRIVER); + else { + wc_test_render_error_message("aesgcm_test failed: ", ret); + return -EINVAL; + } +#else int ret = 0; struct crypto_aead * tfm = NULL; struct aead_request * req = NULL; @@ -1891,17 +2921,9 @@ static int linuxkm_test_aesgcm(void) goto test_gcm_end; } -#ifndef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING - { - const char *driver_name = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); - if (strcmp(driver_name, WOLFKM_AESGCM_DRIVER)) { - pr_err("error: unexpected implementation for %s: %s (expected %s)\n", - WOLFKM_AESGCM_NAME, driver_name, WOLFKM_AESGCM_DRIVER); - ret = -ENOENT; - goto test_gcm_end; - } - } -#endif + ret = check_aead_driver_masking(tfm, WOLFKM_AESGCM_NAME, WOLFKM_AESGCM_DRIVER); + if (ret) + goto test_gcm_end; ret = crypto_aead_setkey(tfm, key32, WC_AES_BLOCK_SIZE * 2); if (ret) { @@ -1998,15 +3020,27 @@ test_gcm_end: free(aes); return ret; +#endif /* WOLFSSL_AESGCM_STREAM */ } -#endif /* HAVE_AESGCM && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESGCM) && - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */ -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + +static int linuxkm_test_aesgcm_rfc4106(void) +{ + wc_test_ret_t ret = aesgcm_test(); + if (ret >= 0) + return check_aead_driver_masking(NULL /* tfm */, WOLFKM_AESGCM_RFC4106_NAME, WOLFKM_AESGCM_RFC4106_DRIVER); + else { + wc_test_render_error_message("aesgcm_test failed: ", ret); + return -EINVAL; + } +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS /* test vectors from * http://csrc.nist.gov/groups/STM/cavp/block-cipher-modes.html @@ -3114,21 +4148,52 @@ out: return ret; } -#endif /* WOLFSSL_AES_XTS && - * (LINUXKM_LKCAPI_REGISTER_ALL || LINUXKM_LKCAPI_REGISTER_AESXTS) - */ +#endif /* LINUXKM_LKCAPI_REGISTER_AESXTS */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + +static int linuxkm_test_aesctr(void) { + wc_test_ret_t ret = aes_ctr_test(); + if (ret >= 0) + return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESCTR_NAME, WOLFKM_AESCTR_DRIVER); + else { + wc_test_render_error_message("aes_ctr_test failed: ", ret); + return -EINVAL; + } +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESCTR */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + +static int linuxkm_test_aesofb(void) { + wc_test_ret_t ret = aesofb_test(); + if (ret >= 0) + return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESOFB_NAME, WOLFKM_AESOFB_DRIVER); + else { + wc_test_render_error_message("aesofb_test failed: ", ret); + return -EINVAL; + } +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESOFB */ + +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + +static int linuxkm_test_aesecb(void) { + wc_test_ret_t ret = aes_test(); + if (ret >= 0) + return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESECB_NAME, WOLFKM_AESECB_DRIVER); + else { + wc_test_render_error_message("aes_test failed: ", ret); + return -EINVAL; + } +} + +#endif /* LINUXKM_LKCAPI_REGISTER_AESECB */ #endif /* !NO_AES */ -#if defined(HAVE_FIPS) && defined(CONFIG_CRYPTO_MANAGER) && \ - !defined(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) - #ifdef CONFIG_CRYPTO_FIPS - #include - #else - #error wolfCrypt FIPS with LINUXKM_LKCAPI_REGISTER and CONFIG_CRYPTO_MANAGER requires CONFIG_CRYPTO_FIPS - #endif -#endif - static int linuxkm_lkcapi_register(void) { int ret = 0; @@ -3188,33 +4253,30 @@ static int linuxkm_lkcapi_register(void) (alg).base.cra_priority); \ } while (0) -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC REGISTER_ALG(cbcAesAlg, crypto_register_skcipher, linuxkm_test_aescbc); #endif - -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB REGISTER_ALG(cfbAesAlg, crypto_register_skcipher, linuxkm_test_aescfb); #endif - -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM REGISTER_ALG(gcmAesAead, crypto_register_aead, linuxkm_test_aesgcm); #endif - -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + REGISTER_ALG(gcmAesAead_rfc4106, crypto_register_aead, linuxkm_test_aesgcm_rfc4106); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS REGISTER_ALG(xtsAesAlg, crypto_register_skcipher, linuxkm_test_aesxts); #endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + REGISTER_ALG(ctrAesAlg, crypto_register_skcipher, linuxkm_test_aesctr); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + REGISTER_ALG(ofbAesAlg, crypto_register_skcipher, linuxkm_test_aesofb); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + REGISTER_ALG(ecbAesAlg, crypto_register_skcipher, linuxkm_test_aesecb); +#endif #undef REGISTER_ALG @@ -3241,30 +4303,30 @@ static void linuxkm_lkcapi_unregister(void) } \ } while (0) -#if defined(HAVE_AES_CBC) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCBC)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC UNREGISTER_ALG(cbcAesAlg, crypto_unregister_skcipher); #endif -#if defined(WOLFSSL_AES_CFB) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESCFB)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB UNREGISTER_ALG(cfbAesAlg, crypto_unregister_skcipher); #endif -#if defined(HAVE_AESGCM) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESGCM)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM UNREGISTER_ALG(gcmAesAead, crypto_unregister_aead); #endif -#if defined(WOLFSSL_AES_XTS) && \ - (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \ - defined(LINUXKM_LKCAPI_REGISTER_AESXTS)) - +#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 + UNREGISTER_ALG(gcmAesAead_rfc4106, crypto_unregister_aead); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS UNREGISTER_ALG(xtsAesAlg, crypto_unregister_skcipher); #endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR + UNREGISTER_ALG(ctrAesAlg, crypto_unregister_skcipher); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB + UNREGISTER_ALG(ofbAesAlg, crypto_unregister_skcipher); +#endif +#ifdef LINUXKM_LKCAPI_REGISTER_AESECB + UNREGISTER_ALG(ecbAesAlg, crypto_unregister_skcipher); +#endif #undef UNREGISTER_ALG } diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index 8b431368e..32b1db9a2 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -45,7 +45,7 @@ #ifdef HAVE_FIPS #include #endif -#ifndef NO_CRYPT_TEST +#if !defined(NO_CRYPT_TEST) || defined(LINUXKM_LKCAPI_REGISTER) #include #endif #include diff --git a/src/include.am b/src/include.am index 9be3060fc..876ba7c65 100644 --- a/src/include.am +++ b/src/include.am @@ -72,10 +72,14 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else +if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif +if BUILD_AESXTS src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S endif endif +endif if BUILD_DES3 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/des3.c @@ -194,10 +198,14 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else +if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif +if BUILD_AESXTS src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S endif endif +endif if BUILD_RISCV_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/riscv/riscv-64-aes.c @@ -397,10 +405,14 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else +if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif +if BUILD_AESXTS src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S endif endif +endif if BUILD_SHA src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha.c @@ -1100,11 +1112,15 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else +if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif +if BUILD_AESXTS src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S endif endif endif +endif if BUILD_CAMELLIA src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/camellia.c diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index b5cc2c918..bb2c4160c 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -4575,12 +4575,53 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #endif /* WC_C_DYNAMIC_FALLBACK */ #ifdef WOLFSSL_AESNI - aes->use_aesni = 0; + + /* The dynamics for determining whether AES-NI will be used are tricky. + * + * First, we check for CPU support and cache the result -- if AES-NI is + * missing, we always shortcut to the AesSetKey_C() path. + * + * Second, if the CPU supports AES-NI, we confirm on a per-call basis + * that it's safe to use in the caller context, using + * SAVE_VECTOR_REGISTERS2(). This is an always-true no-op in user-space + * builds, but has substantive logic behind it in kernel module builds. + * + * The outcome when SAVE_VECTOR_REGISTERS2() fails depends on + * WC_C_DYNAMIC_FALLBACK -- if that's defined, we return immediately with + * success but with AES-NI disabled (the earlier AesSetKey_C() allows + * future encrypt/decrypt calls to succeed), otherwise we fail. + * + * Upon successful return, aes->use_aesni will have a zero value if + * AES-NI is disabled, and a nonzero value if it's enabled. + * + * An additional, optional semantic is available via + * WC_FLAG_DONT_USE_AESNI, and is used in some kernel module builds to + * let the caller inhibit AES-NI. When this macro is defined, + * wc_AesInit() before wc_AesSetKey() is imperative, to avoid a read of + * uninitialized data in aes->use_aesni. That's why support for + * WC_FLAG_DONT_USE_AESNI must remain optional -- wc_AesInit() was only + * added in release 3.11.0, so legacy applications inevitably call + * wc_AesSetKey() on uninitialized Aes contexts. This must continue to + * function correctly with default build settings. + */ + if (checkedAESNI == 0) { haveAESNI = Check_CPU_support_AES(); checkedAESNI = 1; } - if (haveAESNI) { + if (haveAESNI +#if defined(WC_FLAG_DONT_USE_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK) + && (aes->use_aesni != WC_FLAG_DONT_USE_AESNI) +#endif + ) + { +#if defined(WC_FLAG_DONT_USE_AESNI) + if (aes->use_aesni == WC_FLAG_DONT_USE_AESNI) { + aes->use_aesni = 0; + return 0; + } +#endif + aes->use_aesni = 0; #ifdef WOLFSSL_LINUXKM /* runtime alignment check */ if ((wc_ptr_t)&aes->key & (wc_ptr_t)0xf) { @@ -4614,6 +4655,15 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #endif } } + else { + aes->use_aesni = 0; +#ifdef WC_C_DYNAMIC_FALLBACK + /* If WC_C_DYNAMIC_FALLBACK, we already called AesSetKey_C() + * above. + */ + return 0; +#endif + } #endif /* WOLFSSL_AESNI */ #if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ @@ -12993,6 +13043,10 @@ int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz, #ifdef WOLFSSL_AESNI +#if defined(USE_INTEL_SPEEDUP_FOR_AES) && !defined(USE_INTEL_SPEEDUP) + #define USE_INTEL_SPEEDUP +#endif + #if defined(USE_INTEL_SPEEDUP) #define HAVE_INTEL_AVX1 #define HAVE_INTEL_AVX2 diff --git a/wolfcrypt/src/aes_xts_asm.S b/wolfcrypt/src/aes_xts_asm.S index 3d90cec33..7d9f0b989 100644 --- a/wolfcrypt/src/aes_xts_asm.S +++ b/wolfcrypt/src/aes_xts_asm.S @@ -41,7 +41,7 @@ #ifndef HAVE_INTEL_AVX1 #define HAVE_INTEL_AVX1 #endif /* HAVE_INTEL_AVX1 */ -#ifndef NO_AVX2_SUPPORT +#if !defined(NO_AVX2_SUPPORT) && !defined(HAVE_INTEL_AVX2) #define HAVE_INTEL_AVX2 #endif /* NO_AVX2_SUPPORT */ diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index f6167b1c2..1dc48bf54 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -41,7 +41,7 @@ #define WOLFSSL_DEBUG_TRACE_ERROR_CODES_ALWAYS #endif -#ifndef NO_CRYPT_TEST +#if !defined(NO_CRYPT_TEST) || defined(WC_TEST_EXPORT_SUBTESTS) #include #include @@ -832,7 +832,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_eax_test(void); /* Not all unexpected conditions are actually errors .*/ #define WARNING_OUT(err, eLabel) do { ret = (err); goto eLabel; } while (0) -static void render_error_message(const char* msg, wc_test_ret_t es) +void wc_test_render_error_message(const char* msg, wc_test_ret_t es) { (void)msg; (void)es; @@ -917,7 +917,7 @@ static THREAD_RETURN err_sys(const char* msg, int es) static wc_test_ret_t err_sys(const char* msg, wc_test_ret_t es) #endif { - render_error_message(msg, es); + wc_test_render_error_message(msg, es); print_fiducials(); #ifdef WOLFSSL_LINUXKM EXIT_TEST(es); @@ -1433,7 +1433,7 @@ static WOLFSSL_TEST_SUBROUTINE wc_test_ret_t nist_sp80056c_kdf_test(void) #endif #ifdef TEST_ALWAYS_RUN_TO_END - #define TEST_FAIL(msg, retval) do { last_failed_test_ret = (retval); render_error_message(msg, retval); } while (0) + #define TEST_FAIL(msg, retval) do { last_failed_test_ret = (retval); wc_test_render_error_message(msg, retval); } while (0) #elif !defined(TEST_FAIL) #define TEST_FAIL(msg, retval) return err_sys(msg, retval) #endif @@ -60779,8 +60779,8 @@ static void print_fiducials(void) { fiducial1, fiducial2, fiducial3, fiducial4); } -#else +#else /* NO_CRYPT_TEST && !WC_TEST_EXPORT_SUBTESTS */ #ifndef NO_MAIN_DRIVER int main(void) { return 0; } #endif -#endif /* NO_CRYPT_TEST */ +#endif /* NO_CRYPT_TEST && !WC_TEST_EXPORT_SUBTESTS */ diff --git a/wolfcrypt/test/test.h b/wolfcrypt/test/test.h index 7b2ececc9..bd4125b64 100644 --- a/wolfcrypt/test/test.h +++ b/wolfcrypt/test/test.h @@ -45,6 +45,8 @@ THREAD_RETURN WOLFSSL_THREAD wolfcrypt_test(void* args); wc_test_ret_t wolfcrypt_test(void* args); #endif +void wc_test_render_error_message(const char* msg, wc_test_ret_t es); + #ifndef NO_MAIN_DRIVER wc_test_ret_t wolfcrypt_test_main(int argc, char** argv); #endif @@ -100,6 +102,293 @@ wc_static_assert(-(long)MIN_CODE_E < 0x7ffL); #endif /* !WC_TEST_RET_HAVE_CUSTOM_MACROS */ +#ifdef WC_TEST_EXPORT_SUBTESTS + +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t error_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t base64_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t base16_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t asn_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t md2_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t md5_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t md4_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha224_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha256_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha512_test(void); +#if !defined(WOLFSSL_NOSHA512_224) && \ + (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5, 3)) && !defined(HAVE_SELFTEST) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha512_224_test(void); +#endif +#if !defined(WOLFSSL_NOSHA512_256) && \ + (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5, 3)) && !defined(HAVE_SELFTEST) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha512_256_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha384_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha3_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t shake128_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t shake256_test(void); +#ifdef WOLFSSL_SM3 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sm3_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hash_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_md5_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha224_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha256_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha384_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha512_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hmac_sha3_test(void); +#if defined(HAVE_HKDF) && !defined(NO_HMAC) +#if defined(WOLFSSL_AFALG_XILINX) || defined(WOLFSSL_AFALG_XILINX_AES) || \ + defined(WOLFSSL_AFALG_XILINX_SHA3) || defined(WOLFSSL_AFALG_HASH_KEEP) || \ + defined(WOLFSSL_AFALG_XILINX_RSA) +/* hkdf_test has issue with extern WOLFSSL_TEST_SUBROUTINE set on Xilinx with afalg */ +static wc_test_ret_t hkdf_test(void); +#else +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hkdf_test(void); +#endif +#endif /* HAVE_HKDF && ! NO_HMAC */ +#ifdef WOLFSSL_HAVE_PRF +#if defined(HAVE_HKDF) && !defined(NO_HMAC) +#ifdef WOLFSSL_BASE16 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t tls12_kdf_test(void); +#endif /* WOLFSSL_BASE16 */ +#endif /* WOLFSSL_HAVE_HKDF && !NO_HMAC */ +#endif /* WOLFSSL_HAVE_PRF */ +#if defined(WOLFSSL_HAVE_PRF) && !defined(NO_HMAC) && defined(WOLFSSL_SHA384) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t prf_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sshkdf_test(void); +#ifdef WOLFSSL_TLS13 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t tls13_kdf_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t x963kdf_test(void); +#if defined(HAVE_HPKE) && defined(HAVE_ECC) && defined(HAVE_AESGCM) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t hpke_test(void); +#endif +#ifdef WC_SRTP_KDF +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t srtpkdf_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t arc4_test(void); +#ifdef WC_RC2 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t rc2_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t chacha_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t XChaCha_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t chacha20_poly1305_aead_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t XChaCha20Poly1305_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t des_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t des3_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_cbc_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_ctr_test(void); +#if defined(WOLFSSL_AES_CFB) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_cfb_test(void); +#endif +#ifdef WOLFSSL_AES_XTS +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_xts_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes192_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes256_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aesofb_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t cmac_test(void); +#ifdef HAVE_ASCON +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ascon_hash256_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ascon_aead128_test(void); +#endif +#if defined(WOLFSSL_SIPHASH) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t siphash_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t poly1305_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aesgcm_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aesgcm_default_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t gmac_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aesccm_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aeskeywrap_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t camellia_test(void); +#ifdef WOLFSSL_SM4 +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sm4_test(void); +#endif +#ifdef WC_RSA_NO_PADDING +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t rsa_no_pad_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t rsa_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dh_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t srp_test(void); +#ifndef WC_NO_RNG +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t random_test(void); +#endif /* WC_NO_RNG */ +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pwdbased_test(void); +#if defined(USE_CERT_BUFFERS_2048) && \ + defined(HAVE_PKCS12) && \ + !defined(NO_ASN) && !defined(NO_PWDBASED) && !defined(NO_HMAC) && \ + !defined(NO_CERTS) && !defined(NO_DES3) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs12_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ripemd_test(void); +#if defined(OPENSSL_EXTRA) && !defined(WOLFCRYPT_ONLY) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_test(void); /* test mini api */ + +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_pkey0_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_pkey1_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openSSL_evpMD_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_evpSig_test(void); +#endif + +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pbkdf1_test(void); +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs12_pbkdf_test(void); +#if defined(HAVE_PBKDF2) && !defined(NO_SHA256) && !defined(NO_HMAC) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pbkdf2_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t scrypt_test(void); +#ifdef HAVE_ECC + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ecc_test(void); + #if defined(HAVE_ECC_ENCRYPT) && defined(HAVE_AES_CBC) && \ + (defined(WOLFSSL_AES_128) || defined(WOLFSSL_AES_256)) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ecc_encrypt_test(void); + #endif + #if defined(USE_CERT_BUFFERS_256) && !defined(WOLFSSL_ATECC508A) && \ + !defined(WOLFSSL_ATECC608A) && !defined(NO_ECC256) && \ + defined(HAVE_ECC_VERIFY) && defined(HAVE_ECC_SIGN) && \ + !defined(WOLF_CRYPTO_CB_ONLY_ECC) && !defined(NO_ECC_SECP) + /* skip for ATECC508/608A, cannot import private key buffers */ + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ecc_test_buffers(void); + #endif +#endif +#ifdef HAVE_CURVE25519 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t curve25519_test(void); +#endif +#ifdef HAVE_ED25519 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ed25519_test(void); +#endif +#ifdef HAVE_CURVE448 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t curve448_test(void); +#endif +#ifdef HAVE_ED448 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ed448_test(void); +#endif +#ifdef WOLFSSL_HAVE_MLKEM + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t mlkem_test(void); +#endif +#ifdef HAVE_DILITHIUM + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dilithium_test(void); +#endif +#if defined(WOLFSSL_HAVE_XMSS) + #if !defined(WOLFSSL_SMALL_STACK) && WOLFSSL_XMSS_MIN_HEIGHT <= 10 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t xmss_test_verify_only(void); + #endif + #if !defined(WOLFSSL_XMSS_VERIFY_ONLY) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t xmss_test(void); + #endif +#endif +#if defined(WOLFSSL_HAVE_LMS) + #if !defined(WOLFSSL_SMALL_STACK) + #if (defined(WOLFSSL_WC_LMS) && (LMS_MAX_HEIGHT >= 10) && \ + !defined(WOLFSSL_NO_LMS_SHA256_256)) || defined(HAVE_LIBLMS) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t lms_test_verify_only(void); + #endif + #endif + #if !defined(WOLFSSL_LMS_VERIFY_ONLY) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t lms_test(void); + #endif +#endif +#ifdef WOLFCRYPT_HAVE_ECCSI + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t eccsi_test(void); +#endif +#ifdef WOLFCRYPT_HAVE_SAKKE + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sakke_test(void); +#endif +#ifdef HAVE_BLAKE2 + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t blake2b_test(void); +#endif +#ifdef HAVE_BLAKE2S + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t blake2s_test(void); +#endif +#ifdef HAVE_LIBZ + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t compress_test(void); +#endif +#ifdef HAVE_PKCS7 + #ifndef NO_PKCS7_ENCRYPTED_DATA + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7encrypted_test(void); + #endif + #if defined(HAVE_LIBZ) && !defined(NO_PKCS7_COMPRESSED_DATA) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7compressed_test(void); + #endif + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7signed_test(void); + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7enveloped_test(void); + #if defined(HAVE_AESGCM) || defined(HAVE_AESCCM) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7authenveloped_test(void); + #endif + #if !defined(NO_AES) && defined(HAVE_AES_CBC) + extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs7callback_test(byte* cert, word32 certSz, byte* key, + word32 keySz); + #endif +#endif +#if !defined(NO_ASN_TIME) && !defined(NO_RSA) && defined(WOLFSSL_TEST_CERT) && \ + !defined(NO_FILESYSTEM) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t cert_test(void); +#endif +#if defined(WOLFSSL_CERT_EXT) && defined(WOLFSSL_TEST_CERT) && \ + !defined(NO_FILESYSTEM) && !defined(NO_RSA) && defined(WOLFSSL_GEN_CERT) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t certext_test(void); +#endif +#if defined(WOLFSSL_CERT_GEN_CACHE) && defined(WOLFSSL_TEST_CERT) && \ + defined(WOLFSSL_CERT_EXT) && defined(WOLFSSL_CERT_GEN) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t decodedCertCache_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void); +#if defined(WOLFSSL_PUBLIC_MP) && \ + ((defined(WOLFSSL_SP_MATH_ALL) && !defined(WOLFSSL_RSA_VERIFY_ONLY)) || \ + defined(USE_FAST_MATH)) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t mp_test(void); +#endif +#if defined(WOLFSSL_PUBLIC_MP) && defined(WOLFSSL_KEY_GEN) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t prime_test(void); +#endif +#if defined(ASN_BER_TO_DER) && \ + (defined(WOLFSSL_TEST_CERT) || defined(OPENSSL_EXTRA) || \ + defined(OPENSSL_EXTRA_X509_SMALL)) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t berder_test(void); +#endif +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t logging_test(void); +#if !defined(NO_ASN) && !defined(NO_ASN_TIME) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t time_test(void); +#endif +#if defined(__INCLUDE_NUTTX_CONFIG_H) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t wolfcrypt_mutex_test(void); +#else +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t mutex_test(void); +#endif +#if defined(USE_WOLFSSL_MEMORY) && !defined(FREERTOS) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memcb_test(void); +#endif +#ifdef WOLFSSL_CAAM_BLOB +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t blob_test(void); +#endif +#ifdef HAVE_ARIA +#include "wolfssl/wolfcrypt/port/aria/aria-crypt.h" +void printOutput(const char *strName, unsigned char *data, unsigned int dataSz); +extern WOLFSSL_TEST_SUBROUTINE int ariagcm_test(MC_ALGID); +#endif + +#if defined(WOLF_CRYPTO_CB) && !defined(WC_TEST_NO_CRYPTOCB_SW_TEST) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t cryptocb_test(void); +#endif +#ifdef WOLFSSL_CERT_PIV +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t certpiv_test(void); +#endif +#ifdef WOLFSSL_AES_SIV +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_siv_test(void); +#endif + +#if defined(WOLFSSL_AES_EAX) && \ + (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5, 3)) && !defined(HAVE_SELFTEST) +extern WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_eax_test(void); +#endif /* WOLFSSL_AES_EAX */ + +#endif /* WC_TEST_EXPORT_SUBTESTS */ + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/wolfssl/wolfcrypt/aes.h b/wolfssl/wolfcrypt/aes.h index 127c2f97a..0ce8acc02 100644 --- a/wolfssl/wolfcrypt/aes.h +++ b/wolfssl/wolfcrypt/aes.h @@ -303,6 +303,14 @@ struct Aes { #endif #ifdef WOLFSSL_AESNI byte use_aesni; + #if defined(WOLFSSL_LINUXKM) || defined(WC_WANT_FLAG_DONT_USE_AESNI) + /* Note, we can't support WC_FLAG_DONT_USE_AESNI by default because we + * need to support legacy applications that call wc_AesSetKey() on + * uninited struct Aes. For details see the software implementation of + * wc_AesSetKeyLocal() (aes.c). + */ + #define WC_FLAG_DONT_USE_AESNI 2 + #endif #endif /* WOLFSSL_AESNI */ #if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) diff --git a/wolfssl/wolfcrypt/settings.h b/wolfssl/wolfcrypt/settings.h index 6cfd33d46..855cb8d5c 100644 --- a/wolfssl/wolfcrypt/settings.h +++ b/wolfssl/wolfcrypt/settings.h @@ -3607,7 +3607,14 @@ extern void uITRON4_free(void *p) ; #define WOLFSSL_OLD_PRIME_CHECK #endif #ifndef WOLFSSL_TEST_SUBROUTINE - #define WOLFSSL_TEST_SUBROUTINE static + #ifdef LINUXKM_LKCAPI_REGISTER + #define WOLFSSL_TEST_SUBROUTINE + #else + #define WOLFSSL_TEST_SUBROUTINE static + #endif + #endif + #ifdef LINUXKM_LKCAPI_REGISTER + #define WC_TEST_EXPORT_SUBTESTS #endif #undef HAVE_PTHREAD /* linuxkm uses linux/string.h, included by linuxkm_wc_port.h. */ diff --git a/wolfssl/wolfcrypt/sp_int.h b/wolfssl/wolfcrypt/sp_int.h index 7c5fdd1ab..fb202d4c5 100644 --- a/wolfssl/wolfcrypt/sp_int.h +++ b/wolfssl/wolfcrypt/sp_int.h @@ -210,8 +210,10 @@ extern "C" { #elif defined(WOLFSSL_SP_X86_64_ASM) || defined(WOLFSSL_SP_X86_64) #if SP_ULONG_BITS == 64 || SP_ULLONG_BITS == 64 #define SP_WORD_SIZE 64 - #define HAVE_INTEL_AVX1 - #ifndef NO_AVX2_SUPPORT + #ifndef HAVE_INTEL_AVX1 + #define HAVE_INTEL_AVX1 + #endif + #if !defined(NO_AVX2_SUPPORT) && !defined(HAVE_INTEL_AVX2) #define HAVE_INTEL_AVX2 #endif #elif SP_ULONG_BITS == 32