diff --git a/configure.ac b/configure.ac index c75bee31e..53131278f 100644 --- a/configure.ac +++ b/configure.ac @@ -3882,6 +3882,40 @@ AC_ARG_WITH([cavium-v], ) +# Cavium Octeon +OCTEON_ROOT="" +: ${OCTEON_OBJ="obj-octeon2"} + +AC_ARG_WITH([octeon-sync], + [AS_HELP_STRING([--with-octeon-sync=PATH],[PATH to Cavium Octeon SDK dir (sync)])], + [ + AC_MSG_CHECKING([for octeon]) + + if test "x$withval" = "xyes" ; then + AC_MSG_ERROR([need a PATH for --with-octeon]) + fi + if test "x$withval" != "xno" ; then + OCTEON_ROOT=$withval + fi + + AM_CFLAGS="$AM_CFLAGS -DHAVE_CAVIUM_OCTEON_SYNC" + AM_CFLAGS="$AM_CFLAGS -DOCTEON_MODEL=$OCTEON_MODEL -DCVMX_BUILD_FOR_LINUX_HOST" + AM_CFLAGS="$AM_CFLAGS -I$OCTEON_ROOT/executive" + + #-I$OCTEON_ROOT/target/include + AM_LDFLAGS="$AM_LDFLAGS -lrt -Xlinker -T -Xlinker $OCTEON_ROOT/executive/cvmx-shared-linux.ld" + AM_LDFLAGS="$AM_LDFLAGS -L$OCTEON_ROOT/executive/$OCTEON_OBJ -lcvmx -lfdt" + + enable_shared=no + enable_static=yes + + ENABLED_OCTEON_SYNC=yes + AC_MSG_RESULT([yes]) + ], + [ENABLED_OCTEON_SYNC=no] +) + + # Intel QuickAssist QAT_DIR="" BUILD_INTEL_QAT_VERSION=2 @@ -4580,9 +4614,10 @@ AS_IF([test "x$ENABLED_OCSP" = "xyes" && \ test "x$ENABLED_ECC" = "xno"], [AC_MSG_ERROR([please enable rsa or ecc if enabling ocsp.])]) -# Sync Intel QA requires the crypto callback -AS_IF([test "x$ENABLED_CRYPTOCB" = "xno" && test "x$ENABLED_INTEL_QA_SYNC" = "xyes"], - [AC_MSG_ERROR([please enable the crypto callback support using --enable-cryptocb])]) +# Sync Intel QA and Sync Cavium Octeon require the crypto callback +AS_IF([test "x$ENABLED_INTEL_QA_SYNC" = "xyes" || test "x$ENABLED_OCTEON_SYNC" = "xyes"], + [AS_IF([test "x$ENABLED_CRYPTOCB" = "xno"], + [AC_MSG_ERROR([please enable the crypto callback support using --enable-cryptocb])])]) # checks for pkcs7 needed enables AS_IF([test "x$ENABLED_PKCS7" = "xyes" && \ @@ -4900,6 +4935,7 @@ AM_CONDITIONAL([BUILD_LIBZ],[test "x$ENABLED_LIBZ" = "xyes"]) AM_CONDITIONAL([BUILD_PKCS11],[test "x$ENABLED_PKCS11" = "xyes"]) AM_CONDITIONAL([BUILD_CAVIUM],[test "x$ENABLED_CAVIUM" = "xyes"]) AM_CONDITIONAL([BUILD_CAVIUM_V],[test "x$ENABLED_CAVIUM_V" = "xyes"]) +AM_CONDITIONAL([BUILD_OCTEON_SYNC],[test "x$ENABLED_OCTEON_SYNC" = "xyes"]) AM_CONDITIONAL([BUILD_INTEL_QA],[test "x$ENABLED_INTEL_QA" = "xyes"]) AM_CONDITIONAL([BUILD_INTEL_QA_SYNC],[test "x$ENABLED_INTEL_QA_SYNC" = "xyes"]) AM_CONDITIONAL([BUILD_SP],[test "x$ENABLED_SP" = "xyes"]) @@ -5191,16 +5227,17 @@ echo " * Single Precision: $ENABLED_SP" echo " * Async Crypto: $ENABLED_ASYNCCRYPT" echo " * PKCS#11: $ENABLED_PKCS11" echo " * PKCS#12: $ENABLED_PKCS12" -echo " * Cavium: $ENABLED_CAVIUM" +echo " * Cavium Nitox: $ENABLED_CAVIUM" +echo " * Cavium Octeon (Sync): $ENABLED_OCTEON_SYNC" +echo " * Intel Quick Assist: $ENABLED_INTEL_QA" echo " * ARM ASM: $ENABLED_ARMASM" echo " * AES Key Wrap: $ENABLED_AESKEYWRAP" echo " * Write duplicate: $ENABLED_WRITEDUP" -echo " * Intel Quick Assist: $ENABLED_INTEL_QA" echo " * Xilinx Hardware Acc.: $ENABLED_XILINX" echo " * Inline Code: $ENABLED_INLINE" echo " * Linux AF_ALG: $ENABLED_AFALG" echo " * Linux devcrypto: $ENABLED_DEVCRYPTO" -echo " * Crypto callback: $ENABLED_CRYPTOCB" +echo " * Crypto callbacks: $ENABLED_CRYPTOCB" echo "" echo "---" diff --git a/src/sniffer.c b/src/sniffer.c index bad212021..835f06cdd 100644 --- a/src/sniffer.c +++ b/src/sniffer.c @@ -61,6 +61,9 @@ #ifdef HAVE_INTEL_QA_SYNC #include #endif + #ifdef HAVE_CAVIUM_OCTEON_SYNC + #include + #endif #endif @@ -485,9 +488,6 @@ static void UpdateMissedDataSessions(void) #ifdef WOLF_CRYPTO_CB static int CryptoDeviceId = INVALID_DEVID; - #ifdef HAVE_INTEL_QA_SYNC - static IntelQaDev CryptoDevice; - #endif #endif @@ -504,21 +504,15 @@ void ssl_InitSniffer(void) #endif #ifdef WOLF_CRYPTO_CB #ifdef HAVE_INTEL_QA_SYNC - { - int rc; - CryptoDeviceId = IntelQaInit(NULL); - if (CryptoDeviceId == INVALID_DEVID) { - WOLFSSL_MSG("Couldn't init the Intel QA"); - } - rc = IntelQaOpen(&CryptoDevice, CryptoDeviceId); - if (rc != 0) { - WOLFSSL_MSG("Couldn't open the device"); - } - rc = wc_CryptoCb_RegisterDevice(CryptoDeviceId, - IntelQaSymSync_CryptoDevCb, &CryptoDevice); - if (rc != 0) { - WOLFSSL_MSG("Couldn't register the device"); - } + CryptoDeviceId = wc_CryptoCb_InitIntelQa(); + if (INVALID_DEVID == CryptoDeviceId) { + printf("Couldn't init the Intel QA\n"); + } + #endif + #ifdef HAVE_CAVIUM_OCTEON_SYNC + CryptoDeviceId = wc_CryptoCb_InitOcteon(); + if (INVALID_DEVID == CryptoDeviceId) { + printf("Couldn't init the Intel QA\n"); } #endif #endif @@ -652,9 +646,10 @@ void ssl_FreeSniffer(void) #ifdef WOLF_CRYPTO_CB #ifdef HAVE_INTEL_QA_SYNC - wc_CryptoCb_UnRegisterDevice(CryptoDeviceId); - IntelQaClose(&CryptoDevice); - IntelQaDeInit(CryptoDeviceId); + wc_CryptoCb_CleanupIntelQa(&CryptoDeviceId); +#endif +#ifdef HAVE_CAVIUM_OCTEON_SYNC + wc_CryptoCb_CleanupOcteon(&CryptoDeviceId); #endif #endif @@ -1439,7 +1434,7 @@ static int LoadKeyFile(byte** keyBuf, word32* keyBufSz, #ifdef WOLFSSL_SNIFFER_WATCH -static int CreateWatchSnifferServer(char* error) +static int CreateWatchSnifferServer(char* error, int devId) { SnifferServer* sniffer; @@ -1457,7 +1452,8 @@ static int CreateWatchSnifferServer(char* error) return -1; } #ifdef WOLF_CRYPTO_CB - wolfSSL_CTX_SetDevId(sniffer->ctx, CryptoDevId); + if (CryptoDeviceId != INVALID_DEVID) + wolfSSL_CTX_SetDevId(sniffer->ctx, devId); #endif ServerList = sniffer; @@ -4508,10 +4504,17 @@ int ssl_ReadResetStatistics(SSLStats* stats) #ifdef WOLFSSL_SNIFFER_WATCH +int ssl_SetWatchKeyCallback_ex(SSLWatchCb cb, int devId, char* error) +{ + WatchCb = cb; + return CreateWatchSnifferServer(error, devId); +} + + int ssl_SetWatchKeyCallback(SSLWatchCb cb, char* error) { WatchCb = cb; - return CreateWatchSnifferServer(error); + return CreateWatchSnifferServer(error, INVALID_DEVID); } diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index 33e2bd536..af3da627c 100755 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -172,6 +172,16 @@ #include #include +#ifdef WOLF_CRYPTO_CB + #include + #ifdef HAVE_INTEL_QA_SYNC + #include + #endif + #ifdef HAVE_CAVIUM_OCTEON_SYNC + #include + #endif +#endif + #ifdef WOLFSSL_ASYNC_CRYPT #include #endif @@ -1288,6 +1298,21 @@ static void* benchmarks_do(void* args) } #endif +#ifdef WOLF_CRYPTO_CB +#ifdef HAVE_INTEL_QA_SYNC + devId = wc_CryptoCb_InitIntelQa(); + if (devId == INVALID_DEVID) { + printf("Couldn't init the Intel QA\n"); + } +#endif +#ifdef HAVE_CAVIUM_OCTEON_SYNC + devId = wc_CryptoCb_InitOcteon(); + if (devId == INVALID_DEVID) { + printf("Couldn't get the Octeon device ID\n"); + } +#endif +#endif + #if defined(HAVE_LOCAL_RNG) { int rngRet; @@ -1328,7 +1353,7 @@ static void* benchmarks_do(void* args) XMEMSET(bench_plain, 0, (size_t)bench_buf_size); XMEMSET(bench_cipher, 0, (size_t)bench_buf_size); -#ifdef WOLFSSL_ASYNC_CRYPT +#if defined(WOLFSSL_ASYNC_CRYPT) || defined(HAVE_INTEL_QA_SYNC) bench_key = (byte*)XMALLOC(sizeof(bench_key_buf), HEAP_HINT, DYNAMIC_TYPE_WOLF_BIGINT); bench_iv = (byte*)XMALLOC(sizeof(bench_iv_buf), HEAP_HINT, DYNAMIC_TYPE_WOLF_BIGINT); if (bench_key == NULL || bench_iv == NULL) { @@ -1356,7 +1381,8 @@ static void* benchmarks_do(void* args) #ifndef NO_SW_BENCH bench_aescbc(0); #endif - #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES) && \ + #if ((defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_3DES)) || \ + defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)) && \ !defined(NO_HW_BENCH) bench_aescbc(1); #endif @@ -1367,7 +1393,8 @@ static void* benchmarks_do(void* args) #ifndef NO_SW_BENCH bench_aesgcm(0); #endif - #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES) && \ + #if ((defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_3DES)) || \ + defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)) && \ !defined(NO_HW_BENCH) bench_aesgcm(1); #endif @@ -1438,7 +1465,8 @@ static void* benchmarks_do(void* args) #ifndef NO_SW_BENCH bench_des(0); #endif - #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_3DES) && \ + #if ((defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_3DES)) || \ + defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)) && \ !defined(NO_HW_BENCH) bench_des(1); #endif @@ -1775,6 +1803,15 @@ exit: XFREE(bench_iv, HEAP_HINT, DYNAMIC_TYPE_WOLF_BIGINT); #endif +#ifdef WOLF_CRYPTO_CB +#ifdef HAVE_INTEL_QA_SYNC + wc_CryptoCb_CleanupIntelQa(&devId); +#endif +#ifdef HAVE_CAVIUM_OCTEON_SYNC + wc_CryptoCb_CleanupOcteon(&devId); +#endif +#endif + #ifdef WOLFSSL_ASYNC_CRYPT /* free event queue */ wolfEventQueue_Free(&eventQueue); diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 057fcd36d..ce6f6670e 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -3654,6 +3654,12 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) wc_AesGcmSetKey_ex(aes, key, len, 0); #endif +#ifdef WOLF_CRYPTO_CB + if (aes->devId != INVALID_DEVID) { + XMEMCPY(aes->devKey, key, len); + } +#endif + #ifdef WOLFSSL_IMX6_CAAM_BLOB ForceZero(local, sizeof(local)); #endif diff --git a/wolfcrypt/src/cryptocb.c b/wolfcrypt/src/cryptocb.c index 5e7c8fb23..a9ba73007 100644 --- a/wolfcrypt/src/cryptocb.c +++ b/wolfcrypt/src/cryptocb.c @@ -44,7 +44,7 @@ typedef struct CryptoCb { CryptoDevCallbackFunc cb; void* ctx; } CryptoCb; -static CryptoCb gCryptoDev[MAX_CRYPTO_DEVID_CALLBACKS]; +static WOLFSSL_GLOBAL CryptoCb gCryptoDev[MAX_CRYPTO_DEVID_CALLBACKS]; static CryptoCb* wc_CryptoCb_FindDevice(int devId) { diff --git a/wolfcrypt/src/include.am b/wolfcrypt/src/include.am index eae62a108..67cc0003a 100644 --- a/wolfcrypt/src/include.am +++ b/wolfcrypt/src/include.am @@ -105,6 +105,11 @@ src_libwolfssl_la_SOURCES += wolfcrypt/src/port/cavium/cavium_nitrox.c endif EXTRA_DIST += wolfcrypt/src/port/cavium/README.md +if BUILD_OCTEON_SYNC +src_libwolfssl_la_SOURCES += wolfcrypt/src/port/cavium/cavium_octeon_sync.c +endif +EXTRA_DIST += wolfcrypt/src/port/cavium/README_Octeon.md + if BUILD_INTEL_QA src_libwolfssl_la_SOURCES += wolfcrypt/src/port/intel/quickassist.c src_libwolfssl_la_SOURCES += wolfcrypt/src/port/intel/quickassist_mem.c diff --git a/wolfcrypt/src/port/cavium/README_Octeon.md b/wolfcrypt/src/port/cavium/README_Octeon.md new file mode 100644 index 000000000..b2670d02e --- /dev/null +++ b/wolfcrypt/src/port/cavium/README_Octeon.md @@ -0,0 +1,3 @@ +# Cavium Octeon III CN7300 + +Please contact wolfSSL at info@wolfssl.com to request an evaluation. diff --git a/wolfcrypt/src/port/cavium/cavium_octeon_sync.c b/wolfcrypt/src/port/cavium/cavium_octeon_sync.c new file mode 100644 index 000000000..735a9ad3f --- /dev/null +++ b/wolfcrypt/src/port/cavium/cavium_octeon_sync.c @@ -0,0 +1,879 @@ +/* cavium_octeon_sync.c + * + * Copyright(C) 2006-2019 wolfSSL Inc. + * + * This file is part of wolfSSL.(formerly known as CyaSSL) + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + *(at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#ifdef HAVE_CONFIG_H + #include +#endif + +#include +#include +#include + +#ifdef HAVE_CAVIUM_OCTEON_SYNC + +/* Setting NO_MAIN_DRIVER here because this file ends up building + * in the library sources which doesn't have NO_MAIN_DRIVER set, + * as the library expects main to be somehwere else. */ +#undef NO_MAIN_DRIVER +#define NO_MAIN_DRIVER + +#include +#include +#include +#include + +#include "cvmx.h" +#include "cvmx-asm.h" +#include "cvmx-key.h" +#include "cvmx-swap.h" + +#ifndef NO_DES3 + #include +#endif +#ifndef NO_AES + #include +#endif + +#define NOOPT __attribute__((optimize("O0"))) + +static int devId = 1234; + +#ifndef NO_DES3 +static int Octeon_Des3_CbcEncrypt(Des3* des3, + uint64_t *inp64, uint64_t *outp64, size_t inl) +{ + register uint64_t i0, r0; + uint64_t *key, *iv; + + if (des3 == NULL || inp64 == NULL || outp64 == NULL) + return BAD_FUNC_ARG; + + /* expects 64-bit aligned value */ + key = (uint64_t*)des3->devKey; + CVMX_MT_3DES_KEY(key[0], 0); + CVMX_MT_3DES_KEY(key[1], 1); + CVMX_MT_3DES_KEY(key[2], 2); + iv = (uint64_t*)des3->reg; + CVMX_MT_3DES_IV(iv[0]); + + CVMX_PREFETCH0(inp64); + + i0 = *inp64; + + /* DES3 assembly can handle 16-byte chunks */ + if (inl >= 16) { + CVMX_MT_3DES_ENC_CBC(i0); + inl -= 8; + inp64++; + outp64++; + + if (inl >= 8) { + i0 = inp64[0]; + CVMX_MF_3DES_RESULT(r0); + CVMX_MT_3DES_ENC_CBC(i0); + + for (;;) { + outp64[-1] = r0; + inl -= 8; + inp64++; + outp64++; + i0 = *inp64; + + if (inl < 8) + break; + + CVMX_PREFETCH(inp64, 64); + CVMX_MF_3DES_RESULT(r0); + CVMX_MT_3DES_ENC_CBC(i0); + } + } + CVMX_MF_3DES_RESULT(r0); + outp64[-1] = r0; + } + /* remainder */ + if (inl > 0) { + uint64_t r = 0; + if (inl <= 8) { + XMEMCPY(&r, inp64, inl); + CVMX_MT_3DES_ENC_CBC(r); + CVMX_MF_3DES_RESULT(*outp64); + } + else { + i0 = *inp64; + CVMX_MT_3DES_ENC_CBC(i0); + CVMX_MF_3DES_RESULT(*outp64); + inp64++, outp64++; + + XMEMCPY(&r, inp64, inl); + CVMX_MT_3DES_ENC_CBC(r); + CVMX_MF_3DES_RESULT(*outp64); + } + } + + CVMX_MT_3DES_IV(iv[0]); + + return 0; +} + +static int Octeon_Des3_CbcDecrypt(Des3* des3, + uint64_t *inp64, uint64_t *outp64, size_t inl) +{ + register uint64_t i0, r0; + uint64_t *key, *iv; + + if (des3 == NULL || inp64 == NULL || outp64 == NULL) + return BAD_FUNC_ARG; + + /* expects 64-bit aligned value */ + key = (uint64_t*)des3->devKey; + CVMX_MT_3DES_KEY(key[0], 0); + CVMX_MT_3DES_KEY(key[1], 1); + CVMX_MT_3DES_KEY(key[2], 2); + + iv = (uint64_t*)des3->reg; + CVMX_MT_3DES_IV(iv[0]); + + CVMX_PREFETCH0(inp64); + + i0 = *inp64; + + /* DES3 assembly can handle 16-byte chunks */ + if (inl >= 16) { + CVMX_MT_3DES_DEC_CBC(i0); + inl -= 8; + inp64++; + outp64++; + + if (inl >= 8) { + i0 = inp64[0]; + CVMX_MF_3DES_RESULT(r0); + CVMX_MT_3DES_DEC_CBC(i0); + + for (;;) { + outp64[-1] = r0; + inl -= 8; + inp64++; + outp64++; + i0 = *inp64; + + if (inl < 8) + break; + + CVMX_PREFETCH(inp64, 64); + CVMX_MF_3DES_RESULT(r0); + CVMX_MT_3DES_DEC_CBC(i0); + } + } + + CVMX_MF_3DES_RESULT(r0); + outp64[-1] = r0; + } + /* remainder */ + if (inl > 0) { + if (inl <= 8) { + uint64_t r = 0; + XMEMCPY(&r, inp64, inl); + CVMX_MT_3DES_DEC_CBC(r); + CVMX_MF_3DES_RESULT(*outp64); + } + else { + uint64_t r = 0; + i0 = *inp64; + CVMX_MT_3DES_DEC_CBC(i0); + CVMX_MF_3DES_RESULT(*outp64); + inp64++, outp64++; + + XMEMCPY(&r, inp64, inl); + CVMX_MT_3DES_DEC_CBC(r); + CVMX_MF_3DES_RESULT(*outp64); + } + } + + CVMX_MT_3DES_IV(iv[0]); + + return 0; +} +#endif /* !NO_DES3 */ + + +#ifndef NO_AES + +#ifdef HAVE_AES_CBC +static int Octeon_AesCbc_Encrypt(Aes *aes, + uint64_t *inp64, uint64_t *outp64, size_t inl) +{ + register uint64_t i0, i1, r0, r1; + uint64_t *key, *iv; + + if (aes == NULL || inp64 == NULL || outp64 == NULL) { + return BAD_FUNC_ARG; + } + + iv = (uint64_t*)aes->reg; + CVMX_MT_AES_IV(iv[0], 0); + CVMX_MT_AES_IV(iv[1], 1); + + key = (uint64_t*)aes->devKey; + CVMX_MT_AES_KEY(key[0], 0); + CVMX_MT_AES_KEY(key[1], 1); + CVMX_MT_AES_KEY(key[2], 2); + CVMX_MT_AES_KEY(key[3], 3); + + CVMX_MT_AES_KEYLENGTH(aes->keylen/8 - 1); + + CVMX_PREFETCH0(inp64); + + i0 = inp64[0]; + i1 = inp64[1]; + + /* AES assembly can handle 32-byte chunks */ + if (inl >= 32) { + CVMX_MT_AES_ENC_CBC0(i0); + CVMX_MT_AES_ENC_CBC1(i1); + inl -= 16; + inp64 += 2; + outp64 += 2; + + if (inl >= 16) { + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + i0 = inp64[0]; + i1 = inp64[1]; + CVMX_MT_AES_ENC_CBC0(i0); + CVMX_MT_AES_ENC_CBC1(i1); + + for (;;) { + outp64[-2] = r0; + outp64[-1] = r1; + outp64 += 2; + inp64 += 2; + inl -= 16; + i0 = inp64[0]; + i1 = inp64[1]; + + if (inl < 16) + break; + + CVMX_PREFETCH(inp64, 64); + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + CVMX_MT_AES_ENC_CBC0(i0); + CVMX_MT_AES_ENC_CBC1(i1); + } + } + + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + outp64[-2] = r0; + outp64[-1] = r1; + } + /* remainder */ + if (inl > 0) { + uint64_t in64[2] = { 0, 0 }; + if (inl <= 16) { + XMEMCPY(in64, inp64, inl); + CVMX_MT_AES_ENC_CBC0(in64[0]); + CVMX_MT_AES_ENC_CBC1(in64[1]); + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + outp64[0] = r0; + outp64[1] = r1; + } + else { + CVMX_MT_AES_ENC_CBC0(i0); + CVMX_MT_AES_ENC_CBC1(i1); + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + inl -= 16; + outp64[0] = r0; + outp64[1] = r1; + inp64 += 2; + outp64 += 2; + XMEMCPY(in64, inp64, inl); + CVMX_MT_AES_ENC_CBC0(in64[0]); + CVMX_MT_AES_ENC_CBC1(in64[1]); + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + outp64[0] = r0; + outp64[1] = r1; + } + } + + CVMX_MF_AES_IV(iv[0], 0); + CVMX_MF_AES_IV(iv[1], 1); + + return 0; +} + +static int Octeon_AesCbc_Decrypt(Aes *aes, + uint64_t *inp64, uint64_t *outp64, size_t inl) +{ + register uint64_t i0, i1, r0, r1; + uint64_t *key, *iv; + + if (aes == NULL || inp64 == NULL || outp64 == NULL) { + return BAD_FUNC_ARG; + } + + iv = (uint64_t*)aes->reg; + key = (uint64_t*)aes->devKey; + + CVMX_MT_AES_IV(iv[0], 0); + CVMX_MT_AES_IV(iv[1], 1); + + CVMX_MT_AES_KEY(key[0], 0); + CVMX_MT_AES_KEY(key[1], 1); + CVMX_MT_AES_KEY(key[2], 2); + CVMX_MT_AES_KEY(key[3], 3); + CVMX_MT_AES_KEYLENGTH(aes->keylen/8 - 1); + + CVMX_PREFETCH0(inp64); + + i0 = inp64[0]; + i1 = inp64[1]; + + /* AES assembly can handle 32-byte chunks */ + if (inl >= 32) { + CVMX_MT_AES_DEC_CBC0(i0); + CVMX_MT_AES_DEC_CBC1(i1); + inp64 += 2; + outp64 += 2; + inl -= 16; + + if (inl >= 16) { + i0 = inp64[0]; + i1 = inp64[1]; + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + CVMX_MT_AES_DEC_CBC0(i0); + CVMX_MT_AES_DEC_CBC1(i1); + + for (;;) { + outp64[-2] = r0; + outp64[-1] = r1; + outp64 += 2; + inp64 += 2; + inl -= 16; + i0 = inp64[0]; + i1 = inp64[1]; + + if (inl < 16) + break; + + CVMX_PREFETCH(inp64, 64); + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + CVMX_MT_AES_DEC_CBC0(i0); + CVMX_MT_AES_DEC_CBC1(i1); + } + } + + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + outp64[-2] = r0; + outp64[-1] = r1; + } + /* remainder */ + if (inl > 0) { + uint64_t in64[2] = { 0, 0 }; + XMEMCPY(in64, inp64, inl); + CVMX_MT_AES_DEC_CBC0(in64[0]); + CVMX_MT_AES_DEC_CBC1(in64[1]); + CVMX_MF_AES_RESULT(r0, 0); + CVMX_MF_AES_RESULT(r1, 1); + outp64[0] = r0; + outp64[1] = r1; + } + + CVMX_MF_AES_IV(iv[0], 0); + CVMX_MF_AES_IV(iv[1], 1); + + return 0; +} +#endif /* HAVE_AES_CBC */ + + +#ifdef HAVE_AESGCM + +#define CVM_AES_RD_RESULT_WR_DATA(in1, in2, out1, out2) \ + asm volatile(\ + ".set noreorder \n" \ + "dmfc2 %[r1],0x0100\n" \ + "dmfc2 %[r2],0x0101\n" \ + "dmtc2 %[r3],0x010a\n" \ + "dmtc2 %[r4],0x310b\n" \ + ".set reorder \n" \ + : [r1] "=&d"(in1) , [r2] "=&d"(in2) \ + : [r3] "d"(out1), [r4] "d"(out2)) + +static NOOPT void Octeon_GHASH_Restore(word16 poly, byte* h) +{ + word64* bigH = (word64*)h; + CVMX_MT_GFM_POLY((word64)poly); + CVMX_MT_GFM_MUL(bigH[0], 0); + CVMX_MT_GFM_MUL(bigH[1], 1); +} + + +static NOOPT void Octeon_GHASH_Init(word16 poly, byte* h) +{ + Octeon_GHASH_Restore(poly, h); + CVMX_MT_GFM_RESINP(0, 0); + CVMX_MT_GFM_RESINP(0, 1); +} + + +static NOOPT void Octeon_GHASH_Update(byte* in) +{ + word64* bigIn = (word64*)in; + CVMX_MT_GFM_XOR0(bigIn[0]); + CVMX_MT_GFM_XORMUL1(bigIn[1]); +} + + +static NOOPT void Octeon_GHASH_Final(byte* out, word64 authInSz, word64 inSz) +{ + word64* bigOut = (word64*)out; + + CVMX_MT_GFM_XOR0(authInSz * 8); + CVMX_MT_GFM_XORMUL1(inSz * 8); + CVMX_MF_GFM_RESINP(bigOut[0], 0); + CVMX_MF_GFM_RESINP(bigOut[1], 1); +} + + +/* Sets the Octeon key with the key found in the Aes record. */ +static NOOPT int Octeon_AesGcm_SetKey(Aes* aes) +{ + int ret = 0; + + if (aes == NULL) + ret = BAD_FUNC_ARG; + + if (ret == 0) { + uint64_t* key = (uint64_t*)aes->devKey; + + CVMX_MT_AES_KEY(key[0], 0); + CVMX_MT_AES_KEY(key[1], 1); + CVMX_MT_AES_KEY(key[2], 2); + CVMX_MT_AES_KEY(key[3], 3); + CVMX_MT_AES_KEYLENGTH((aes->keylen / 8) - 1); + + if (!aes->keySet) { + uint64_t* bigH = (uint64_t*)aes->H; + CVMX_MT_AES_ENC0(0); + CVMX_MT_AES_ENC1(0); + CVMX_MF_AES_RESULT(bigH[0], 0); + CVMX_MF_AES_RESULT(bigH[1], 1); + aes->keySet = 1; + } + } + + return ret; +} + + +static NOOPT int Octeon_AesGcm_SetIV(Aes* aes, byte* iv, word32 ivSz) +{ + int ret = 0; + + if (aes == NULL || iv == NULL) + ret = BAD_FUNC_ARG; + + if (ret == 0) { + if (ivSz == GCM_NONCE_MID_SZ) { + XMEMSET((byte*)aes->reg, 0, sizeof(aes->reg)); + XMEMCPY((byte*)aes->reg, iv, ivSz); + aes->reg[3] = 1; + } + else { + int blocks, remainder, i; + byte aesBlock[AES_BLOCK_SIZE]; + + blocks = ivSz / AES_BLOCK_SIZE; + remainder = ivSz % AES_BLOCK_SIZE; + + for (i = 0; i < blocks; i++, iv += AES_BLOCK_SIZE) + Octeon_GHASH_Update(iv); + + XMEMSET(aesBlock, 0, sizeof(aesBlock)); + for (i = 0; i < remainder; i++) + aesBlock[i] = iv[i]; + Octeon_GHASH_Update(aesBlock); + + Octeon_GHASH_Final((byte*)aes->reg, 0, ivSz); + } + + aes->y0 = aes->reg[3]; + aes->reg[3]++; + + Octeon_GHASH_Init(0xe100, aes->H); + } + + return ret; +} + + +static NOOPT int Octeon_AesGcm_SetAAD(Aes* aes, byte* aad, word32 aadSz) +{ + word64* p; + ALIGN16 byte aesBlock[AES_BLOCK_SIZE]; + int blocks, remainder, i; + + if (aes == NULL || (aadSz != 0 && aad == NULL)) + return BAD_FUNC_ARG; + + if (aadSz == 0) + return 0; + + blocks = aadSz / AES_BLOCK_SIZE; + remainder = aadSz % AES_BLOCK_SIZE; + + Octeon_GHASH_Restore(0xe100, aes->H); + + p = (word64*)aesBlock; + + for (i = 0; i < blocks; i++, aad += AES_BLOCK_SIZE) { + CVMX_LOADUNA_INT64(p[0], aad, 0); + CVMX_LOADUNA_INT64(p[1], aad, 8); + CVMX_MT_GFM_XOR0(p[0]); + CVMX_MT_GFM_XORMUL1(p[1]); + } + + XMEMSET(aesBlock, 0, sizeof(aesBlock)); + + for (i = 0; i < remainder; i++) + aesBlock[i] = aad[i]; + + CVMX_MT_GFM_XOR0(p[0]); + CVMX_MT_GFM_XORMUL1(p[1]); + + return 0; +} + + +static int Octeon_AesGcm_SetEncrypt(Aes* aes, byte* in, byte* out, word32 inSz, + int encrypt) +{ + word32 i, blocks, remainder; + ALIGN16 byte aesBlockIn[AES_BLOCK_SIZE]; + ALIGN16 byte aesBlockOut[AES_BLOCK_SIZE]; + word64* pIn; + word64* pOut; + word64* pIv; + + if (aes == NULL || in == NULL || out == NULL) + return BAD_FUNC_ARG; + + pIn = (word64*)aesBlockIn; + pOut = (word64*)aesBlockOut; + pIv = (word64*)aes->reg; + + CVMX_PREFETCH0(in); + + CVMX_MT_AES_ENC0(pIv[0]); + CVMX_MT_AES_ENC1(pIv[1]); + + blocks = inSz / AES_BLOCK_SIZE; + remainder = inSz % AES_BLOCK_SIZE; + + for (i = 0; i < blocks; + i++, in += AES_BLOCK_SIZE, out += AES_BLOCK_SIZE) { + CVMX_PREFETCH128(in); + aes->reg[3]++; + + CVMX_LOADUNA_INT64(pIn[0], in, 0); + CVMX_LOADUNA_INT64(pIn[1], in, 8); + + CVM_AES_RD_RESULT_WR_DATA(pOut[0], pOut[1], pIv[0], pIv[1]); + + if (encrypt) { + pOut[0] ^= pIn[0]; + pOut[1] ^= pIn[1]; + CVMX_MT_GFM_XOR0(pOut[0]); + CVMX_MT_GFM_XORMUL1(pOut[1]); + } + else { + CVMX_MT_GFM_XOR0(pIn[0]); + CVMX_MT_GFM_XORMUL1(pIn[1]); + pOut[0] ^= pIn[0]; + pOut[1] ^= pIn[1]; + } + + CVMX_STOREUNA_INT64(pOut[0], out, 0); + CVMX_STOREUNA_INT64(pOut[1], out, 8); + } + + if (remainder > 0) { + ALIGN16 byte aesBlockMask[AES_BLOCK_SIZE]; + word64* pMask = (word64*)aesBlockMask; + + XMEMSET(aesBlockOut, 0, sizeof(aesBlockOut)); + XMEMSET(aesBlockMask, 0, sizeof(aesBlockMask)); + for (i = 0; i < remainder; i++) { + aesBlockIn[i] = in[i]; + aesBlockMask[i] = 0xFF; + } + + if (encrypt) { + CVMX_MF_AES_RESULT(pOut[0], 0); + CVMX_MF_AES_RESULT(pOut[1], 1); + + pOut[0] ^= pIn[0]; + pOut[1] ^= pIn[1]; + + pOut[0] &= pMask[0]; + pOut[1] &= pMask[1]; + + CVMX_MT_GFM_XOR0(pOut[0]); + CVMX_MT_GFM_XORMUL1(pOut[1]); + } + else { + CVMX_MT_GFM_XOR0(pIn[0]); + CVMX_MT_GFM_XORMUL1(pIn[1]); + + CVMX_MF_AES_RESULT(pOut[0], 0); + CVMX_MF_AES_RESULT(pOut[1], 1); + + pOut[0] ^= pIn[0]; + pOut[1] ^= pIn[1]; + + pOut[0] &= pMask[0]; + pOut[1] &= pMask[1]; + } + + for (i = 0; i < remainder; i++) + out[i] = aesBlockOut[i]; + } + + return 0; +} + + +static NOOPT int Octeon_AesGcm_Finalize(Aes* aes, word32 inSz, word32 aadSz, + byte* tag) +{ + word64 bigSz; + word64* pIv; + word64* pIn; + word64* pOut; + uint32_t countSave; + ALIGN16 byte aesBlockIn[AES_BLOCK_SIZE]; + ALIGN16 byte aesBlockOut[AES_BLOCK_SIZE]; + + countSave = aes->reg[3]; + aes->reg[3] = aes->y0; + + pIv = (word64*)aes->reg; + CVMX_MT_AES_ENC0(pIv[0]); + CVMX_MT_AES_ENC1(pIv[1]); + + bigSz = (word64)aadSz * 8; + CVMX_MT_GFM_XOR0(bigSz); + bigSz = (word64)inSz * 8; + CVMX_MT_GFM_XORMUL1(bigSz); + + aes->reg[3] = countSave; + + pIn = (word64*)aesBlockIn; + CVMX_MF_AES_RESULT(pIn[0], 0); + CVMX_MF_AES_RESULT(pIn[1], 1); + + pOut = (word64*)aesBlockOut; + CVMX_MF_GFM_RESINP(pOut[0], 0); + CVMX_MF_GFM_RESINP(pOut[1], 1); + + pOut[0] ^= pIn[0]; + pOut[1] ^= pIn[1]; + + CVMX_STOREUNA_INT64(pOut[0], tag, 0); + CVMX_STOREUNA_INT64(pOut[1], tag, 8); + + return 0; +} + + +static int Octeon_AesGcm_Encrypt(Aes* aes, byte* in, byte* out, word32 inSz, + byte* iv, word32 ivSz, byte* aad, word32 aadSz, byte* tag) +{ + int ret = 0; + + if (aes == NULL) + ret = BAD_FUNC_ARG; + + if (ret == 0) + ret = Octeon_AesGcm_SetKey(aes); + + if (ret == 0) + ret = Octeon_AesGcm_SetIV(aes, iv, ivSz); + + if (ret == 0) + ret = Octeon_AesGcm_SetAAD(aes, aad, aadSz); + + if (ret == 0) + ret = Octeon_AesGcm_SetEncrypt(aes, in, out, inSz, 1); + + if (ret == 0) + ret = Octeon_AesGcm_Finalize(aes, inSz, aadSz, tag); + + return ret; +} + + +static int Octeon_AesGcm_Decrypt(Aes* aes, byte* in, byte* out, word32 inSz, + byte* iv, word32 ivSz, byte* aad, word32 aadSz, byte* tag) +{ + int ret = 0; + + if (aes == NULL) + ret = BAD_FUNC_ARG; + + if (ret == 0) + ret = Octeon_AesGcm_SetKey(aes); + + if (ret == 0) + ret = Octeon_AesGcm_SetIV(aes, iv, ivSz); + + if (ret == 0) + ret = Octeon_AesGcm_SetAAD(aes, aad, aadSz); + + if (ret == 0) + ret = Octeon_AesGcm_SetEncrypt(aes, in, out, inSz, 0); + + if (ret == 0) + ret = Octeon_AesGcm_Finalize(aes, inSz, aadSz, tag); + + return ret; +} + +#endif /* HAVE_AESGCM */ + +#endif /* !NO_AES */ + +#ifdef WOLF_CRYPTO_CB + +#include + + +static int myCryptoDevCb(int devIdArg, wc_CryptoInfo* info, void* ctx) +{ + int ret = NOT_COMPILED_IN; /* return this to bypass HW and use SW */ + + if (info == NULL) + return BAD_FUNC_ARG; + +#ifdef DEBUG_WOLFSSL + printf("CryptoDevCb: Algo Type %d\n", info->algo_type); +#endif + + if (info->algo_type == WC_ALGO_TYPE_CIPHER) { +#if !defined(NO_AES) || !defined(NO_DES3) + #ifdef HAVE_AESGCM + if (info->cipher.type == WC_CIPHER_AES_GCM) { + if (info->cipher.enc) { + ret = Octeon_AesGcm_Encrypt( + info->cipher.aesgcm_enc.aes, + (byte*)info->cipher.aesgcm_enc.in, + (byte*)info->cipher.aesgcm_enc.out, + info->cipher.aesgcm_enc.sz, + (byte*)info->cipher.aesgcm_enc.iv, + info->cipher.aesgcm_enc.ivSz, + (byte*)info->cipher.aesgcm_enc.authIn, + info->cipher.aesgcm_enc.authInSz, + (byte*)info->cipher.aesgcm_enc.authTag); + } + else { + ret = Octeon_AesGcm_Decrypt( + info->cipher.aesgcm_dec.aes, + (byte*)info->cipher.aesgcm_dec.in, + (byte*)info->cipher.aesgcm_dec.out, + info->cipher.aesgcm_dec.sz, + (byte*)info->cipher.aesgcm_dec.iv, + info->cipher.aesgcm_dec.ivSz, + (byte*)info->cipher.aesgcm_dec.authIn, + info->cipher.aesgcm_dec.authInSz, + (byte*)info->cipher.aesgcm_dec.authTag); + } + } + #endif /* HAVE_AESGCM */ + #ifdef HAVE_AES_CBC + if (info->cipher.type == WC_CIPHER_AES_CBC) { + if (info->cipher.enc) { + ret = Octeon_AesCbc_Encrypt( + info->cipher.aescbc.aes, + (word64*)info->cipher.aescbc.in, + (word64*)info->cipher.aescbc.out, + info->cipher.aescbc.sz); + } + else { + ret = Octeon_AesCbc_Decrypt( + info->cipher.aescbc.aes, + (word64*)info->cipher.aescbc.in, + (word64*)info->cipher.aescbc.out, + info->cipher.aescbc.sz); + } + } + #endif /* HAVE_AES_CBC */ + #ifndef NO_DES3 + if (info->cipher.type == WC_CIPHER_DES3) { + if (info->cipher.enc) { + ret = Octeon_Des3_CbcEncrypt( + info->cipher.des3.des, + (word64*)info->cipher.des3.in, + (word64*)info->cipher.des3.out, + info->cipher.des3.sz); + } + else { + ret = Octeon_Des3_CbcDecrypt( + info->cipher.des3.des, + (word64*)info->cipher.des3.in, + (word64*)info->cipher.des3.out, + info->cipher.des3.sz); + } + } + #endif /* !NO_DES3 */ +#endif /* !NO_AES || !NO_DES3 */ + } + + (void)devIdArg; + (void)ctx; + + return ret; +} + +int wc_CryptoCb_InitOcteon(void) +{ + if (wc_CryptoCb_RegisterDevice(devId, myCryptoDevCb, NULL) < 0) { + return INVALID_DEVID; + } + + return devId; +} + +void wc_CryptoCb_CleanupOcteon(int* id) +{ + wc_CryptoCb_UnRegisterDevice(*id); + *id = INVALID_DEVID; +} + +#endif /* WOLF_CRYPTO_CB */ + +#endif /* HAVE_CAVIUM_OCTEON_SYNC */ diff --git a/wolfcrypt/src/port/intel/quickassist_sync.c b/wolfcrypt/src/port/intel/quickassist_sync.c index 19a576000..a148d0b83 100644 --- a/wolfcrypt/src/port/intel/quickassist_sync.c +++ b/wolfcrypt/src/port/intel/quickassist_sync.c @@ -42,6 +42,18 @@ #include #include +#include "cpa.h" +#include "cpa_cy_im.h" +#include "cpa_cy_sym.h" +#include "cpa_cy_rsa.h" +#include "cpa_cy_ln.h" +#include "cpa_cy_ecdh.h" +#include "cpa_cy_ecdsa.h" +#include "cpa_cy_dh.h" +#include "cpa_cy_drbg.h" +#include "cpa_cy_nrbg.h" +#include "cpa_cy_prime.h" + #include "icp_sal_user.h" #include "icp_sal_poll.h" @@ -53,6 +65,164 @@ #include #endif +/* User space utils */ +#include +#include +#include +#include + +#if 0 + /* Optional feature for partial QAT hashing support */ + /* This will process updates through hardware instead of caching them */ + #define QAT_HASH_ENABLE_PARTIAL +#endif +#ifdef QAT_HASH_ENABLE_PARTIAL + #define MAX_QAT_HASH_BUFFERS 2 +#endif + +/* Detect QAT driver version */ +#if defined(CPA_CY_API_VERSION_NUM_MAJOR) && CPA_CY_API_VERSION_NUM_MAJOR > 1 + #define QAT_V2 +#endif + +#ifdef QAT_V2 + /* quickassist/utilities/libusdm_drv/qae_mem.h */ + /* Provides user-space API's for accessing NUMA allocated memory through usdm_drv */ + #include "qae_mem.h" +#include "linux/include/qae_mem_utils.h" +#endif + +#ifdef QAT_USE_POLLING_THREAD + #include +#endif + +/* Tunable parameters */ +#ifndef QAT_PROCESS_NAME + #define QAT_PROCESS_NAME "SSL" +#endif +#ifndef QAT_LIMIT_DEV_ACCESS + #define QAT_LIMIT_DEV_ACCESS CPA_FALSE +#endif +#ifndef QAT_MAX_DEVICES + #define QAT_MAX_DEVICES (1) /* maximum number of QAT cards */ +#endif + +#ifndef QAT_RETRY_LIMIT + #define QAT_RETRY_LIMIT (100) +#endif +#ifndef QAT_POLL_RESP_QUOTA + #define QAT_POLL_RESP_QUOTA (0) /* all pending */ +#endif + +#if !defined(NO_AES) || !defined(NO_DES3) + #define QAT_ENABLE_CRYPTO +#endif + +/* Pre-declarations */ +struct IntelQaDev; +struct wc_CryptoInfo; +struct WC_BIGINT; +struct WC_RNG; + + +#if defined(QAT_ENABLE_HASH) || defined(QAT_ENABLE_CRYPTO) +/* symmetric context */ +typedef struct IntelQaSymCtx { + CpaCySymOpData opData; + CpaCySymSessionCtx symCtxSrc; + CpaCySymSessionCtx symCtx; + word32 symCtxSize; + + /* flags */ + word32 isOpen:1; + word32 isCopy:1; +} IntelQaSymCtx; +#endif + +typedef void (*IntelQaFreeFunc)(struct IntelQaDev*); + + +/* QuickAssist device */ +typedef struct IntelQaDev { + CpaInstanceHandle handle; + int devId; + void* heap; + + /* callback return info */ + int ret; + byte* out; + union { + word32* outLenPtr; + word32 outLen; + }; + + /* operations */ + IntelQaFreeFunc freeFunc; + union { + #ifdef QAT_ENABLE_CRYPTO + struct { + IntelQaSymCtx ctx; + CpaBufferList bufferList; + CpaFlatBuffer flatBuffer; + byte* authTag; + word32 authTagSz; + } cipher; + #endif + } op; + +#ifdef QAT_USE_POLLING_THREAD + pthread_t pollingThread; + byte pollingCy; +#endif +} IntelQaDev; + + +/* Interface */ +static int IntelQaHardwareStart(const char*, int); +static void IntelQaHardwareStop(void); +static int IntelQaInit(void*); +static void IntelQaDeInit(int); +static int IntelQaNumInstances(void); +static int IntelQaOpen(IntelQaDev*, int); +static void IntelQaClose(IntelQaDev*); +static int IntelQaDevCopy(IntelQaDev*, IntelQaDev*); +static int IntelQaPoll(IntelQaDev*); +static int IntelQaGetCyInstanceCount(void); + +#ifndef NO_AES + #ifdef HAVE_AES_CBC + static int IntelQaSymAesCbcEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32); + #ifdef HAVE_AES_DECRYPT + static int IntelQaSymAesCbcDecrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32); + #endif /* HAVE_AES_DECRYPT */ + #endif /* HAVE_AES_CBC */ + + #ifdef HAVE_AESGCM + static int IntelQaSymAesGcmEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32, + byte*, word32, const byte*, word32); + #ifdef HAVE_AES_DECRYPT + static int IntelQaSymAesGcmDecrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32, + const byte*, word32, const byte*, word32); + #endif /* HAVE_AES_DECRYPT */ + #endif /* HAVE_AESGCM */ +#endif /* !NO_AES */ + +#ifndef NO_DES3 + static int IntelQaSymDes3CbcEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte* iv, word32); + static int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, byte*, + const byte*, word32, const byte*, word32, const byte* iv, word32); +#endif /*! NO_DES3 */ + +#ifdef WOLF_CRYPTO_CB + static int IntelQaSymSync_CryptoDevCb(int, struct wc_CryptoInfo*, + void*); +#endif /* WOLF_CRYPTO_CB */ + #ifdef QAT_DEBUG #define QLOG(...) do { printf(__VA_ARGS__); } while (0) @@ -94,6 +264,9 @@ static qatCapabilities_t g_qatCapabilities = {0}; extern Cpa32U osalLogLevelSet(Cpa32U level); +static IntelQaDev qaDev; + + /* -------------------------------------------------------------------------- */ /* Polling */ /* -------------------------------------------------------------------------- */ @@ -743,7 +916,11 @@ static void IntelQaSymCipherFree(IntelQaDev* dev) dev->out = NULL; dev->outLen = 0; #ifndef NO_AES - dev->op.cipher.authTag = NULL; + if (dev->op.cipher.authTag != NULL) { + XMEMSET(dev->op.cipher.authTag, 0, dev->op.cipher.authTagSz); + XFREE(dev->op.cipher.authTag, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); + dev->op.cipher.authTag = NULL; + } dev->op.cipher.authTagSz = 0; #endif } @@ -772,7 +949,9 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in, Cpa32U metaSize = 0; Cpa8U* authInBuf = NULL; Cpa32U authInSzAligned = authInSz; + Cpa8U* authTagBuf = NULL; IntelQaSymCtx* ctx; + CpaBoolean verifyResult = CPA_FALSE; QLOG("IntelQaSymCipher: dev %p, out %p, in %p, inOutSz %d, op %d, " "algo %d, dir %d, hash %d\n", @@ -806,19 +985,15 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in, bufferList = &dev->op.cipher.bufferList; flatBuffer = &dev->op.cipher.flatBuffer; metaBuf = XMALLOC(metaSize, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); -#ifndef WOLFSSL_SNIFFER - dataBuf = XREALLOC((byte*)in, dataLen, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); - ivBuf = XREALLOC((byte*)iv, AES_BLOCK_SIZE, dev->heap, - DYNAMIC_TYPE_ASYNC_NUMA); -#else dataBuf = XMALLOC(dataLen, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); XMEMCPY(dataBuf, in, inOutSz); ivBuf = XMALLOC(AES_BLOCK_SIZE, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); XMEMCPY(ivBuf, iv, ivSz); -#endif + authTagBuf = XMALLOC(authTagSz, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); /* check allocations */ - if (ivBuf == NULL || metaBuf == NULL || dataBuf == NULL) { + if (ivBuf == NULL || metaBuf == NULL || dataBuf == NULL || + authTagBuf == NULL) { ret = MEMORY_E; goto exit; } @@ -830,14 +1005,9 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in, (authInSzAligned % AES_BLOCK_SIZE); } -#ifndef WOLFSSL_SNIFFER - authInBuf = XREALLOC((byte*)authIn, authInSzAligned, dev->heap, - DYNAMIC_TYPE_ASYNC_NUMA); -#else authInBuf = XMALLOC(authInSzAligned, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); XMEMCPY(authInBuf, authIn, authInSz); -#endif if (authInBuf == NULL) { ret = MEMORY_E; goto exit; } @@ -878,7 +1048,10 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in, setup.hashSetupData.digestResultLenInBytes = authTagSz; setup.hashSetupData.authModeSetupData.aadLenInBytes = authInSz; - setup.digestIsAppended = CPA_TRUE; + if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) + setup.digestIsAppended = CPA_TRUE; + else + setup.digestIsAppended = CPA_FALSE; } /* open session */ @@ -903,13 +1076,19 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in, XMEMCPY(flatBuffer->pData + inOutSz, authTag, authTagSz); } } + else { + if (authTag && authTagSz > 0) { + XMEMCPY(authTagBuf, authTag, authTagSz); + } + } /* store info needed for output */ dev->out = out; dev->outLen = inOutSz; if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) { - dev->op.cipher.authTag = authTag; + dev->op.cipher.authTag = authTagBuf; dev->op.cipher.authTagSz = authTagSz; + opData->pDigestResult = authTagBuf; } else { dev->op.cipher.authTag = NULL; @@ -920,11 +1099,16 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in, /* perform symmetric AES operation async */ /* use same buffer list for in-place operation */ status = cpaCySymPerformOp(dev->handle, dev, opData, - bufferList, bufferList, NULL); - - if (ret == WC_PENDING_E) - return ret; + bufferList, bufferList, &verifyResult); + if (symOperation == CPA_CY_SYM_OP_ALGORITHM_CHAINING && + cipherAlgorithm == CPA_CY_SYM_CIPHER_AES_GCM && + cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT && + hashAlgorithm == CPA_CY_SYM_HASH_AES_GCM) { + if (verifyResult == CPA_FALSE) { + ret = AES_GCM_AUTH_E; + } + } exit: if (ret != 0) { @@ -932,10 +1116,13 @@ exit: dev, status, ret); } -#ifdef WOLFSSL_SNIFFER /* Capture the inline decrypt into the output. */ XMEMCPY(out, dataBuf, inOutSz); -#endif + if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) { + if (authTag != NULL && authTagSz > 0) { + XMEMCPY(authTag, authTagBuf, authTagSz); + } + } /* handle cleanup */ IntelQaSymCipherFree(dev); @@ -949,11 +1136,14 @@ int IntelQaSymAesCbcEncrypt(IntelQaDev* dev, const byte* key, word32 keySz, const byte* iv, word32 ivSz) { - return IntelQaSymCipher(dev, out, in, sz, + int ret = IntelQaSymCipher(dev, out, in, sz, key, keySz, iv, ivSz, CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_AES_CBC, CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT, CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0); + + XMEMCPY((byte*)iv, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE); + return ret; } #ifdef HAVE_AES_DECRYPT @@ -962,11 +1152,18 @@ int IntelQaSymAesCbcDecrypt(IntelQaDev* dev, const byte* key, word32 keySz, const byte* iv, word32 ivSz) { - return IntelQaSymCipher(dev, out, in, sz, + byte nextIv[AES_BLOCK_SIZE]; + int ret; + + XMEMCPY(nextIv, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE); + ret = IntelQaSymCipher(dev, out, in, sz, key, keySz, iv, ivSz, CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_AES_CBC, CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT, CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0); + + XMEMCPY((byte*)iv, nextIv, AES_BLOCK_SIZE); + return ret; } #endif /* HAVE_AES_DECRYPT */ #endif /* HAVE_AES_CBC */ @@ -1143,6 +1340,52 @@ int IntelQaSymSync_CryptoDevCb(int devId, struct wc_CryptoInfo* info, void* ctx) return rc; } +/* -------------------------------------------------------------------------- */ +/* Public API */ +/* -------------------------------------------------------------------------- */ + +int wc_CryptoCb_InitIntelQa(void) +{ + int devId, rc; + + devId = IntelQaInit(NULL); + if (devId < 0) { + QLOG("Couldn't init the Intel QA\n"); + devId = INVALID_DEVID; + } + else { + rc = IntelQaOpen(&qaDev, devId); + if (rc != 0) { + QLOG("Couldn't open the device\n"); + IntelQaDeInit(devId); + devId = INVALID_DEVID; + } + else { + rc = wc_CryptoCb_RegisterDevice(devId, + IntelQaSymSync_CryptoDevCb, &qaDev); + if (rc != 0) { + QLOG("Couldn't register the device\n"); + IntelQaClose(&qaDev); + IntelQaDeInit(devId); + devId = INVALID_DEVID; + } + } + } + + return devId; +} + + +void wc_CryptoCb_CleanupIntelQa(int* id) +{ + if (INVALID_DEVID != *id) { + wc_CryptoCb_UnRegisterDevice(*id); + IntelQaClose(&qaDev); + IntelQaDeInit(*id); + *id = INVALID_DEVID; + } +} + #endif /* WOLF_CRYPTO_CB */ @@ -1451,7 +1694,7 @@ static void* _qaeMemAlloc(size_t size, void* heap, int type } /* Public Functions */ -void* IntelQaMalloc(size_t size, void* heap, int type +void* wc_CryptoCb_IntelQaMalloc(size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif @@ -1480,7 +1723,7 @@ void* IntelQaMalloc(size_t size, void* heap, int type return ptr; } -void IntelQaFree(void *ptr, void* heap, int type +void wc_CryptoCb_IntelQaFree(void *ptr, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif @@ -1505,7 +1748,7 @@ void IntelQaFree(void *ptr, void* heap, int type #endif } -void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type +void* wc_CryptoCb_IntelQaRealloc(void *ptr, size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif diff --git a/wolfcrypt/src/sha.c b/wolfcrypt/src/sha.c index 1eee40bc3..cdd7146c3 100644 --- a/wolfcrypt/src/sha.c +++ b/wolfcrypt/src/sha.c @@ -499,6 +499,7 @@ int wc_ShaUpdate(wc_Sha* sha, const byte* data, word32 len) ret = wc_CryptoCb_ShaHash(sha, data, len, NULL); if (ret != CRYPTOCB_UNAVAILABLE) return ret; + ret = 0; /* reset ret */ /* fall-through when unavailable */ } #endif @@ -653,6 +654,7 @@ int wc_ShaFinal(wc_Sha* sha, byte* hash) ret = wc_CryptoCb_ShaHash(sha, NULL, 0, hash); if (ret != CRYPTOCB_UNAVAILABLE) return ret; + ret = 0; /* reset ret */ /* fall-through when unavailable */ } #endif diff --git a/wolfcrypt/src/wc_port.c b/wolfcrypt/src/wc_port.c index b0b0faf83..d24a21569 100644 --- a/wolfcrypt/src/wc_port.c +++ b/wolfcrypt/src/wc_port.c @@ -78,6 +78,10 @@ #include #endif +#ifdef HAVE_CAVIUM_OCTEON_SYNC + #include +#endif + #ifdef _MSC_VER /* 4996 warning to use MS extensions e.g., strcpy_s instead of strncpy */ #pragma warning(disable: 4996) @@ -129,10 +133,6 @@ int wolfCrypt_Init(void) } #endif - #ifdef HAVE_INTEL_QA_SYNC - ret = IntelQaHardwareStart(QAT_PROCESS_NAME, QAT_LIMIT_DEV_ACCESS); - #endif - #if defined(WOLFSSL_TRACK_MEMORY) && !defined(WOLFSSL_STATIC_MEMORY) ret = InitMemoryTracker(); if (ret != 0) { @@ -266,10 +266,6 @@ int wolfCrypt_Cleanup(void) wolfAsync_HardwareStop(); #endif - #ifdef HAVE_INTEL_QA_SYNC - IntelQaHardwareStop(); - #endif - #if defined(WOLFSSL_IMX6_CAAM) || defined(WOLFSSL_IMX6_CAAM_RNG) || \ defined(WOLFSSL_IMX6_CAAM_BLOB) wc_caamFree(); diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index e3078fb48..ad5c2c67b 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -196,6 +196,12 @@ #endif #ifdef WOLF_CRYPTO_CB #include + #ifdef HAVE_INTEL_QA_SYNC + #include + #endif + #ifdef HAVE_CAVIUM_OCTEON_SYNC + #include + #endif #endif #ifdef _MSC_VER @@ -560,6 +566,21 @@ initDefaultName(); (void)devId; #endif /* WOLFSSL_ASYNC_CRYPT */ +#ifdef WOLF_CRYPTO_CB +#ifdef HAVE_INTEL_QA_SYNC + devId = wc_CryptoCb_InitIntelQa(); + if (INVALID_DEVID == devId) { + printf("Couldn't init the Intel QA\n"); + } +#endif +#ifdef HAVE_CAVIUM_OCTEON_SYNC + devId = wc_CryptoCb_InitOcteon(); + if (INVALID_DEVID == devId) { + printf("Couldn't init the Cavium Octeon\n"); + } +#endif +#endif + #ifdef HAVE_SELFTEST if ( (ret = wolfCrypt_SelfTest()) != 0) return err_sys("CAVP selftest failed!\n", ret); @@ -847,7 +868,9 @@ initDefaultName(); if ( (ret = aesgcm_test()) != 0) return err_sys("AES-GCM test failed!\n", ret); #endif - #if !defined(WOLFSSL_AFALG_XILINX_AES) && !defined(WOLFSSL_XILINX_CRYPT) + #if !defined(WOLFSSL_AFALG_XILINX_AES) && !defined(WOLFSSL_XILINX_CRYPT) && \ + !(defined(WOLF_CRYPTO_CB) && \ + (defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC))) if ((ret = aesgcm_default_test()) != 0) { return err_sys("AES-GCM test failed!\n", ret); } @@ -1108,7 +1131,8 @@ initDefaultName(); test_pass("blob test passed!\n"); #endif -#ifdef WOLF_CRYPTO_CB +#if defined(WOLF_CRYPTO_CB) && \ + !(defined(HAVE_INTEL_QAT_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)) if ( (ret = cryptocb_test()) != 0) return err_sys("crypto callback test failed!\n", ret); else @@ -1122,6 +1146,15 @@ initDefaultName(); test_pass("cert piv test passed!\n"); #endif +#ifdef WOLF_CRYPTO_CB +#ifdef HAVE_INTEL_QA_SYNC + wc_CryptoCb_CleanupIntelQa(&devId); +#endif +#ifdef HAVE_CAVIUM_OCTEON_SYNC + wc_CryptoCb_CleanupOcteon(&devId); +#endif +#endif + #ifdef WOLFSSL_ASYNC_CRYPT wolfAsync_DevClose(&devId); #endif @@ -7202,7 +7235,6 @@ int aesgcm_default_test(void) 0x3a, 0x7a, 0x56, 0x05, 0x09, 0xa2, 0xd9, 0xf2 }; - byte key2[] = { 0x01, 0x6d, 0xbb, 0x38, 0xda, 0xa7, 0x6d, 0xfe, 0x7d, 0xa3, 0x84, 0xeb, 0xf1, 0x24, 0x03, 0x64 @@ -7250,7 +7282,6 @@ int aesgcm_default_test(void) if (ret != 0) { return ret; } - ret = aesgcm_default_test_helper(key2, sizeof(key2), iv2, sizeof(iv2), plain2, sizeof(plain2), cipher2, sizeof(cipher2), NULL, 0, tag2, sizeof(tag2)); @@ -7336,7 +7367,9 @@ int aesgcm_test(void) #if !defined(HAVE_FIPS) && \ !defined(WOLFSSL_PIC32MZ_CRYPT) && \ !defined(FREESCALE_LTC) && !defined(FREESCALE_MMCAU) && \ - !defined(WOLFSSL_XILINX_CRYPT) && !defined(WOLFSSL_AFALG_XILINX_AES) + !defined(WOLFSSL_XILINX_CRYPT) && !defined(WOLFSSL_AFALG_XILINX_AES) && \ + !(defined(WOLF_CRYPTO_CB) && \ + (defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC))) #define ENABLE_NON_12BYTE_IV_TEST #ifdef WOLFSSL_AES_192 @@ -7542,6 +7575,7 @@ int aesgcm_test(void) } #endif +#if !(defined(WOLF_CRYPTO_CB) && defined(HAVE_INTEL_QA_SYNC)) /* Variable authenticated data length test */ for (alen=0; alen<(int)sizeof(p); alen++) { /* AES-GCM encrypt and decrypt both use AES encrypt internally */ @@ -7562,6 +7596,7 @@ int aesgcm_test(void) return -5713; #endif /* HAVE_AES_DECRYPT */ } +#endif #if !defined(WOLFSSL_AFALG_XILINX_AES) && !defined(WOLFSSL_XILINX_CRYPT) #ifdef BENCH_AESGCM_LARGE @@ -7680,7 +7715,9 @@ int aesgcm_test(void) #endif /* ENABLE_NON_12BYTE_IV_TEST */ #if defined(WOLFSSL_AES_256) && !defined(WOLFSSL_AFALG_XILINX_AES) && \ - !defined(WOLFSSL_XILINX_CRYPT) + !defined(WOLFSSL_XILINX_CRYPT) && \ + !(defined(WOLF_CRYPTO_CB) && \ + defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)) XMEMSET(resultT, 0, sizeof(resultT)); XMEMSET(resultC, 0, sizeof(resultC)); XMEMSET(resultP, 0, sizeof(resultP)); @@ -7715,7 +7752,8 @@ int aesgcm_test(void) #if !defined(HAVE_FIPS) || \ (defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2)) /* Test encrypt with internally generated IV */ -#if defined(WOLFSSL_AES_256) && !(defined(WC_NO_RNG) || defined(HAVE_SELFTEST)) +#if defined(WOLFSSL_AES_256) && !(defined(WC_NO_RNG) || defined(HAVE_SELFTEST)) \ + && !(defined(WOLF_CRYPTO_CB) && defined(HAVE_CAVIUM_OCTEON_SYNC)) { WC_RNG rng; byte randIV[12]; @@ -7841,6 +7879,7 @@ int gmac_test(void) byte tag[16]; XMEMSET(&gmac, 0, sizeof(Gmac)); /* clear context */ + wc_AesInit((Aes*)&gmac, HEAP_HINT, INVALID_DEVID); /* Make sure devId updated */ XMEMSET(tag, 0, sizeof(tag)); wc_GmacSetKey(&gmac, k1, sizeof(k1)); wc_GmacUpdate(&gmac, iv1, sizeof(iv1), a1, sizeof(a1), tag, sizeof(t1)); diff --git a/wolfssl/wolfcrypt/aes.h b/wolfssl/wolfcrypt/aes.h index 50ba80246..a9547cff9 100644 --- a/wolfssl/wolfcrypt/aes.h +++ b/wolfssl/wolfcrypt/aes.h @@ -159,6 +159,9 @@ struct Aes { /* key-based fast multiplication table. */ ALIGN16 byte M0[256][AES_BLOCK_SIZE]; #endif /* GCM_TABLE */ +#ifdef HAVE_CAVIUM_OCTEON_SYNC + word32 y0; +#endif #endif /* HAVE_AESGCM */ #ifdef WOLFSSL_AESNI byte use_aesni; @@ -197,6 +200,9 @@ struct Aes { (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))) || \ (defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)) word32 devKey[AES_MAX_KEY_SIZE/WOLFSSL_BIT_SIZE/sizeof(word32)]; /* raw key */ +#ifdef HAVE_CAVIUM_OCTEON_SYNC + int keySet; +#endif #endif #if defined(WOLFSSL_DEVCRYPTO) && \ (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC)) diff --git a/wolfssl/wolfcrypt/include.am b/wolfssl/wolfcrypt/include.am index 9b9d76588..56f5efe96 100644 --- a/wolfssl/wolfcrypt/include.am +++ b/wolfssl/wolfcrypt/include.am @@ -106,6 +106,10 @@ if BUILD_CAVIUM nobase_include_HEADERS+= wolfssl/wolfcrypt/port/cavium/cavium_nitrox.h endif +if BUILD_OCTEON_SYNC +nobase_include_HEADERS+= wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h +endif + if BUILD_INTEL_QA nobase_include_HEADERS+= wolfssl/wolfcrypt/port/intel/quickassist.h nobase_include_HEADERS+= wolfssl/wolfcrypt/port/intel/quickassist_mem.h diff --git a/wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h b/wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h new file mode 100644 index 000000000..8e28477d0 --- /dev/null +++ b/wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h @@ -0,0 +1,31 @@ +/* cavium_octeon_sync.h + * + * Copyright (C) 2006-2019 wolfSSL Inc. + * + * This file is part of wolfSSL. (formerly known as CyaSSL) + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#ifndef _CAVIUM_OCTEON_SYNC_H_ +#define _CAVIUM_OCTEON_SYNC_H_ + +#ifdef HAVE_CAVIUM_OCTEON_SYNC + +WOLFSSL_API int wc_CryptoCb_InitOcteon(void); +WOLFSSL_API void wc_CryptoCb_CleanupOcteon(int* id); + +#endif /* HAVE_CAVIUM_OCTEON_SYNC */ +#endif /* _CAVIUM_OCTEON_SYNC_H_ */ diff --git a/wolfssl/wolfcrypt/port/intel/quickassist_sync.h b/wolfssl/wolfcrypt/port/intel/quickassist_sync.h index 974d2b500..ff87b2ac3 100644 --- a/wolfssl/wolfcrypt/port/intel/quickassist_sync.h +++ b/wolfssl/wolfcrypt/port/intel/quickassist_sync.h @@ -24,206 +24,30 @@ #ifdef HAVE_INTEL_QA_SYNC -#include "cpa.h" -#include "cpa_cy_im.h" -#include "cpa_cy_sym.h" -#include "cpa_cy_rsa.h" -#include "cpa_cy_ln.h" -#include "cpa_cy_ecdh.h" -#include "cpa_cy_ecdsa.h" -#include "cpa_cy_dh.h" -#include "cpa_cy_drbg.h" -#include "cpa_cy_nrbg.h" -#include "cpa_cy_prime.h" +WOLFSSL_API int wc_CryptoCb_InitIntelQa(void); +WOLFSSL_API void wc_CryptoCb_CleanupIntelQa(int* id); -/* User space utils */ -#include -#include -#include -#include - - -#if 0 - /* Optional feature for partial QAT hashing support */ - /* This will process updates through hardware instead of caching them */ - #define QAT_HASH_ENABLE_PARTIAL -#endif -#ifdef QAT_HASH_ENABLE_PARTIAL - #define MAX_QAT_HASH_BUFFERS 2 -#endif - -/* Detect QAT driver version */ -#if defined(CPA_CY_API_VERSION_NUM_MAJOR) && CPA_CY_API_VERSION_NUM_MAJOR > 1 - #define QAT_V2 -#endif - -#ifdef QAT_V2 - /* quickassist/utilities/libusdm_drv/qae_mem.h */ - /* Provides user-space API's for accessing NUMA allocated memory through usdm_drv */ - #include "qae_mem.h" -#include "linux/include/qae_mem_utils.h" -#endif - -#ifdef QAT_USE_POLLING_THREAD - #include -#endif -#ifdef QA_DEMO_MAIN - #include -#endif - - -/* Tunable parameters */ -#ifndef QAT_PROCESS_NAME - #define QAT_PROCESS_NAME "SSL" -#endif -#ifndef QAT_LIMIT_DEV_ACCESS - #define QAT_LIMIT_DEV_ACCESS CPA_FALSE -#endif -#ifndef QAT_MAX_DEVICES - #define QAT_MAX_DEVICES (1) /* maximum number of QAT cards */ -#endif - -#ifndef QAT_RETRY_LIMIT - #define QAT_RETRY_LIMIT (100) -#endif -#ifndef QAT_POLL_RESP_QUOTA - #define QAT_POLL_RESP_QUOTA (0) /* all pending */ -#endif - -#if !defined(NO_AES) || !defined(NO_DES3) - #define QAT_ENABLE_CRYPTO -#endif - - -/* Pre-declarations */ -struct IntelQaDev; -struct wc_CryptoInfo; -struct WC_BIGINT; -struct WC_RNG; - - -#if defined(QAT_ENABLE_HASH) || defined(QAT_ENABLE_CRYPTO) -/* symmetric context */ -typedef struct IntelQaSymCtx { - CpaCySymOpData opData; - CpaCySymSessionCtx symCtxSrc; - CpaCySymSessionCtx symCtx; - word32 symCtxSize; - - /* flags */ - word32 isOpen:1; - word32 isCopy:1; -} IntelQaSymCtx; -#endif - -typedef void (*IntelQaFreeFunc)(struct IntelQaDev*); - - -/* QuickAssist device */ -typedef struct IntelQaDev { - CpaInstanceHandle handle; - int devId; - void* heap; - - /* callback return info */ - int ret; - byte* out; - union { - word32* outLenPtr; - word32 outLen; - }; - - /* operations */ - IntelQaFreeFunc freeFunc; - union { - #ifdef QAT_ENABLE_CRYPTO - struct { - IntelQaSymCtx ctx; - CpaBufferList bufferList; - CpaFlatBuffer flatBuffer; - byte* authTag; - word32 authTagSz; - } cipher; - #endif - } op; - -#ifdef QAT_USE_POLLING_THREAD - pthread_t pollingThread; - byte pollingCy; -#endif -} IntelQaDev; - - -/* Interface */ -WOLFSSL_LOCAL int IntelQaHardwareStart(const char*, int); -WOLFSSL_LOCAL void IntelQaHardwareStop(void); -WOLFSSL_LOCAL int IntelQaInit(void*); -WOLFSSL_LOCAL void IntelQaDeInit(int); -WOLFSSL_LOCAL int IntelQaNumInstances(void); -WOLFSSL_LOCAL int IntelQaOpen(IntelQaDev*, int); -WOLFSSL_LOCAL void IntelQaClose(IntelQaDev*); -WOLFSSL_LOCAL int IntelQaDevCopy(IntelQaDev*, IntelQaDev*); -WOLFSSL_LOCAL int IntelQaPoll(IntelQaDev*); -WOLFSSL_LOCAL int IntelQaGetCyInstanceCount(void); - -#ifndef NO_AES - #ifdef HAVE_AES_CBC - WOLFSSL_LOCAL int IntelQaSymAesCbcEncrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte*, word32); - #ifdef HAVE_AES_DECRYPT - WOLFSSL_LOCAL int IntelQaSymAesCbcDecrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte*, word32); - #endif /* HAVE_AES_DECRYPT */ - #endif /* HAVE_AES_CBC */ - - #ifdef HAVE_AESGCM - WOLFSSL_LOCAL int IntelQaSymAesGcmEncrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte*, word32, - byte*, word32, const byte*, word32); - #ifdef HAVE_AES_DECRYPT - WOLFSSL_LOCAL int IntelQaSymAesGcmDecrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte*, word32, - const byte*, word32, const byte*, word32); - #endif /* HAVE_AES_DECRYPT */ - #endif /* HAVE_AESGCM */ -#endif /* !NO_AES */ - -#ifndef NO_DES3 - WOLFSSL_LOCAL int IntelQaSymDes3CbcEncrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte* iv, word32); - WOLFSSL_LOCAL int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, byte*, - const byte*, word32, const byte*, word32, const byte* iv, word32); -#endif /*! NO_DES3 */ - -#ifdef WOLF_CRYPTO_CB - WOLFSSL_LOCAL int IntelQaSymSync_CryptoDevCb(int, struct wc_CryptoInfo*, - void*); -#endif /* WOLF_CRYPTO_CB */ - - -#ifdef WOLFSSL_TRACK_MEMORY - WOLFSSL_API int InitMemoryTracker(void); - WOLFSSL_API void ShowMemoryTracker(void); -#endif - - -WOLFSSL_API void* IntelQaMalloc(size_t size, void* heap, int type +WOLFSSL_API void* wc_CryptoCb_IntelQaMalloc(size_t size, + void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ); -WOLFSSL_API void IntelQaFree(void *ptr, void* heap, int type +WOLFSSL_API void wc_CryptoCb_IntelQaFree(void *ptr, + void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ); -WOLFSSL_API void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type +WOLFSSL_API void* wc_CryptoCb_IntelQaRealloc(void *ptr, + size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ); + #endif /* HAVE_INTEL_QA_SYNC */ #endif /* _INTEL_QUICKASSIST_SYNC_H_ */ diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 4cedf31b6..483c6a2ce 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -236,20 +236,30 @@ #elif (defined(WOLFSSL_ASYNC_CRYPT) && defined(HAVE_INTEL_QA)) || \ defined(HAVE_INTEL_QA_SYNC) #ifndef HAVE_INTEL_QA_SYNC - #include + #include + #undef USE_WOLFSSL_MEMORY + #ifdef WOLFSSL_DEBUG_MEMORY + #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t), __func__, __LINE__) + #define XFREE(p, h, t) IntelQaFree((p), (h), (t), __func__, __LINE__) + #define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__) + #else + #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t)) + #define XFREE(p, h, t) IntelQaFree((p), (h), (t)) + #define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t)) + #endif /* WOLFSSL_DEBUG_MEMORY */ #else #include + #undef USE_WOLFSSL_MEMORY + #ifdef WOLFSSL_DEBUG_MEMORY + #define XMALLOC(s, h, t) wc_CryptoCb_IntelQaMalloc((s), (h), (t), __func__, __LINE__) + #define XFREE(p, h, t) wc_CryptoCb_IntelQaFree((p), (h), (t), __func__, __LINE__) + #define XREALLOC(p, n, h, t) wc_CryptoCb_IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__) + #else + #define XMALLOC(s, h, t) wc_CryptoCb_IntelQaMalloc((s), (h), (t)) + #define XFREE(p, h, t) wc_CryptoCb_IntelQaFree((p), (h), (t)) + #define XREALLOC(p, n, h, t) wc_CryptoCb_IntelQaRealloc((p), (n), (h), (t)) + #endif /* WOLFSSL_DEBUG_MEMORY */ #endif - #undef USE_WOLFSSL_MEMORY - #ifdef WOLFSSL_DEBUG_MEMORY - #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t), __func__, __LINE__) - #define XFREE(p, h, t) IntelQaFree((p), (h), (t), __func__, __LINE__) - #define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__) - #else - #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t)) - #define XFREE(p, h, t) IntelQaFree((p), (h), (t)) - #define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t)) - #endif /* WOLFSSL_DEBUG_MEMORY */ #elif defined(XMALLOC_USER) /* prototypes for user heap override functions */ #include /* for size_t */ diff --git a/wolfssl/wolfcrypt/wc_port.h b/wolfssl/wolfcrypt/wc_port.h index 225876f22..3a1dd2b8b 100644 --- a/wolfssl/wolfcrypt/wc_port.h +++ b/wolfssl/wolfcrypt/wc_port.h @@ -731,6 +731,16 @@ WOLFSSL_API int wolfCrypt_Cleanup(void); #endif +#ifdef HAVE_NITROX_OCTEON + /* By default, the OCTEON's global variables are all thread local. This + * tag allows them to be shared between threads. */ + #include "cvmx-platform.h" + #define WOLFSSL_GLOBAL CVMX_SHARED +#else + #define WOLFSSL_GLOBAL +#endif + + #ifdef __cplusplus } /* extern "C" */ #endif