feedc0de/wolfssl
1
0
Fork 0
forked from wolfSSL/wolfssl
Code Pull Requests Activity

Merge pull request #2504 from ejohnstown/sync-octeon

Browse Source 
Synchronous OCTEON Sniffer Support
This commit is contained in:
toddouska
2019-10-28 12:25:57 -07:00
committed by GitHub
parent 0aeb87f284 17f44d4d29
commit 140796d2aa
18 changed files with 1407 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
configure.ac
View File

@ -3882,6 +3882,40 @@ AC_ARG_WITH([cavium-v],
)
# Cavium Octeon
OCTEON_ROOT=""
: ${OCTEON_OBJ="obj-octeon2"}
AC_ARG_WITH([octeon-sync],
[AS_HELP_STRING([--with-octeon-sync=PATH],[PATH to Cavium Octeon SDK dir (sync)])],
[
AC_MSG_CHECKING([for octeon])
if test "x$withval" = "xyes" ; then
AC_MSG_ERROR([need a PATH for --with-octeon])
fi
if test "x$withval" != "xno" ; then
OCTEON_ROOT=$withval
fi
AM_CFLAGS="$AM_CFLAGS -DHAVE_CAVIUM_OCTEON_SYNC"
AM_CFLAGS="$AM_CFLAGS -DOCTEON_MODEL=$OCTEON_MODEL -DCVMX_BUILD_FOR_LINUX_HOST"
AM_CFLAGS="$AM_CFLAGS -I$OCTEON_ROOT/executive"
#-I$OCTEON_ROOT/target/include
AM_LDFLAGS="$AM_LDFLAGS -lrt -Xlinker -T -Xlinker $OCTEON_ROOT/executive/cvmx-shared-linux.ld"
AM_LDFLAGS="$AM_LDFLAGS -L$OCTEON_ROOT/executive/$OCTEON_OBJ -lcvmx -lfdt"
enable_shared=no
enable_static=yes
ENABLED_OCTEON_SYNC=yes
AC_MSG_RESULT([yes])
],
[ENABLED_OCTEON_SYNC=no]
)
# Intel QuickAssist
QAT_DIR=""
BUILD_INTEL_QAT_VERSION=2
@ -4580,9 +4614,10 @@ AS_IF([test "x$ENABLED_OCSP" = "xyes" && \
test "x$ENABLED_ECC" = "xno"],
[AC_MSG_ERROR([please enable rsa or ecc if enabling ocsp.])])
# Sync Intel QA requires the crypto callback
AS_IF([test "x$ENABLED_CRYPTOCB" = "xno" && test "x$ENABLED_INTEL_QA_SYNC" = "xyes"],
[AC_MSG_ERROR([please enable the crypto callback support using --enable-cryptocb])])
# Sync Intel QA and Sync Cavium Octeon require the crypto callback
AS_IF([test "x$ENABLED_INTEL_QA_SYNC" = "xyes" || test "x$ENABLED_OCTEON_SYNC" = "xyes"],
[AS_IF([test "x$ENABLED_CRYPTOCB" = "xno"],
[AC_MSG_ERROR([please enable the crypto callback support using --enable-cryptocb])])])
# checks for pkcs7 needed enables
AS_IF([test "x$ENABLED_PKCS7" = "xyes" && \
@ -4900,6 +4935,7 @@ AM_CONDITIONAL([BUILD_LIBZ],[test "x$ENABLED_LIBZ" = "xyes"])
AM_CONDITIONAL([BUILD_PKCS11],[test "x$ENABLED_PKCS11" = "xyes"])
AM_CONDITIONAL([BUILD_CAVIUM],[test "x$ENABLED_CAVIUM" = "xyes"])
AM_CONDITIONAL([BUILD_CAVIUM_V],[test "x$ENABLED_CAVIUM_V" = "xyes"])
AM_CONDITIONAL([BUILD_OCTEON_SYNC],[test "x$ENABLED_OCTEON_SYNC" = "xyes"])
AM_CONDITIONAL([BUILD_INTEL_QA],[test "x$ENABLED_INTEL_QA" = "xyes"])
AM_CONDITIONAL([BUILD_INTEL_QA_SYNC],[test "x$ENABLED_INTEL_QA_SYNC" = "xyes"])
AM_CONDITIONAL([BUILD_SP],[test "x$ENABLED_SP" = "xyes"])
@ -5191,16 +5227,17 @@ echo " * Single Precision: $ENABLED_SP"
echo " * Async Crypto: $ENABLED_ASYNCCRYPT"
echo " * PKCS#11: $ENABLED_PKCS11"
echo " * PKCS#12: $ENABLED_PKCS12"
echo " * Cavium: $ENABLED_CAVIUM"
echo " * Cavium Nitox: $ENABLED_CAVIUM"
echo " * Cavium Octeon (Sync): $ENABLED_OCTEON_SYNC"
echo " * Intel Quick Assist: $ENABLED_INTEL_QA"
echo " * ARM ASM: $ENABLED_ARMASM"
echo " * AES Key Wrap: $ENABLED_AESKEYWRAP"
echo " * Write duplicate: $ENABLED_WRITEDUP"
echo " * Intel Quick Assist: $ENABLED_INTEL_QA"
echo " * Xilinx Hardware Acc.: $ENABLED_XILINX"
echo " * Inline Code: $ENABLED_INLINE"
echo " * Linux AF_ALG: $ENABLED_AFALG"
echo " * Linux devcrypto: $ENABLED_DEVCRYPTO"
echo " * Crypto callback: $ENABLED_CRYPTOCB"
echo " * Crypto callbacks: $ENABLED_CRYPTOCB"
echo ""
echo "---"

51
src/sniffer.c
View File

@ -61,6 +61,9 @@
#ifdef HAVE_INTEL_QA_SYNC
#include <wolfssl/wolfcrypt/port/intel/quickassist_sync.h>
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
#include <wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h>
#endif
#endif
@ -485,9 +488,6 @@ static void UpdateMissedDataSessions(void)
#ifdef WOLF_CRYPTO_CB
static int CryptoDeviceId = INVALID_DEVID;
#ifdef HAVE_INTEL_QA_SYNC
static IntelQaDev CryptoDevice;
#endif
#endif
@ -504,21 +504,15 @@ void ssl_InitSniffer(void)
#endif
#ifdef WOLF_CRYPTO_CB
#ifdef HAVE_INTEL_QA_SYNC
{
int rc;
CryptoDeviceId = IntelQaInit(NULL);
if (CryptoDeviceId == INVALID_DEVID) {
WOLFSSL_MSG("Couldn't init the Intel QA");
}
rc = IntelQaOpen(&CryptoDevice, CryptoDeviceId);
if (rc != 0) {
WOLFSSL_MSG("Couldn't open the device");
}
rc = wc_CryptoCb_RegisterDevice(CryptoDeviceId,
IntelQaSymSync_CryptoDevCb, &CryptoDevice);
if (rc != 0) {
WOLFSSL_MSG("Couldn't register the device");
}
CryptoDeviceId = wc_CryptoCb_InitIntelQa();
if (INVALID_DEVID == CryptoDeviceId) {
printf("Couldn't init the Intel QA\n");
}
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
CryptoDeviceId = wc_CryptoCb_InitOcteon();
if (INVALID_DEVID == CryptoDeviceId) {
printf("Couldn't init the Intel QA\n");
}
#endif
#endif
@ -652,9 +646,10 @@ void ssl_FreeSniffer(void)
#ifdef WOLF_CRYPTO_CB
#ifdef HAVE_INTEL_QA_SYNC
wc_CryptoCb_UnRegisterDevice(CryptoDeviceId);
IntelQaClose(&CryptoDevice);
IntelQaDeInit(CryptoDeviceId);
wc_CryptoCb_CleanupIntelQa(&CryptoDeviceId);
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
wc_CryptoCb_CleanupOcteon(&CryptoDeviceId);
#endif
#endif
@ -1439,7 +1434,7 @@ static int LoadKeyFile(byte** keyBuf, word32* keyBufSz,
#ifdef WOLFSSL_SNIFFER_WATCH
static int CreateWatchSnifferServer(char* error)
static int CreateWatchSnifferServer(char* error, int devId)
{
SnifferServer* sniffer;
@ -1457,7 +1452,8 @@ static int CreateWatchSnifferServer(char* error)
return -1;
}
#ifdef WOLF_CRYPTO_CB
wolfSSL_CTX_SetDevId(sniffer->ctx, CryptoDevId);
if (CryptoDeviceId != INVALID_DEVID)
wolfSSL_CTX_SetDevId(sniffer->ctx, devId);
#endif
ServerList = sniffer;
@ -4508,10 +4504,17 @@ int ssl_ReadResetStatistics(SSLStats* stats)
#ifdef WOLFSSL_SNIFFER_WATCH
int ssl_SetWatchKeyCallback_ex(SSLWatchCb cb, int devId, char* error)
{
WatchCb = cb;
return CreateWatchSnifferServer(error, devId);
}
int ssl_SetWatchKeyCallback(SSLWatchCb cb, char* error)
{
WatchCb = cb;
return CreateWatchSnifferServer(error);
return CreateWatchSnifferServer(error, INVALID_DEVID);
}

45
wolfcrypt/benchmark/benchmark.c
View File

@ -172,6 +172,16 @@
#include <wolfssl/wolfcrypt/error-crypt.h>
#include <wolfssl/wolfcrypt/types.h>
#ifdef WOLF_CRYPTO_CB
#include <wolfssl/wolfcrypt/cryptocb.h>
#ifdef HAVE_INTEL_QA_SYNC
#include <wolfssl/wolfcrypt/port/intel/quickassist_sync.h>
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
#include <wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h>
#endif
#endif
#ifdef WOLFSSL_ASYNC_CRYPT
#include <wolfssl/wolfcrypt/async.h>
#endif
@ -1288,6 +1298,21 @@ static void* benchmarks_do(void* args)
}
#endif
#ifdef WOLF_CRYPTO_CB
#ifdef HAVE_INTEL_QA_SYNC
devId = wc_CryptoCb_InitIntelQa();
if (devId == INVALID_DEVID) {
printf("Couldn't init the Intel QA\n");
}
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
devId = wc_CryptoCb_InitOcteon();
if (devId == INVALID_DEVID) {
printf("Couldn't get the Octeon device ID\n");
}
#endif
#endif
#if defined(HAVE_LOCAL_RNG)
{
int rngRet;
@ -1328,7 +1353,7 @@ static void* benchmarks_do(void* args)
XMEMSET(bench_plain, 0, (size_t)bench_buf_size);
XMEMSET(bench_cipher, 0, (size_t)bench_buf_size);
#ifdef WOLFSSL_ASYNC_CRYPT
#if defined(WOLFSSL_ASYNC_CRYPT) || defined(HAVE_INTEL_QA_SYNC)
bench_key = (byte*)XMALLOC(sizeof(bench_key_buf), HEAP_HINT, DYNAMIC_TYPE_WOLF_BIGINT);
bench_iv = (byte*)XMALLOC(sizeof(bench_iv_buf), HEAP_HINT, DYNAMIC_TYPE_WOLF_BIGINT);
if (bench_key == NULL || bench_iv == NULL) {
@ -1356,7 +1381,8 @@ static void* benchmarks_do(void* args)
#ifndef NO_SW_BENCH
bench_aescbc(0);
#endif
#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES) && \
#if ((defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_3DES)) || \
defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)) && \
!defined(NO_HW_BENCH)
bench_aescbc(1);
#endif
@ -1367,7 +1393,8 @@ static void* benchmarks_do(void* args)
#ifndef NO_SW_BENCH
bench_aesgcm(0);
#endif
#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES) && \
#if ((defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_3DES)) || \
defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)) && \
!defined(NO_HW_BENCH)
bench_aesgcm(1);
#endif
@ -1438,7 +1465,8 @@ static void* benchmarks_do(void* args)
#ifndef NO_SW_BENCH
bench_des(0);
#endif
#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_3DES) && \
#if ((defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_3DES)) || \
defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)) && \
!defined(NO_HW_BENCH)
bench_des(1);
#endif
@ -1775,6 +1803,15 @@ exit:
XFREE(bench_iv, HEAP_HINT, DYNAMIC_TYPE_WOLF_BIGINT);
#endif
#ifdef WOLF_CRYPTO_CB
#ifdef HAVE_INTEL_QA_SYNC
wc_CryptoCb_CleanupIntelQa(&devId);
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
wc_CryptoCb_CleanupOcteon(&devId);
#endif
#endif
#ifdef WOLFSSL_ASYNC_CRYPT
/* free event queue */
wolfEventQueue_Free(&eventQueue);

6
wolfcrypt/src/aes.c
View File

@ -3654,6 +3654,12 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
wc_AesGcmSetKey_ex(aes, key, len, 0);
#endif
#ifdef WOLF_CRYPTO_CB
if (aes->devId != INVALID_DEVID) {
XMEMCPY(aes->devKey, key, len);
}
#endif
#ifdef WOLFSSL_IMX6_CAAM_BLOB
ForceZero(local, sizeof(local));
#endif

2
wolfcrypt/src/cryptocb.c
View File

@ -44,7 +44,7 @@ typedef struct CryptoCb {
CryptoDevCallbackFunc cb;
void* ctx;
} CryptoCb;
static CryptoCb gCryptoDev[MAX_CRYPTO_DEVID_CALLBACKS];
static WOLFSSL_GLOBAL CryptoCb gCryptoDev[MAX_CRYPTO_DEVID_CALLBACKS];
static CryptoCb* wc_CryptoCb_FindDevice(int devId)
{

5
wolfcrypt/src/include.am
View File

@ -105,6 +105,11 @@ src_libwolfssl_la_SOURCES += wolfcrypt/src/port/cavium/cavium_nitrox.c
endif
EXTRA_DIST += wolfcrypt/src/port/cavium/README.md
if BUILD_OCTEON_SYNC
src_libwolfssl_la_SOURCES += wolfcrypt/src/port/cavium/cavium_octeon_sync.c
endif
EXTRA_DIST += wolfcrypt/src/port/cavium/README_Octeon.md
if BUILD_INTEL_QA
src_libwolfssl_la_SOURCES += wolfcrypt/src/port/intel/quickassist.c
src_libwolfssl_la_SOURCES += wolfcrypt/src/port/intel/quickassist_mem.c

3
wolfcrypt/src/port/cavium/README_Octeon.md Normal file
View File

@ -0,0 +1,3 @@
# Cavium Octeon III CN7300
Please contact wolfSSL at info@wolfssl.com to request an evaluation.

879
wolfcrypt/src/port/cavium/cavium_octeon_sync.c Normal file
View File

@ -0,0 +1,879 @@
/* cavium_octeon_sync.c
*
* Copyright(C) 2006-2019 wolfSSL Inc.
*
* This file is part of wolfSSL.(formerly known as CyaSSL)
*
* wolfSSL is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
*(at your option) any later version.
*
* wolfSSL is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <wolfssl/wolfcrypt/settings.h>
#include <wolfssl/wolfcrypt/error-crypt.h>
#include <wolfssl/wolfcrypt/wc_port.h>
#ifdef HAVE_CAVIUM_OCTEON_SYNC
/* Setting NO_MAIN_DRIVER here because this file ends up building
* in the library sources which doesn't have NO_MAIN_DRIVER set,
* as the library expects main to be somehwere else. */
#undef NO_MAIN_DRIVER
#define NO_MAIN_DRIVER
#include <wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include "cvmx.h"
#include "cvmx-asm.h"
#include "cvmx-key.h"
#include "cvmx-swap.h"
#ifndef NO_DES3
#include <wolfssl/wolfcrypt/des3.h>
#endif
#ifndef NO_AES
#include <wolfssl/wolfcrypt/aes.h>
#endif
#define NOOPT __attribute__((optimize("O0")))
static int devId = 1234;
#ifndef NO_DES3
static int Octeon_Des3_CbcEncrypt(Des3* des3,
uint64_t *inp64, uint64_t *outp64, size_t inl)
{
register uint64_t i0, r0;
uint64_t *key, *iv;
if (des3 == NULL || inp64 == NULL || outp64 == NULL)
return BAD_FUNC_ARG;
/* expects 64-bit aligned value */
key = (uint64_t*)des3->devKey;
CVMX_MT_3DES_KEY(key[0], 0);
CVMX_MT_3DES_KEY(key[1], 1);
CVMX_MT_3DES_KEY(key[2], 2);
iv = (uint64_t*)des3->reg;
CVMX_MT_3DES_IV(iv[0]);
CVMX_PREFETCH0(inp64);
i0 = *inp64;
/* DES3 assembly can handle 16-byte chunks */
if (inl >= 16) {
CVMX_MT_3DES_ENC_CBC(i0);
inl -= 8;
inp64++;
outp64++;
if (inl >= 8) {
i0 = inp64[0];
CVMX_MF_3DES_RESULT(r0);
CVMX_MT_3DES_ENC_CBC(i0);
for (;;) {
outp64[-1] = r0;
inl -= 8;
inp64++;
outp64++;
i0 = *inp64;
if (inl < 8)
break;
CVMX_PREFETCH(inp64, 64);
CVMX_MF_3DES_RESULT(r0);
CVMX_MT_3DES_ENC_CBC(i0);
}
}
CVMX_MF_3DES_RESULT(r0);
outp64[-1] = r0;
}
/* remainder */
if (inl > 0) {
uint64_t r = 0;
if (inl <= 8) {
XMEMCPY(&r, inp64, inl);
CVMX_MT_3DES_ENC_CBC(r);
CVMX_MF_3DES_RESULT(*outp64);
}
else {
i0 = *inp64;
CVMX_MT_3DES_ENC_CBC(i0);
CVMX_MF_3DES_RESULT(*outp64);
inp64++, outp64++;
XMEMCPY(&r, inp64, inl);
CVMX_MT_3DES_ENC_CBC(r);
CVMX_MF_3DES_RESULT(*outp64);
}
}
CVMX_MT_3DES_IV(iv[0]);
return 0;
}
static int Octeon_Des3_CbcDecrypt(Des3* des3,
uint64_t *inp64, uint64_t *outp64, size_t inl)
{
register uint64_t i0, r0;
uint64_t *key, *iv;
if (des3 == NULL || inp64 == NULL || outp64 == NULL)
return BAD_FUNC_ARG;
/* expects 64-bit aligned value */
key = (uint64_t*)des3->devKey;
CVMX_MT_3DES_KEY(key[0], 0);
CVMX_MT_3DES_KEY(key[1], 1);
CVMX_MT_3DES_KEY(key[2], 2);
iv = (uint64_t*)des3->reg;
CVMX_MT_3DES_IV(iv[0]);
CVMX_PREFETCH0(inp64);
i0 = *inp64;
/* DES3 assembly can handle 16-byte chunks */
if (inl >= 16) {
CVMX_MT_3DES_DEC_CBC(i0);
inl -= 8;
inp64++;
outp64++;
if (inl >= 8) {
i0 = inp64[0];
CVMX_MF_3DES_RESULT(r0);
CVMX_MT_3DES_DEC_CBC(i0);
for (;;) {
outp64[-1] = r0;
inl -= 8;
inp64++;
outp64++;
i0 = *inp64;
if (inl < 8)
break;
CVMX_PREFETCH(inp64, 64);
CVMX_MF_3DES_RESULT(r0);
CVMX_MT_3DES_DEC_CBC(i0);
}
}
CVMX_MF_3DES_RESULT(r0);
outp64[-1] = r0;
}
/* remainder */
if (inl > 0) {
if (inl <= 8) {
uint64_t r = 0;
XMEMCPY(&r, inp64, inl);
CVMX_MT_3DES_DEC_CBC(r);
CVMX_MF_3DES_RESULT(*outp64);
}
else {
uint64_t r = 0;
i0 = *inp64;
CVMX_MT_3DES_DEC_CBC(i0);
CVMX_MF_3DES_RESULT(*outp64);
inp64++, outp64++;
XMEMCPY(&r, inp64, inl);
CVMX_MT_3DES_DEC_CBC(r);
CVMX_MF_3DES_RESULT(*outp64);
}
}
CVMX_MT_3DES_IV(iv[0]);
return 0;
}
#endif /* !NO_DES3 */
#ifndef NO_AES
#ifdef HAVE_AES_CBC
static int Octeon_AesCbc_Encrypt(Aes *aes,
uint64_t *inp64, uint64_t *outp64, size_t inl)
{
register uint64_t i0, i1, r0, r1;
uint64_t *key, *iv;
if (aes == NULL || inp64 == NULL || outp64 == NULL) {
return BAD_FUNC_ARG;
}
iv = (uint64_t*)aes->reg;
CVMX_MT_AES_IV(iv[0], 0);
CVMX_MT_AES_IV(iv[1], 1);
key = (uint64_t*)aes->devKey;
CVMX_MT_AES_KEY(key[0], 0);
CVMX_MT_AES_KEY(key[1], 1);
CVMX_MT_AES_KEY(key[2], 2);
CVMX_MT_AES_KEY(key[3], 3);
CVMX_MT_AES_KEYLENGTH(aes->keylen/8 - 1);
CVMX_PREFETCH0(inp64);
i0 = inp64[0];
i1 = inp64[1];
/* AES assembly can handle 32-byte chunks */
if (inl >= 32) {
CVMX_MT_AES_ENC_CBC0(i0);
CVMX_MT_AES_ENC_CBC1(i1);
inl -= 16;
inp64 += 2;
outp64 += 2;
if (inl >= 16) {
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
i0 = inp64[0];
i1 = inp64[1];
CVMX_MT_AES_ENC_CBC0(i0);
CVMX_MT_AES_ENC_CBC1(i1);
for (;;) {
outp64[-2] = r0;
outp64[-1] = r1;
outp64 += 2;
inp64 += 2;
inl -= 16;
i0 = inp64[0];
i1 = inp64[1];
if (inl < 16)
break;
CVMX_PREFETCH(inp64, 64);
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
CVMX_MT_AES_ENC_CBC0(i0);
CVMX_MT_AES_ENC_CBC1(i1);
}
}
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
outp64[-2] = r0;
outp64[-1] = r1;
}
/* remainder */
if (inl > 0) {
uint64_t in64[2] = { 0, 0 };
if (inl <= 16) {
XMEMCPY(in64, inp64, inl);
CVMX_MT_AES_ENC_CBC0(in64[0]);
CVMX_MT_AES_ENC_CBC1(in64[1]);
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
outp64[0] = r0;
outp64[1] = r1;
}
else {
CVMX_MT_AES_ENC_CBC0(i0);
CVMX_MT_AES_ENC_CBC1(i1);
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
inl -= 16;
outp64[0] = r0;
outp64[1] = r1;
inp64 += 2;
outp64 += 2;
XMEMCPY(in64, inp64, inl);
CVMX_MT_AES_ENC_CBC0(in64[0]);
CVMX_MT_AES_ENC_CBC1(in64[1]);
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
outp64[0] = r0;
outp64[1] = r1;
}
}
CVMX_MF_AES_IV(iv[0], 0);
CVMX_MF_AES_IV(iv[1], 1);
return 0;
}
static int Octeon_AesCbc_Decrypt(Aes *aes,
uint64_t *inp64, uint64_t *outp64, size_t inl)
{
register uint64_t i0, i1, r0, r1;
uint64_t *key, *iv;
if (aes == NULL || inp64 == NULL || outp64 == NULL) {
return BAD_FUNC_ARG;
}
iv = (uint64_t*)aes->reg;
key = (uint64_t*)aes->devKey;
CVMX_MT_AES_IV(iv[0], 0);
CVMX_MT_AES_IV(iv[1], 1);
CVMX_MT_AES_KEY(key[0], 0);
CVMX_MT_AES_KEY(key[1], 1);
CVMX_MT_AES_KEY(key[2], 2);
CVMX_MT_AES_KEY(key[3], 3);
CVMX_MT_AES_KEYLENGTH(aes->keylen/8 - 1);
CVMX_PREFETCH0(inp64);
i0 = inp64[0];
i1 = inp64[1];
/* AES assembly can handle 32-byte chunks */
if (inl >= 32) {
CVMX_MT_AES_DEC_CBC0(i0);
CVMX_MT_AES_DEC_CBC1(i1);
inp64 += 2;
outp64 += 2;
inl -= 16;
if (inl >= 16) {
i0 = inp64[0];
i1 = inp64[1];
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
CVMX_MT_AES_DEC_CBC0(i0);
CVMX_MT_AES_DEC_CBC1(i1);
for (;;) {
outp64[-2] = r0;
outp64[-1] = r1;
outp64 += 2;
inp64 += 2;
inl -= 16;
i0 = inp64[0];
i1 = inp64[1];
if (inl < 16)
break;
CVMX_PREFETCH(inp64, 64);
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
CVMX_MT_AES_DEC_CBC0(i0);
CVMX_MT_AES_DEC_CBC1(i1);
}
}
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
outp64[-2] = r0;
outp64[-1] = r1;
}
/* remainder */
if (inl > 0) {
uint64_t in64[2] = { 0, 0 };
XMEMCPY(in64, inp64, inl);
CVMX_MT_AES_DEC_CBC0(in64[0]);
CVMX_MT_AES_DEC_CBC1(in64[1]);
CVMX_MF_AES_RESULT(r0, 0);
CVMX_MF_AES_RESULT(r1, 1);
outp64[0] = r0;
outp64[1] = r1;
}
CVMX_MF_AES_IV(iv[0], 0);
CVMX_MF_AES_IV(iv[1], 1);
return 0;
}
#endif /* HAVE_AES_CBC */
#ifdef HAVE_AESGCM
#define CVM_AES_RD_RESULT_WR_DATA(in1, in2, out1, out2) \
asm volatile(\
".set noreorder \n" \
"dmfc2 %[r1],0x0100\n" \
"dmfc2 %[r2],0x0101\n" \
"dmtc2 %[r3],0x010a\n" \
"dmtc2 %[r4],0x310b\n" \
".set reorder \n" \
: [r1] "=&d"(in1) , [r2] "=&d"(in2) \
: [r3] "d"(out1), [r4] "d"(out2))
static NOOPT void Octeon_GHASH_Restore(word16 poly, byte* h)
{
word64* bigH = (word64*)h;
CVMX_MT_GFM_POLY((word64)poly);
CVMX_MT_GFM_MUL(bigH[0], 0);
CVMX_MT_GFM_MUL(bigH[1], 1);
}
static NOOPT void Octeon_GHASH_Init(word16 poly, byte* h)
{
Octeon_GHASH_Restore(poly, h);
CVMX_MT_GFM_RESINP(0, 0);
CVMX_MT_GFM_RESINP(0, 1);
}
static NOOPT void Octeon_GHASH_Update(byte* in)
{
word64* bigIn = (word64*)in;
CVMX_MT_GFM_XOR0(bigIn[0]);
CVMX_MT_GFM_XORMUL1(bigIn[1]);
}
static NOOPT void Octeon_GHASH_Final(byte* out, word64 authInSz, word64 inSz)
{
word64* bigOut = (word64*)out;
CVMX_MT_GFM_XOR0(authInSz * 8);
CVMX_MT_GFM_XORMUL1(inSz * 8);
CVMX_MF_GFM_RESINP(bigOut[0], 0);
CVMX_MF_GFM_RESINP(bigOut[1], 1);
}
/* Sets the Octeon key with the key found in the Aes record. */
static NOOPT int Octeon_AesGcm_SetKey(Aes* aes)
{
int ret = 0;
if (aes == NULL)
ret = BAD_FUNC_ARG;
if (ret == 0) {
uint64_t* key = (uint64_t*)aes->devKey;
CVMX_MT_AES_KEY(key[0], 0);
CVMX_MT_AES_KEY(key[1], 1);
CVMX_MT_AES_KEY(key[2], 2);
CVMX_MT_AES_KEY(key[3], 3);
CVMX_MT_AES_KEYLENGTH((aes->keylen / 8) - 1);
if (!aes->keySet) {
uint64_t* bigH = (uint64_t*)aes->H;
CVMX_MT_AES_ENC0(0);
CVMX_MT_AES_ENC1(0);
CVMX_MF_AES_RESULT(bigH[0], 0);
CVMX_MF_AES_RESULT(bigH[1], 1);
aes->keySet = 1;
}
}
return ret;
}
static NOOPT int Octeon_AesGcm_SetIV(Aes* aes, byte* iv, word32 ivSz)
{
int ret = 0;
if (aes == NULL || iv == NULL)
ret = BAD_FUNC_ARG;
if (ret == 0) {
if (ivSz == GCM_NONCE_MID_SZ) {
XMEMSET((byte*)aes->reg, 0, sizeof(aes->reg));
XMEMCPY((byte*)aes->reg, iv, ivSz);
aes->reg[3] = 1;
}
else {
int blocks, remainder, i;
byte aesBlock[AES_BLOCK_SIZE];
blocks = ivSz / AES_BLOCK_SIZE;
remainder = ivSz % AES_BLOCK_SIZE;
for (i = 0; i < blocks; i++, iv += AES_BLOCK_SIZE)
Octeon_GHASH_Update(iv);
XMEMSET(aesBlock, 0, sizeof(aesBlock));
for (i = 0; i < remainder; i++)
aesBlock[i] = iv[i];
Octeon_GHASH_Update(aesBlock);
Octeon_GHASH_Final((byte*)aes->reg, 0, ivSz);
}
aes->y0 = aes->reg[3];
aes->reg[3]++;
Octeon_GHASH_Init(0xe100, aes->H);
}
return ret;
}
static NOOPT int Octeon_AesGcm_SetAAD(Aes* aes, byte* aad, word32 aadSz)
{
word64* p;
ALIGN16 byte aesBlock[AES_BLOCK_SIZE];
int blocks, remainder, i;
if (aes == NULL || (aadSz != 0 && aad == NULL))
return BAD_FUNC_ARG;
if (aadSz == 0)
return 0;
blocks = aadSz / AES_BLOCK_SIZE;
remainder = aadSz % AES_BLOCK_SIZE;
Octeon_GHASH_Restore(0xe100, aes->H);
p = (word64*)aesBlock;
for (i = 0; i < blocks; i++, aad += AES_BLOCK_SIZE) {
CVMX_LOADUNA_INT64(p[0], aad, 0);
CVMX_LOADUNA_INT64(p[1], aad, 8);
CVMX_MT_GFM_XOR0(p[0]);
CVMX_MT_GFM_XORMUL1(p[1]);
}
XMEMSET(aesBlock, 0, sizeof(aesBlock));
for (i = 0; i < remainder; i++)
aesBlock[i] = aad[i];
CVMX_MT_GFM_XOR0(p[0]);
CVMX_MT_GFM_XORMUL1(p[1]);
return 0;
}
static int Octeon_AesGcm_SetEncrypt(Aes* aes, byte* in, byte* out, word32 inSz,
int encrypt)
{
word32 i, blocks, remainder;
ALIGN16 byte aesBlockIn[AES_BLOCK_SIZE];
ALIGN16 byte aesBlockOut[AES_BLOCK_SIZE];
word64* pIn;
word64* pOut;
word64* pIv;
if (aes == NULL || in == NULL || out == NULL)
return BAD_FUNC_ARG;
pIn = (word64*)aesBlockIn;
pOut = (word64*)aesBlockOut;
pIv = (word64*)aes->reg;
CVMX_PREFETCH0(in);
CVMX_MT_AES_ENC0(pIv[0]);
CVMX_MT_AES_ENC1(pIv[1]);
blocks = inSz / AES_BLOCK_SIZE;
remainder = inSz % AES_BLOCK_SIZE;
for (i = 0; i < blocks;
i++, in += AES_BLOCK_SIZE, out += AES_BLOCK_SIZE) {
CVMX_PREFETCH128(in);
aes->reg[3]++;
CVMX_LOADUNA_INT64(pIn[0], in, 0);
CVMX_LOADUNA_INT64(pIn[1], in, 8);
CVM_AES_RD_RESULT_WR_DATA(pOut[0], pOut[1], pIv[0], pIv[1]);
if (encrypt) {
pOut[0] ^= pIn[0];
pOut[1] ^= pIn[1];
CVMX_MT_GFM_XOR0(pOut[0]);
CVMX_MT_GFM_XORMUL1(pOut[1]);
}
else {
CVMX_MT_GFM_XOR0(pIn[0]);
CVMX_MT_GFM_XORMUL1(pIn[1]);
pOut[0] ^= pIn[0];
pOut[1] ^= pIn[1];
}
CVMX_STOREUNA_INT64(pOut[0], out, 0);
CVMX_STOREUNA_INT64(pOut[1], out, 8);
}
if (remainder > 0) {
ALIGN16 byte aesBlockMask[AES_BLOCK_SIZE];
word64* pMask = (word64*)aesBlockMask;
XMEMSET(aesBlockOut, 0, sizeof(aesBlockOut));
XMEMSET(aesBlockMask, 0, sizeof(aesBlockMask));
for (i = 0; i < remainder; i++) {
aesBlockIn[i] = in[i];
aesBlockMask[i] = 0xFF;
}
if (encrypt) {
CVMX_MF_AES_RESULT(pOut[0], 0);
CVMX_MF_AES_RESULT(pOut[1], 1);
pOut[0] ^= pIn[0];
pOut[1] ^= pIn[1];
pOut[0] &= pMask[0];
pOut[1] &= pMask[1];
CVMX_MT_GFM_XOR0(pOut[0]);
CVMX_MT_GFM_XORMUL1(pOut[1]);
}
else {
CVMX_MT_GFM_XOR0(pIn[0]);
CVMX_MT_GFM_XORMUL1(pIn[1]);
CVMX_MF_AES_RESULT(pOut[0], 0);
CVMX_MF_AES_RESULT(pOut[1], 1);
pOut[0] ^= pIn[0];
pOut[1] ^= pIn[1];
pOut[0] &= pMask[0];
pOut[1] &= pMask[1];
}
for (i = 0; i < remainder; i++)
out[i] = aesBlockOut[i];
}
return 0;
}
static NOOPT int Octeon_AesGcm_Finalize(Aes* aes, word32 inSz, word32 aadSz,
byte* tag)
{
word64 bigSz;
word64* pIv;
word64* pIn;
word64* pOut;
uint32_t countSave;
ALIGN16 byte aesBlockIn[AES_BLOCK_SIZE];
ALIGN16 byte aesBlockOut[AES_BLOCK_SIZE];
countSave = aes->reg[3];
aes->reg[3] = aes->y0;
pIv = (word64*)aes->reg;
CVMX_MT_AES_ENC0(pIv[0]);
CVMX_MT_AES_ENC1(pIv[1]);
bigSz = (word64)aadSz * 8;
CVMX_MT_GFM_XOR0(bigSz);
bigSz = (word64)inSz * 8;
CVMX_MT_GFM_XORMUL1(bigSz);
aes->reg[3] = countSave;
pIn = (word64*)aesBlockIn;
CVMX_MF_AES_RESULT(pIn[0], 0);
CVMX_MF_AES_RESULT(pIn[1], 1);
pOut = (word64*)aesBlockOut;
CVMX_MF_GFM_RESINP(pOut[0], 0);
CVMX_MF_GFM_RESINP(pOut[1], 1);
pOut[0] ^= pIn[0];
pOut[1] ^= pIn[1];
CVMX_STOREUNA_INT64(pOut[0], tag, 0);
CVMX_STOREUNA_INT64(pOut[1], tag, 8);
return 0;
}
static int Octeon_AesGcm_Encrypt(Aes* aes, byte* in, byte* out, word32 inSz,
byte* iv, word32 ivSz, byte* aad, word32 aadSz, byte* tag)
{
int ret = 0;
if (aes == NULL)
ret = BAD_FUNC_ARG;
if (ret == 0)
ret = Octeon_AesGcm_SetKey(aes);
if (ret == 0)
ret = Octeon_AesGcm_SetIV(aes, iv, ivSz);
if (ret == 0)
ret = Octeon_AesGcm_SetAAD(aes, aad, aadSz);
if (ret == 0)
ret = Octeon_AesGcm_SetEncrypt(aes, in, out, inSz, 1);
if (ret == 0)
ret = Octeon_AesGcm_Finalize(aes, inSz, aadSz, tag);
return ret;
}
static int Octeon_AesGcm_Decrypt(Aes* aes, byte* in, byte* out, word32 inSz,
byte* iv, word32 ivSz, byte* aad, word32 aadSz, byte* tag)
{
int ret = 0;
if (aes == NULL)
ret = BAD_FUNC_ARG;
if (ret == 0)
ret = Octeon_AesGcm_SetKey(aes);
if (ret == 0)
ret = Octeon_AesGcm_SetIV(aes, iv, ivSz);
if (ret == 0)
ret = Octeon_AesGcm_SetAAD(aes, aad, aadSz);
if (ret == 0)
ret = Octeon_AesGcm_SetEncrypt(aes, in, out, inSz, 0);
if (ret == 0)
ret = Octeon_AesGcm_Finalize(aes, inSz, aadSz, tag);
return ret;
}
#endif /* HAVE_AESGCM */
#endif /* !NO_AES */
#ifdef WOLF_CRYPTO_CB
#include <wolfssl/wolfcrypt/cryptocb.h>
static int myCryptoDevCb(int devIdArg, wc_CryptoInfo* info, void* ctx)
{
int ret = NOT_COMPILED_IN; /* return this to bypass HW and use SW */
if (info == NULL)
return BAD_FUNC_ARG;
#ifdef DEBUG_WOLFSSL
printf("CryptoDevCb: Algo Type %d\n", info->algo_type);
#endif
if (info->algo_type == WC_ALGO_TYPE_CIPHER) {
#if !defined(NO_AES) || !defined(NO_DES3)
#ifdef HAVE_AESGCM
if (info->cipher.type == WC_CIPHER_AES_GCM) {
if (info->cipher.enc) {
ret = Octeon_AesGcm_Encrypt(
info->cipher.aesgcm_enc.aes,
(byte*)info->cipher.aesgcm_enc.in,
(byte*)info->cipher.aesgcm_enc.out,
info->cipher.aesgcm_enc.sz,
(byte*)info->cipher.aesgcm_enc.iv,
info->cipher.aesgcm_enc.ivSz,
(byte*)info->cipher.aesgcm_enc.authIn,
info->cipher.aesgcm_enc.authInSz,
(byte*)info->cipher.aesgcm_enc.authTag);
}
else {
ret = Octeon_AesGcm_Decrypt(
info->cipher.aesgcm_dec.aes,
(byte*)info->cipher.aesgcm_dec.in,
(byte*)info->cipher.aesgcm_dec.out,
info->cipher.aesgcm_dec.sz,
(byte*)info->cipher.aesgcm_dec.iv,
info->cipher.aesgcm_dec.ivSz,
(byte*)info->cipher.aesgcm_dec.authIn,
info->cipher.aesgcm_dec.authInSz,
(byte*)info->cipher.aesgcm_dec.authTag);
}
}
#endif /* HAVE_AESGCM */
#ifdef HAVE_AES_CBC
if (info->cipher.type == WC_CIPHER_AES_CBC) {
if (info->cipher.enc) {
ret = Octeon_AesCbc_Encrypt(
info->cipher.aescbc.aes,
(word64*)info->cipher.aescbc.in,
(word64*)info->cipher.aescbc.out,
info->cipher.aescbc.sz);
}
else {
ret = Octeon_AesCbc_Decrypt(
info->cipher.aescbc.aes,
(word64*)info->cipher.aescbc.in,
(word64*)info->cipher.aescbc.out,
info->cipher.aescbc.sz);
}
}
#endif /* HAVE_AES_CBC */
#ifndef NO_DES3
if (info->cipher.type == WC_CIPHER_DES3) {
if (info->cipher.enc) {
ret = Octeon_Des3_CbcEncrypt(
info->cipher.des3.des,
(word64*)info->cipher.des3.in,
(word64*)info->cipher.des3.out,
info->cipher.des3.sz);
}
else {
ret = Octeon_Des3_CbcDecrypt(
info->cipher.des3.des,
(word64*)info->cipher.des3.in,
(word64*)info->cipher.des3.out,
info->cipher.des3.sz);
}
}
#endif /* !NO_DES3 */
#endif /* !NO_AES || !NO_DES3 */
}
(void)devIdArg;
(void)ctx;
return ret;
}
int wc_CryptoCb_InitOcteon(void)
{
if (wc_CryptoCb_RegisterDevice(devId, myCryptoDevCb, NULL) < 0) {
return INVALID_DEVID;
}
return devId;
}
void wc_CryptoCb_CleanupOcteon(int* id)
{
wc_CryptoCb_UnRegisterDevice(*id);
*id = INVALID_DEVID;
}
#endif /* WOLF_CRYPTO_CB */
#endif /* HAVE_CAVIUM_OCTEON_SYNC */

295
wolfcrypt/src/port/intel/quickassist_sync.c
View File

@ -42,6 +42,18 @@
#include <wolfssl/wolfcrypt/cryptocb.h>
#include <wolfssl/wolfcrypt/port/intel/quickassist_sync.h>
#include "cpa.h"
#include "cpa_cy_im.h"
#include "cpa_cy_sym.h"
#include "cpa_cy_rsa.h"
#include "cpa_cy_ln.h"
#include "cpa_cy_ecdh.h"
#include "cpa_cy_ecdsa.h"
#include "cpa_cy_dh.h"
#include "cpa_cy_drbg.h"
#include "cpa_cy_nrbg.h"
#include "cpa_cy_prime.h"
#include "icp_sal_user.h"
#include "icp_sal_poll.h"
@ -53,6 +65,164 @@
#include <wolfcrypt/src/misc.c>
#endif
/* User space utils */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#if 0
/* Optional feature for partial QAT hashing support */
/* This will process updates through hardware instead of caching them */
#define QAT_HASH_ENABLE_PARTIAL
#endif
#ifdef QAT_HASH_ENABLE_PARTIAL
#define MAX_QAT_HASH_BUFFERS 2
#endif
/* Detect QAT driver version */
#if defined(CPA_CY_API_VERSION_NUM_MAJOR) && CPA_CY_API_VERSION_NUM_MAJOR > 1
#define QAT_V2
#endif
#ifdef QAT_V2
/* quickassist/utilities/libusdm_drv/qae_mem.h */
/* Provides user-space API's for accessing NUMA allocated memory through usdm_drv */
#include "qae_mem.h"
#include "linux/include/qae_mem_utils.h"
#endif
#ifdef QAT_USE_POLLING_THREAD
#include <pthread.h>
#endif
/* Tunable parameters */
#ifndef QAT_PROCESS_NAME
#define QAT_PROCESS_NAME "SSL"
#endif
#ifndef QAT_LIMIT_DEV_ACCESS
#define QAT_LIMIT_DEV_ACCESS CPA_FALSE
#endif
#ifndef QAT_MAX_DEVICES
#define QAT_MAX_DEVICES (1) /* maximum number of QAT cards */
#endif
#ifndef QAT_RETRY_LIMIT
#define QAT_RETRY_LIMIT (100)
#endif
#ifndef QAT_POLL_RESP_QUOTA
#define QAT_POLL_RESP_QUOTA (0) /* all pending */
#endif
#if !defined(NO_AES) || !defined(NO_DES3)
#define QAT_ENABLE_CRYPTO
#endif
/* Pre-declarations */
struct IntelQaDev;
struct wc_CryptoInfo;
struct WC_BIGINT;
struct WC_RNG;
#if defined(QAT_ENABLE_HASH) || defined(QAT_ENABLE_CRYPTO)
/* symmetric context */
typedef struct IntelQaSymCtx {
CpaCySymOpData opData;
CpaCySymSessionCtx symCtxSrc;
CpaCySymSessionCtx symCtx;
word32 symCtxSize;
/* flags */
word32 isOpen:1;
word32 isCopy:1;
} IntelQaSymCtx;
#endif
typedef void (*IntelQaFreeFunc)(struct IntelQaDev*);
/* QuickAssist device */
typedef struct IntelQaDev {
CpaInstanceHandle handle;
int devId;
void* heap;
/* callback return info */
int ret;
byte* out;
union {
word32* outLenPtr;
word32 outLen;
};
/* operations */
IntelQaFreeFunc freeFunc;
union {
#ifdef QAT_ENABLE_CRYPTO
struct {
IntelQaSymCtx ctx;
CpaBufferList bufferList;
CpaFlatBuffer flatBuffer;
byte* authTag;
word32 authTagSz;
} cipher;
#endif
} op;
#ifdef QAT_USE_POLLING_THREAD
pthread_t pollingThread;
byte pollingCy;
#endif
} IntelQaDev;
/* Interface */
static int IntelQaHardwareStart(const char*, int);
static void IntelQaHardwareStop(void);
static int IntelQaInit(void*);
static void IntelQaDeInit(int);
static int IntelQaNumInstances(void);
static int IntelQaOpen(IntelQaDev*, int);
static void IntelQaClose(IntelQaDev*);
static int IntelQaDevCopy(IntelQaDev*, IntelQaDev*);
static int IntelQaPoll(IntelQaDev*);
static int IntelQaGetCyInstanceCount(void);
#ifndef NO_AES
#ifdef HAVE_AES_CBC
static int IntelQaSymAesCbcEncrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte*, word32);
#ifdef HAVE_AES_DECRYPT
static int IntelQaSymAesCbcDecrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte*, word32);
#endif /* HAVE_AES_DECRYPT */
#endif /* HAVE_AES_CBC */
#ifdef HAVE_AESGCM
static int IntelQaSymAesGcmEncrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte*, word32,
byte*, word32, const byte*, word32);
#ifdef HAVE_AES_DECRYPT
static int IntelQaSymAesGcmDecrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte*, word32,
const byte*, word32, const byte*, word32);
#endif /* HAVE_AES_DECRYPT */
#endif /* HAVE_AESGCM */
#endif /* !NO_AES */
#ifndef NO_DES3
static int IntelQaSymDes3CbcEncrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte* iv, word32);
static int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, byte*,
const byte*, word32, const byte*, word32, const byte* iv, word32);
#endif /*! NO_DES3 */
#ifdef WOLF_CRYPTO_CB
static int IntelQaSymSync_CryptoDevCb(int, struct wc_CryptoInfo*,
void*);
#endif /* WOLF_CRYPTO_CB */
#ifdef QAT_DEBUG
#define QLOG(...) do { printf(__VA_ARGS__); } while (0)
@ -94,6 +264,9 @@ static qatCapabilities_t g_qatCapabilities = {0};
extern Cpa32U osalLogLevelSet(Cpa32U level);
static IntelQaDev qaDev;
/* -------------------------------------------------------------------------- */
/* Polling */
/* -------------------------------------------------------------------------- */
@ -743,7 +916,11 @@ static void IntelQaSymCipherFree(IntelQaDev* dev)
dev->out = NULL;
dev->outLen = 0;
#ifndef NO_AES
dev->op.cipher.authTag = NULL;
if (dev->op.cipher.authTag != NULL) {
XMEMSET(dev->op.cipher.authTag, 0, dev->op.cipher.authTagSz);
XFREE(dev->op.cipher.authTag, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
dev->op.cipher.authTag = NULL;
}
dev->op.cipher.authTagSz = 0;
#endif
}
@ -772,7 +949,9 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in,
Cpa32U metaSize = 0;
Cpa8U* authInBuf = NULL;
Cpa32U authInSzAligned = authInSz;
Cpa8U* authTagBuf = NULL;
IntelQaSymCtx* ctx;
CpaBoolean verifyResult = CPA_FALSE;
QLOG("IntelQaSymCipher: dev %p, out %p, in %p, inOutSz %d, op %d, "
"algo %d, dir %d, hash %d\n",
@ -806,19 +985,15 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in,
bufferList = &dev->op.cipher.bufferList;
flatBuffer = &dev->op.cipher.flatBuffer;
metaBuf = XMALLOC(metaSize, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
#ifndef WOLFSSL_SNIFFER
dataBuf = XREALLOC((byte*)in, dataLen, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
ivBuf = XREALLOC((byte*)iv, AES_BLOCK_SIZE, dev->heap,
DYNAMIC_TYPE_ASYNC_NUMA);
#else
dataBuf = XMALLOC(dataLen, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
XMEMCPY(dataBuf, in, inOutSz);
ivBuf = XMALLOC(AES_BLOCK_SIZE, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
XMEMCPY(ivBuf, iv, ivSz);
#endif
authTagBuf = XMALLOC(authTagSz, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
/* check allocations */
if (ivBuf == NULL || metaBuf == NULL || dataBuf == NULL) {
if (ivBuf == NULL || metaBuf == NULL || dataBuf == NULL ||
authTagBuf == NULL) {
ret = MEMORY_E; goto exit;
}
@ -830,14 +1005,9 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in,
(authInSzAligned % AES_BLOCK_SIZE);
}
#ifndef WOLFSSL_SNIFFER
authInBuf = XREALLOC((byte*)authIn, authInSzAligned, dev->heap,
DYNAMIC_TYPE_ASYNC_NUMA);
#else
authInBuf = XMALLOC(authInSzAligned, dev->heap,
DYNAMIC_TYPE_ASYNC_NUMA);
XMEMCPY(authInBuf, authIn, authInSz);
#endif
if (authInBuf == NULL) {
ret = MEMORY_E; goto exit;
}
@ -878,7 +1048,10 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in,
setup.hashSetupData.digestResultLenInBytes = authTagSz;
setup.hashSetupData.authModeSetupData.aadLenInBytes = authInSz;
setup.digestIsAppended = CPA_TRUE;
if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT)
setup.digestIsAppended = CPA_TRUE;
else
setup.digestIsAppended = CPA_FALSE;
}
/* open session */
@ -903,13 +1076,19 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in,
XMEMCPY(flatBuffer->pData + inOutSz, authTag, authTagSz);
}
}
else {
if (authTag && authTagSz > 0) {
XMEMCPY(authTagBuf, authTag, authTagSz);
}
}
/* store info needed for output */
dev->out = out;
dev->outLen = inOutSz;
if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) {
dev->op.cipher.authTag = authTag;
dev->op.cipher.authTag = authTagBuf;
dev->op.cipher.authTagSz = authTagSz;
opData->pDigestResult = authTagBuf;
}
else {
dev->op.cipher.authTag = NULL;
@ -920,11 +1099,16 @@ static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in,
/* perform symmetric AES operation async */
/* use same buffer list for in-place operation */
status = cpaCySymPerformOp(dev->handle, dev, opData,
bufferList, bufferList, NULL);
if (ret == WC_PENDING_E)
return ret;
bufferList, bufferList, &verifyResult);
if (symOperation == CPA_CY_SYM_OP_ALGORITHM_CHAINING &&
cipherAlgorithm == CPA_CY_SYM_CIPHER_AES_GCM &&
cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT &&
hashAlgorithm == CPA_CY_SYM_HASH_AES_GCM) {
if (verifyResult == CPA_FALSE) {
ret = AES_GCM_AUTH_E;
}
}
exit:
if (ret != 0) {
@ -932,10 +1116,13 @@ exit:
dev, status, ret);
}
#ifdef WOLFSSL_SNIFFER
/* Capture the inline decrypt into the output. */
XMEMCPY(out, dataBuf, inOutSz);
#endif
if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) {
if (authTag != NULL && authTagSz > 0) {
XMEMCPY(authTag, authTagBuf, authTagSz);
}
}
/* handle cleanup */
IntelQaSymCipherFree(dev);
@ -949,11 +1136,14 @@ int IntelQaSymAesCbcEncrypt(IntelQaDev* dev,
const byte* key, word32 keySz,
const byte* iv, word32 ivSz)
{
return IntelQaSymCipher(dev, out, in, sz,
int ret = IntelQaSymCipher(dev, out, in, sz,
key, keySz, iv, ivSz,
CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_AES_CBC,
CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0);
XMEMCPY((byte*)iv, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
return ret;
}
#ifdef HAVE_AES_DECRYPT
@ -962,11 +1152,18 @@ int IntelQaSymAesCbcDecrypt(IntelQaDev* dev,
const byte* key, word32 keySz,
const byte* iv, word32 ivSz)
{
return IntelQaSymCipher(dev, out, in, sz,
byte nextIv[AES_BLOCK_SIZE];
int ret;
XMEMCPY(nextIv, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
ret = IntelQaSymCipher(dev, out, in, sz,
key, keySz, iv, ivSz,
CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_AES_CBC,
CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0);
XMEMCPY((byte*)iv, nextIv, AES_BLOCK_SIZE);
return ret;
}
#endif /* HAVE_AES_DECRYPT */
#endif /* HAVE_AES_CBC */
@ -1143,6 +1340,52 @@ int IntelQaSymSync_CryptoDevCb(int devId, struct wc_CryptoInfo* info, void* ctx)
return rc;
}
/* -------------------------------------------------------------------------- */
/* Public API */
/* -------------------------------------------------------------------------- */
int wc_CryptoCb_InitIntelQa(void)
{
int devId, rc;
devId = IntelQaInit(NULL);
if (devId < 0) {
QLOG("Couldn't init the Intel QA\n");
devId = INVALID_DEVID;
}
else {
rc = IntelQaOpen(&qaDev, devId);
if (rc != 0) {
QLOG("Couldn't open the device\n");
IntelQaDeInit(devId);
devId = INVALID_DEVID;
}
else {
rc = wc_CryptoCb_RegisterDevice(devId,
IntelQaSymSync_CryptoDevCb, &qaDev);
if (rc != 0) {
QLOG("Couldn't register the device\n");
IntelQaClose(&qaDev);
IntelQaDeInit(devId);
devId = INVALID_DEVID;
}
}
}
return devId;
}
void wc_CryptoCb_CleanupIntelQa(int* id)
{
if (INVALID_DEVID != *id) {
wc_CryptoCb_UnRegisterDevice(*id);
IntelQaClose(&qaDev);
IntelQaDeInit(*id);
*id = INVALID_DEVID;
}
}
#endif /* WOLF_CRYPTO_CB */
@ -1451,7 +1694,7 @@ static void* _qaeMemAlloc(size_t size, void* heap, int type
}
/* Public Functions */
void* IntelQaMalloc(size_t size, void* heap, int type
void* wc_CryptoCb_IntelQaMalloc(size_t size, void* heap, int type
#ifdef WOLFSSL_DEBUG_MEMORY
, const char* func, unsigned int line
#endif
@ -1480,7 +1723,7 @@ void* IntelQaMalloc(size_t size, void* heap, int type
return ptr;
}
void IntelQaFree(void *ptr, void* heap, int type
void wc_CryptoCb_IntelQaFree(void *ptr, void* heap, int type
#ifdef WOLFSSL_DEBUG_MEMORY
, const char* func, unsigned int line
#endif
@ -1505,7 +1748,7 @@ void IntelQaFree(void *ptr, void* heap, int type
#endif
}
void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type
void* wc_CryptoCb_IntelQaRealloc(void *ptr, size_t size, void* heap, int type
#ifdef WOLFSSL_DEBUG_MEMORY
, const char* func, unsigned int line
#endif

2
wolfcrypt/src/sha.c
View File

@ -499,6 +499,7 @@ int wc_ShaUpdate(wc_Sha* sha, const byte* data, word32 len)
ret = wc_CryptoCb_ShaHash(sha, data, len, NULL);
if (ret != CRYPTOCB_UNAVAILABLE)
return ret;
ret = 0; /* reset ret */
/* fall-through when unavailable */
}
#endif
@ -653,6 +654,7 @@ int wc_ShaFinal(wc_Sha* sha, byte* hash)
ret = wc_CryptoCb_ShaHash(sha, NULL, 0, hash);
if (ret != CRYPTOCB_UNAVAILABLE)
return ret;
ret = 0; /* reset ret */
/* fall-through when unavailable */
}
#endif

12
wolfcrypt/src/wc_port.c
View File

@ -78,6 +78,10 @@
#include <wolfssl/wolfcrypt/port/intel/quickassist_sync.h>
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
#include <wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h>
#endif
#ifdef _MSC_VER
/* 4996 warning to use MS extensions e.g., strcpy_s instead of strncpy */
#pragma warning(disable: 4996)
@ -129,10 +133,6 @@ int wolfCrypt_Init(void)
}
#endif
#ifdef HAVE_INTEL_QA_SYNC
ret = IntelQaHardwareStart(QAT_PROCESS_NAME, QAT_LIMIT_DEV_ACCESS);
#endif
#if defined(WOLFSSL_TRACK_MEMORY) && !defined(WOLFSSL_STATIC_MEMORY)
ret = InitMemoryTracker();
if (ret != 0) {
@ -266,10 +266,6 @@ int wolfCrypt_Cleanup(void)
wolfAsync_HardwareStop();
#endif
#ifdef HAVE_INTEL_QA_SYNC
IntelQaHardwareStop();
#endif
#if defined(WOLFSSL_IMX6_CAAM) || defined(WOLFSSL_IMX6_CAAM_RNG) || \
defined(WOLFSSL_IMX6_CAAM_BLOB)
wc_caamFree();

53
wolfcrypt/test/test.c
View File

@ -196,6 +196,12 @@
#endif
#ifdef WOLF_CRYPTO_CB
#include <wolfssl/wolfcrypt/cryptocb.h>
#ifdef HAVE_INTEL_QA_SYNC
#include <wolfssl/wolfcrypt/port/intel/quickassist_sync.h>
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
#include <wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h>
#endif
#endif
#ifdef _MSC_VER
@ -560,6 +566,21 @@ initDefaultName();
(void)devId;
#endif /* WOLFSSL_ASYNC_CRYPT */
#ifdef WOLF_CRYPTO_CB
#ifdef HAVE_INTEL_QA_SYNC
devId = wc_CryptoCb_InitIntelQa();
if (INVALID_DEVID == devId) {
printf("Couldn't init the Intel QA\n");
}
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
devId = wc_CryptoCb_InitOcteon();
if (INVALID_DEVID == devId) {
printf("Couldn't init the Cavium Octeon\n");
}
#endif
#endif
#ifdef HAVE_SELFTEST
if ( (ret = wolfCrypt_SelfTest()) != 0)
return err_sys("CAVP selftest failed!\n", ret);
@ -847,7 +868,9 @@ initDefaultName();
if ( (ret = aesgcm_test()) != 0)
return err_sys("AES-GCM test failed!\n", ret);
#endif
#if !defined(WOLFSSL_AFALG_XILINX_AES) && !defined(WOLFSSL_XILINX_CRYPT)
#if !defined(WOLFSSL_AFALG_XILINX_AES) && !defined(WOLFSSL_XILINX_CRYPT) && \
!(defined(WOLF_CRYPTO_CB) && \
(defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)))
if ((ret = aesgcm_default_test()) != 0) {
return err_sys("AES-GCM test failed!\n", ret);
}
@ -1108,7 +1131,8 @@ initDefaultName();
test_pass("blob test passed!\n");
#endif
#ifdef WOLF_CRYPTO_CB
#if defined(WOLF_CRYPTO_CB) && \
!(defined(HAVE_INTEL_QAT_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC))
if ( (ret = cryptocb_test()) != 0)
return err_sys("crypto callback test failed!\n", ret);
else
@ -1122,6 +1146,15 @@ initDefaultName();
test_pass("cert piv test passed!\n");
#endif
#ifdef WOLF_CRYPTO_CB
#ifdef HAVE_INTEL_QA_SYNC
wc_CryptoCb_CleanupIntelQa(&devId);
#endif
#ifdef HAVE_CAVIUM_OCTEON_SYNC
wc_CryptoCb_CleanupOcteon(&devId);
#endif
#endif
#ifdef WOLFSSL_ASYNC_CRYPT
wolfAsync_DevClose(&devId);
#endif
@ -7202,7 +7235,6 @@ int aesgcm_default_test(void)
0x3a, 0x7a, 0x56, 0x05, 0x09, 0xa2, 0xd9, 0xf2
};
byte key2[] = {
0x01, 0x6d, 0xbb, 0x38, 0xda, 0xa7, 0x6d, 0xfe,
0x7d, 0xa3, 0x84, 0xeb, 0xf1, 0x24, 0x03, 0x64
@ -7250,7 +7282,6 @@ int aesgcm_default_test(void)
if (ret != 0) {
return ret;
}
ret = aesgcm_default_test_helper(key2, sizeof(key2), iv2, sizeof(iv2),
plain2, sizeof(plain2), cipher2, sizeof(cipher2),
NULL, 0, tag2, sizeof(tag2));
@ -7336,7 +7367,9 @@ int aesgcm_test(void)
#if !defined(HAVE_FIPS) && \
!defined(WOLFSSL_PIC32MZ_CRYPT) && \
!defined(FREESCALE_LTC) && !defined(FREESCALE_MMCAU) && \
!defined(WOLFSSL_XILINX_CRYPT) && !defined(WOLFSSL_AFALG_XILINX_AES)
!defined(WOLFSSL_XILINX_CRYPT) && !defined(WOLFSSL_AFALG_XILINX_AES) && \
!(defined(WOLF_CRYPTO_CB) && \
(defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC)))
#define ENABLE_NON_12BYTE_IV_TEST
#ifdef WOLFSSL_AES_192
@ -7542,6 +7575,7 @@ int aesgcm_test(void)
}
#endif
#if !(defined(WOLF_CRYPTO_CB) && defined(HAVE_INTEL_QA_SYNC))
/* Variable authenticated data length test */
for (alen=0; alen<(int)sizeof(p); alen++) {
/* AES-GCM encrypt and decrypt both use AES encrypt internally */
@ -7562,6 +7596,7 @@ int aesgcm_test(void)
return -5713;
#endif /* HAVE_AES_DECRYPT */
}
#endif
#if !defined(WOLFSSL_AFALG_XILINX_AES) && !defined(WOLFSSL_XILINX_CRYPT)
#ifdef BENCH_AESGCM_LARGE
@ -7680,7 +7715,9 @@ int aesgcm_test(void)
#endif /* ENABLE_NON_12BYTE_IV_TEST */
#if defined(WOLFSSL_AES_256) && !defined(WOLFSSL_AFALG_XILINX_AES) && \
!defined(WOLFSSL_XILINX_CRYPT)
!defined(WOLFSSL_XILINX_CRYPT) && \
!(defined(WOLF_CRYPTO_CB) && \
defined(HAVE_INTEL_QA_SYNC) || defined(HAVE_CAVIUM_OCTEON_SYNC))
XMEMSET(resultT, 0, sizeof(resultT));
XMEMSET(resultC, 0, sizeof(resultC));
XMEMSET(resultP, 0, sizeof(resultP));
@ -7715,7 +7752,8 @@ int aesgcm_test(void)
#if !defined(HAVE_FIPS) || \
(defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2))
/* Test encrypt with internally generated IV */
#if defined(WOLFSSL_AES_256) && !(defined(WC_NO_RNG) || defined(HAVE_SELFTEST))
#if defined(WOLFSSL_AES_256) && !(defined(WC_NO_RNG) || defined(HAVE_SELFTEST)) \
&& !(defined(WOLF_CRYPTO_CB) && defined(HAVE_CAVIUM_OCTEON_SYNC))
{
WC_RNG rng;
byte randIV[12];
@ -7841,6 +7879,7 @@ int gmac_test(void)
byte tag[16];
XMEMSET(&gmac, 0, sizeof(Gmac)); /* clear context */
wc_AesInit((Aes*)&gmac, HEAP_HINT, INVALID_DEVID); /* Make sure devId updated */
XMEMSET(tag, 0, sizeof(tag));
wc_GmacSetKey(&gmac, k1, sizeof(k1));
wc_GmacUpdate(&gmac, iv1, sizeof(iv1), a1, sizeof(a1), tag, sizeof(t1));

6
wolfssl/wolfcrypt/aes.h
View File

@ -159,6 +159,9 @@ struct Aes {
/* key-based fast multiplication table. */
ALIGN16 byte M0[256][AES_BLOCK_SIZE];
#endif /* GCM_TABLE */
#ifdef HAVE_CAVIUM_OCTEON_SYNC
word32 y0;
#endif
#endif /* HAVE_AESGCM */
#ifdef WOLFSSL_AESNI
byte use_aesni;
@ -197,6 +200,9 @@ struct Aes {
(defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))) || \
(defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES))
word32 devKey[AES_MAX_KEY_SIZE/WOLFSSL_BIT_SIZE/sizeof(word32)]; /* raw key */
#ifdef HAVE_CAVIUM_OCTEON_SYNC
int keySet;
#endif
#endif
#if defined(WOLFSSL_DEVCRYPTO) && \
(defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))

4
wolfssl/wolfcrypt/include.am
View File

@ -106,6 +106,10 @@ if BUILD_CAVIUM
nobase_include_HEADERS+= wolfssl/wolfcrypt/port/cavium/cavium_nitrox.h
endif
if BUILD_OCTEON_SYNC
nobase_include_HEADERS+= wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h
endif
if BUILD_INTEL_QA
nobase_include_HEADERS+= wolfssl/wolfcrypt/port/intel/quickassist.h
nobase_include_HEADERS+= wolfssl/wolfcrypt/port/intel/quickassist_mem.h

31
wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h Normal file
View File

@ -0,0 +1,31 @@
/* cavium_octeon_sync.h
*
* Copyright (C) 2006-2019 wolfSSL Inc.
*
* This file is part of wolfSSL. (formerly known as CyaSSL)
*
* wolfSSL is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* wolfSSL is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*/
#ifndef _CAVIUM_OCTEON_SYNC_H_
#define _CAVIUM_OCTEON_SYNC_H_
#ifdef HAVE_CAVIUM_OCTEON_SYNC
WOLFSSL_API int wc_CryptoCb_InitOcteon(void);
WOLFSSL_API void wc_CryptoCb_CleanupOcteon(int* id);
#endif /* HAVE_CAVIUM_OCTEON_SYNC */
#endif /* _CAVIUM_OCTEON_SYNC_H_ */

194
wolfssl/wolfcrypt/port/intel/quickassist_sync.h
View File

@ -24,206 +24,30 @@
#ifdef HAVE_INTEL_QA_SYNC
#include "cpa.h"
#include "cpa_cy_im.h"
#include "cpa_cy_sym.h"
#include "cpa_cy_rsa.h"
#include "cpa_cy_ln.h"
#include "cpa_cy_ecdh.h"
#include "cpa_cy_ecdsa.h"
#include "cpa_cy_dh.h"
#include "cpa_cy_drbg.h"
#include "cpa_cy_nrbg.h"
#include "cpa_cy_prime.h"
WOLFSSL_API int wc_CryptoCb_InitIntelQa(void);
WOLFSSL_API void wc_CryptoCb_CleanupIntelQa(int* id);
/* User space utils */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#if 0
/* Optional feature for partial QAT hashing support */
/* This will process updates through hardware instead of caching them */
#define QAT_HASH_ENABLE_PARTIAL
#endif
#ifdef QAT_HASH_ENABLE_PARTIAL
#define MAX_QAT_HASH_BUFFERS 2
#endif
/* Detect QAT driver version */
#if defined(CPA_CY_API_VERSION_NUM_MAJOR) && CPA_CY_API_VERSION_NUM_MAJOR > 1
#define QAT_V2
#endif
#ifdef QAT_V2
/* quickassist/utilities/libusdm_drv/qae_mem.h */
/* Provides user-space API's for accessing NUMA allocated memory through usdm_drv */
#include "qae_mem.h"
#include "linux/include/qae_mem_utils.h"
#endif
#ifdef QAT_USE_POLLING_THREAD
#include <pthread.h>
#endif
#ifdef QA_DEMO_MAIN
#include <semaphore.h>
#endif
/* Tunable parameters */
#ifndef QAT_PROCESS_NAME
#define QAT_PROCESS_NAME "SSL"
#endif
#ifndef QAT_LIMIT_DEV_ACCESS
#define QAT_LIMIT_DEV_ACCESS CPA_FALSE
#endif
#ifndef QAT_MAX_DEVICES
#define QAT_MAX_DEVICES (1) /* maximum number of QAT cards */
#endif
#ifndef QAT_RETRY_LIMIT
#define QAT_RETRY_LIMIT (100)
#endif
#ifndef QAT_POLL_RESP_QUOTA
#define QAT_POLL_RESP_QUOTA (0) /* all pending */
#endif
#if !defined(NO_AES) || !defined(NO_DES3)
#define QAT_ENABLE_CRYPTO
#endif
/* Pre-declarations */
struct IntelQaDev;
struct wc_CryptoInfo;
struct WC_BIGINT;
struct WC_RNG;
#if defined(QAT_ENABLE_HASH) || defined(QAT_ENABLE_CRYPTO)
/* symmetric context */
typedef struct IntelQaSymCtx {
CpaCySymOpData opData;
CpaCySymSessionCtx symCtxSrc;
CpaCySymSessionCtx symCtx;
word32 symCtxSize;
/* flags */
word32 isOpen:1;
word32 isCopy:1;
} IntelQaSymCtx;
#endif
typedef void (*IntelQaFreeFunc)(struct IntelQaDev*);
/* QuickAssist device */
typedef struct IntelQaDev {
CpaInstanceHandle handle;
int devId;
void* heap;
/* callback return info */
int ret;
byte* out;
union {
word32* outLenPtr;
word32 outLen;
};
/* operations */
IntelQaFreeFunc freeFunc;
union {
#ifdef QAT_ENABLE_CRYPTO
struct {
IntelQaSymCtx ctx;
CpaBufferList bufferList;
CpaFlatBuffer flatBuffer;
byte* authTag;
word32 authTagSz;
} cipher;
#endif
} op;
#ifdef QAT_USE_POLLING_THREAD
pthread_t pollingThread;
byte pollingCy;
#endif
} IntelQaDev;
/* Interface */
WOLFSSL_LOCAL int IntelQaHardwareStart(const char*, int);
WOLFSSL_LOCAL void IntelQaHardwareStop(void);
WOLFSSL_LOCAL int IntelQaInit(void*);
WOLFSSL_LOCAL void IntelQaDeInit(int);
WOLFSSL_LOCAL int IntelQaNumInstances(void);
WOLFSSL_LOCAL int IntelQaOpen(IntelQaDev*, int);
WOLFSSL_LOCAL void IntelQaClose(IntelQaDev*);
WOLFSSL_LOCAL int IntelQaDevCopy(IntelQaDev*, IntelQaDev*);
WOLFSSL_LOCAL int IntelQaPoll(IntelQaDev*);
WOLFSSL_LOCAL int IntelQaGetCyInstanceCount(void);
#ifndef NO_AES
#ifdef HAVE_AES_CBC
WOLFSSL_LOCAL int IntelQaSymAesCbcEncrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte*, word32);
#ifdef HAVE_AES_DECRYPT
WOLFSSL_LOCAL int IntelQaSymAesCbcDecrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte*, word32);
#endif /* HAVE_AES_DECRYPT */
#endif /* HAVE_AES_CBC */
#ifdef HAVE_AESGCM
WOLFSSL_LOCAL int IntelQaSymAesGcmEncrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte*, word32,
byte*, word32, const byte*, word32);
#ifdef HAVE_AES_DECRYPT
WOLFSSL_LOCAL int IntelQaSymAesGcmDecrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte*, word32,
const byte*, word32, const byte*, word32);
#endif /* HAVE_AES_DECRYPT */
#endif /* HAVE_AESGCM */
#endif /* !NO_AES */
#ifndef NO_DES3
WOLFSSL_LOCAL int IntelQaSymDes3CbcEncrypt(IntelQaDev*, byte*,
const byte*, word32, const byte*, word32, const byte* iv, word32);
WOLFSSL_LOCAL int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, byte*,
const byte*, word32, const byte*, word32, const byte* iv, word32);
#endif /*! NO_DES3 */
#ifdef WOLF_CRYPTO_CB
WOLFSSL_LOCAL int IntelQaSymSync_CryptoDevCb(int, struct wc_CryptoInfo*,
void*);
#endif /* WOLF_CRYPTO_CB */
#ifdef WOLFSSL_TRACK_MEMORY
WOLFSSL_API int InitMemoryTracker(void);
WOLFSSL_API void ShowMemoryTracker(void);
#endif
WOLFSSL_API void* IntelQaMalloc(size_t size, void* heap, int type
WOLFSSL_API void* wc_CryptoCb_IntelQaMalloc(size_t size,
void* heap, int type
#ifdef WOLFSSL_DEBUG_MEMORY
, const char* func, unsigned int line
#endif
);
WOLFSSL_API void IntelQaFree(void *ptr, void* heap, int type
WOLFSSL_API void wc_CryptoCb_IntelQaFree(void *ptr,
void* heap, int type
#ifdef WOLFSSL_DEBUG_MEMORY
, const char* func, unsigned int line
#endif
);
WOLFSSL_API void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type
WOLFSSL_API void* wc_CryptoCb_IntelQaRealloc(void *ptr,
size_t size, void* heap, int type
#ifdef WOLFSSL_DEBUG_MEMORY
, const char* func, unsigned int line
#endif
);
#endif /* HAVE_INTEL_QA_SYNC */
#endif /* _INTEL_QUICKASSIST_SYNC_H_ */

32
wolfssl/wolfcrypt/types.h
View File

@ -236,20 +236,30 @@
#elif (defined(WOLFSSL_ASYNC_CRYPT) && defined(HAVE_INTEL_QA)) || \
defined(HAVE_INTEL_QA_SYNC)
#ifndef HAVE_INTEL_QA_SYNC
#include <wolfssl/wolfcrypt/port/intel/quickassist_mem.h>
#include <wolfssl/wolfcrypt/port/intel/quickassist.h>
#undef USE_WOLFSSL_MEMORY
#ifdef WOLFSSL_DEBUG_MEMORY
#define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t), __func__, __LINE__)
#define XFREE(p, h, t) IntelQaFree((p), (h), (t), __func__, __LINE__)
#define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__)
#else
#define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t))
#define XFREE(p, h, t) IntelQaFree((p), (h), (t))
#define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t))
#endif /* WOLFSSL_DEBUG_MEMORY */
#else
#include <wolfssl/wolfcrypt/port/intel/quickassist_sync.h>
#undef USE_WOLFSSL_MEMORY
#ifdef WOLFSSL_DEBUG_MEMORY
#define XMALLOC(s, h, t) wc_CryptoCb_IntelQaMalloc((s), (h), (t), __func__, __LINE__)
#define XFREE(p, h, t) wc_CryptoCb_IntelQaFree((p), (h), (t), __func__, __LINE__)
#define XREALLOC(p, n, h, t) wc_CryptoCb_IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__)
#else
#define XMALLOC(s, h, t) wc_CryptoCb_IntelQaMalloc((s), (h), (t))
#define XFREE(p, h, t) wc_CryptoCb_IntelQaFree((p), (h), (t))
#define XREALLOC(p, n, h, t) wc_CryptoCb_IntelQaRealloc((p), (n), (h), (t))
#endif /* WOLFSSL_DEBUG_MEMORY */
#endif
#undef USE_WOLFSSL_MEMORY
#ifdef WOLFSSL_DEBUG_MEMORY
#define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t), __func__, __LINE__)
#define XFREE(p, h, t) IntelQaFree((p), (h), (t), __func__, __LINE__)
#define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__)
#else
#define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t))
#define XFREE(p, h, t) IntelQaFree((p), (h), (t))
#define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t))
#endif /* WOLFSSL_DEBUG_MEMORY */
#elif defined(XMALLOC_USER)
/* prototypes for user heap override functions */
#include <stddef.h> /* for size_t */

10
wolfssl/wolfcrypt/wc_port.h
View File

@ -731,6 +731,16 @@ WOLFSSL_API int wolfCrypt_Cleanup(void);
#endif
#ifdef HAVE_NITROX_OCTEON
/* By default, the OCTEON's global variables are all thread local. This
* tag allows them to be shared between threads. */
#include "cvmx-platform.h"
#define WOLFSSL_GLOBAL CVMX_SHARED
#else
#define WOLFSSL_GLOBAL
#endif
#ifdef __cplusplus
} /* extern "C" */
#endif