diff --git a/configure.ac b/configure.ac index 33fb74625..91c3ab1e0 100644 --- a/configure.ac +++ b/configure.ac @@ -3884,6 +3884,7 @@ AC_ARG_WITH([cavium-v], # Cavium Octeon OCTEON_ROOT="" +: ${OCTEON_OBJ="obj-octeon2"} AC_ARG_WITH([octeon-sync], [AS_HELP_STRING([--with-octeon-sync=PATH],[PATH to Cavium Octeon SDK dir (sync)])], @@ -3903,7 +3904,7 @@ AC_ARG_WITH([octeon-sync], #-I$OCTEON_ROOT/target/include LDFLAGS="$LDFLAGS -lrt -Xlinker -T -Xlinker $OCTEON_ROOT/executive/cvmx-shared-linux.ld" - LIBS="$LIBS $OCTEON_ROOT/executive/obj-octeon3/libcvmx.a $OCTEON_ROOT/executive/obj-octeon3/libfdt.a" + LIBS="$LIBS $OCTEON_ROOT/executive/$OCTEON_OBJ/libcvmx.a $OCTEON_ROOT/executive/$OCTEON_OBJ/libfdt.a" enable_shared=no enable_static=yes diff --git a/src/sniffer.c b/src/sniffer.c index c25e43413..f25f30b71 100644 --- a/src/sniffer.c +++ b/src/sniffer.c @@ -488,9 +488,6 @@ static void UpdateMissedDataSessions(void) #ifdef WOLF_CRYPTO_CB static int CryptoDeviceId = INVALID_DEVID; - #ifdef HAVE_INTEL_QA_SYNC - static IntelQaDev CryptoDevice; - #endif #endif @@ -507,13 +504,13 @@ void ssl_InitSniffer(void) #endif #ifdef WOLF_CRYPTO_CB #ifdef HAVE_INTEL_QA_SYNC - CryptoDeviceId = wc_CryptoCb_InitIntelQa(&CryptoDevice); + CryptoDeviceId = wc_CryptoCb_InitIntelQa(); if (INVALID_DEVID == CryptoDeviceId) { printf("Couldn't init the Intel QA\n"); } #endif #ifdef HAVE_CAVIUM_OCTEON_SYNC - CryptoDeviceId = wc_CryptoCb_InitOcteon(NULL); + CryptoDeviceId = wc_CryptoCb_InitOcteon(); if (INVALID_DEVID == CryptoDeviceId) { printf("Couldn't init the Intel QA\n"); #endif @@ -648,10 +645,10 @@ void ssl_FreeSniffer(void) #ifdef WOLF_CRYPTO_CB #ifdef HAVE_INTEL_QA_SYNC - wc_CryptoCb_CleanupIntelQa(&CryptoDeviceId, &CryptoDevice); + wc_CryptoCb_CleanupIntelQa(&CryptoDeviceId); #endif #ifdef HAVE_CAVIUM_OCTEON_SYNC - wc_CryptoCb_CleanupOcteon(&CryptoDeviceId, NULL); + wc_CryptoCb_CleanupOcteon(&CryptoDeviceId); #endif #endif @@ -1454,8 +1451,8 @@ static int CreateWatchSnifferServer(char* error) return -1; } #ifdef WOLF_CRYPTO_CB - if (CryptoDevId != INVALID_DEVID) - wolfSSL_CTX_SetDevId(sniffer->ctx, CryptoDevId); + if (CryptoDeviceId != INVALID_DEVID) + wolfSSL_CTX_SetDevId(sniffer->ctx, CryptoDeviceId); #endif ServerList = sniffer; diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index cac7e7235..af3da627c 100755 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -693,10 +693,6 @@ static const char* bench_result_words2[][5] = { /* Asynchronous helper macros */ static THREAD_LS_T int devId = INVALID_DEVID; -#if defined(WOLF_CRYPTO_CB) && defined(HAVE_INTEL_QA_SYNC) - static THREAD_LS_T IntelQaDev devQat; -#endif - #ifdef WOLFSSL_ASYNC_CRYPT static WOLF_EVENT_QUEUE eventQueue; @@ -1304,13 +1300,13 @@ static void* benchmarks_do(void* args) #ifdef WOLF_CRYPTO_CB #ifdef HAVE_INTEL_QA_SYNC - devId = wc_CryptoCb_InitIntelQa(&devQat); + devId = wc_CryptoCb_InitIntelQa(); if (devId == INVALID_DEVID) { printf("Couldn't init the Intel QA\n"); } #endif #ifdef HAVE_CAVIUM_OCTEON_SYNC - devId = wc_CryptoCb_InitOcteon(NULL); + devId = wc_CryptoCb_InitOcteon(); if (devId == INVALID_DEVID) { printf("Couldn't get the Octeon device ID\n"); } @@ -1809,10 +1805,10 @@ exit: #ifdef WOLF_CRYPTO_CB #ifdef HAVE_INTEL_QA_SYNC - wc_CryptoCb_CleanupIntelQa(&devId, &devQat); + wc_CryptoCb_CleanupIntelQa(&devId); #endif #ifdef HAVE_CAVIUM_OCTEON_SYNC - wc_CryptoCb_CleanupOcteon(&devId, NULL); + wc_CryptoCb_CleanupOcteon(&devId); #endif #endif diff --git a/wolfcrypt/src/port/cavium/cavium_octeon_sync.c b/wolfcrypt/src/port/cavium/cavium_octeon_sync.c index a287f4345..38aec4455 100644 --- a/wolfcrypt/src/port/cavium/cavium_octeon_sync.c +++ b/wolfcrypt/src/port/cavium/cavium_octeon_sync.c @@ -36,11 +36,28 @@ #define NO_MAIN_DRIVER #include +#include +#include +#include + +#include "cvmx.h" +#include "cvmx-asm.h" +#include "cvmx-key.h" +#include "cvmx-swap.h" + +#ifndef NO_DES3 + #include +#endif +#ifndef NO_AES + #include +#endif + static int devId = 1234; #ifndef NO_DES3 -int Octeon_Des3_CbcEncrypt(Des3* des3, uint64_t *inp64, uint64_t *outp64, size_t inl) +static int Octeon_Des3_CbcEncrypt(Des3* des3, + uint64_t *inp64, uint64_t *outp64, size_t inl) { register uint64_t i0, r0; uint64_t *key, *iv; @@ -115,7 +132,8 @@ int Octeon_Des3_CbcEncrypt(Des3* des3, uint64_t *inp64, uint64_t *outp64, size_t return 0; } -int Octeon_Des3_CbcDecrypt(Des3* des3, uint64_t *inp64, uint64_t *outp64, size_t inl) +static int Octeon_Des3_CbcDecrypt(Des3* des3, + uint64_t *inp64, uint64_t *outp64, size_t inl) { register uint64_t i0, r0; uint64_t *key, *iv; @@ -197,63 +215,9 @@ int Octeon_Des3_CbcDecrypt(Des3* des3, uint64_t *inp64, uint64_t *outp64, size_t #ifndef NO_AES -#ifdef WOLFSSL_AES_DIRECT -/* Perform Single Block ECB Encrypt */ -int Octeon_AesEcb_Encrypt(Aes* aes, const unsigned char *in, unsigned char *out) -{ - uint64_t *in64, *out64, *key; - - if (aes == NULL || in == NULL || out == NULL) { - return BAD_FUNC_ARG; - } - - key = (uint64_t*)aes->devKey; - CVMX_MT_AES_KEY(key[0], 0); - CVMX_MT_AES_KEY(key[1], 1); - CVMX_MT_AES_KEY(key[2], 2); - CVMX_MT_AES_KEY(key[3], 3); - CVMX_MT_AES_KEYLENGTH(aes->keylen/8 - 1); - - in64 = (uint64_t*)in; - out64 = (uint64_t*)out; - - CVMX_MT_AES_ENC0(in64[0]); - CVMX_MT_AES_ENC1(in64[1]); - CVMX_MF_AES_RESULT(out64[0],0); - CVMX_MF_AES_RESULT(out64[1],1); - - return 0; -} - -/* Perform Single Block ECB Decrypt */ -int Octeon_AesEcb_Decrypt(Aes* aes, const unsigned char *in, unsigned char *out) -{ - uint64_t *in64, *out64, *key; - - if (aes == NULL || in == NULL || out == NULL) { - return BAD_FUNC_ARG; - } - - key = (uint64_t*)aes->devKey; - CVMX_MT_AES_KEY(key[0], 0); - CVMX_MT_AES_KEY(key[1], 1); - CVMX_MT_AES_KEY(key[2], 2); - CVMX_MT_AES_KEY(key[3], 3); - CVMX_MT_AES_KEYLENGTH(aes->keylen/8 - 1); - - in64 = (uint64_t*)in; - out64 = (uint64_t*)out; - CVMX_MT_AES_DEC0(in64[0]); - CVMX_MT_AES_DEC1(in64[1]); - CVMX_MF_AES_RESULT(out64[0],0); - CVMX_MF_AES_RESULT(out64[1],1); - - return 0; -} -#endif /* WOLFSSL_AES_DIRECT */ - #ifdef HAVE_AES_CBC -int Octeon_AesCbc_Encrypt(Aes *aes, uint64_t *inp64, uint64_t *outp64, size_t inl) +static int Octeon_AesCbc_Encrypt(Aes *aes, + uint64_t *inp64, uint64_t *outp64, size_t inl) { register uint64_t i0, i1, r0, r1; uint64_t *key, *iv; @@ -358,7 +322,8 @@ int Octeon_AesCbc_Encrypt(Aes *aes, uint64_t *inp64, uint64_t *outp64, size_t in return 0; } -int Octeon_AesCbc_Decrypt(Aes *aes, uint64_t *inp64, uint64_t *outp64, size_t inl) +static int Octeon_AesCbc_Decrypt(Aes *aes, + uint64_t *inp64, uint64_t *outp64, size_t inl) { register uint64_t i0, i1, r0, r1; uint64_t *key, *iv; @@ -458,7 +423,7 @@ int Octeon_AesCbc_Decrypt(Aes *aes, uint64_t *inp64, uint64_t *outp64, size_t in : [r1] "=&d"(in1) , [r2] "=&d"(in2) \ : [r3] "d"(out1), [r4] "d"(out2)) -static inline void Octeon_GHASH_Restore(word16 poly, byte* h) +static void Octeon_GHASH_Restore(word16 poly, byte* h) { word64* bigH = (word64*)h; CVMX_MT_GFM_POLY((word64)poly); @@ -467,7 +432,7 @@ static inline void Octeon_GHASH_Restore(word16 poly, byte* h) } -static inline void Octeon_GHASH_Init(word16 poly, byte* h) +static void Octeon_GHASH_Init(word16 poly, byte* h) { Octeon_GHASH_Restore(poly, h); CVMX_MT_GFM_RESINP(0, 0); @@ -475,7 +440,7 @@ static inline void Octeon_GHASH_Init(word16 poly, byte* h) } -static inline void Octeon_GHASH_Update(byte* in) +static void Octeon_GHASH_Update(byte* in) { word64* bigIn = (word64*)in; CVMX_MT_GFM_XOR0(bigIn[0]); @@ -483,7 +448,7 @@ static inline void Octeon_GHASH_Update(byte* in) } -static inline void Octeon_GHASH_Final(byte* out, word64 authInSz, word64 inSz) +static void Octeon_GHASH_Final(byte* out, word64 authInSz, word64 inSz) { word64* bigOut = (word64*)out; @@ -745,7 +710,7 @@ static int Octeon_AesGcm_Finalize(Aes* aes, word32 inSz, word32 aadSz, } -int Octeon_AesGcm_Encrypt(Aes* aes, byte* in, byte* out, word32 inSz, +static int Octeon_AesGcm_Encrypt(Aes* aes, byte* in, byte* out, word32 inSz, byte* iv, word32 ivSz, byte* aad, word32 aadSz, byte* tag) { int ret = 0; @@ -772,7 +737,7 @@ int Octeon_AesGcm_Encrypt(Aes* aes, byte* in, byte* out, word32 inSz, } -int Octeon_AesGcm_Decrypt(Aes* aes, byte* in, byte* out, word32 inSz, +static int Octeon_AesGcm_Decrypt(Aes* aes, byte* in, byte* out, word32 inSz, byte* iv, word32 ivSz, byte* aad, word32 aadSz, byte* tag) { int ret = 0; @@ -893,10 +858,8 @@ static int myCryptoDevCb(int devIdArg, wc_CryptoInfo* info, void* ctx) return ret; } -int wc_CryptoCb_InitOcteon(void* unused) +int wc_CryptoCb_InitOcteon(void) { - (void)unused; - if (wc_CryptoCb_RegisterDevice(devId, myCryptoDevCb, NULL) < 0) { return INVALID_DEVID; } @@ -904,107 +867,12 @@ int wc_CryptoCb_InitOcteon(void* unused) return devId; } -void wc_CryptoCb_CleanupOcteon(int* id, void* unused) +void wc_CryptoCb_CleanupOcteon(int* id) { - (void)unused; wc_CryptoCb_UnRegisterDevice(*id); *id = INVALID_DEVID; } #endif /* WOLF_CRYPTO_CB */ - -#ifndef NO_MAIN_DRIVER - -#ifndef NO_DES3 -static int des3_test(void) -{ - const byte vector[] = { /* "Now is the time for all " w/o trailing 0 */ - 0x4e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74, - 0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20, - 0x66,0x6f,0x72,0x20,0x61,0x6c,0x6c,0x20 - }; - - byte plain[24]; - byte cipher[24]; - - Des3 enc; - Des3 dec; - - const byte key3[] = - { - 0x01,0x23,0x45,0x67,0x89,0xab,0xcd,0xef, - 0xfe,0xde,0xba,0x98,0x76,0x54,0x32,0x10, - 0x89,0xab,0xcd,0xef,0x01,0x23,0x45,0x67 - }; - const byte iv3[] = - { - 0x12,0x34,0x56,0x78,0x90,0xab,0xcd,0xef, - 0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01, - 0x11,0x21,0x31,0x41,0x51,0x61,0x71,0x81 - - }; - - const byte verify3[] = - { - 0x43,0xa0,0x29,0x7e,0xd1,0x84,0xf8,0x0e, - 0x89,0x64,0x84,0x32,0x12,0xd5,0x08,0x98, - 0x18,0x94,0x15,0x74,0x87,0x12,0x7d,0xb0 - }; - - int ret; - - - if (wc_Des3Init(&enc, NULL, devId) != 0) - return -4700; - if (wc_Des3Init(&dec, NULL, devId) != 0) - return -4701; - - ret = wc_Des3_SetKey(&enc, key3, iv3, DES_ENCRYPTION); - if (ret != 0) - return -4702; - ret = wc_Des3_SetKey(&dec, key3, iv3, DES_DECRYPTION); - if (ret != 0) - return -4703; - ret = wc_Des3_CbcEncrypt(&enc, cipher, vector, sizeof(vector)); - if (ret != 0) - return -4704; - ret = wc_Des3_CbcDecrypt(&dec, plain, cipher, sizeof(cipher)); - if (ret != 0) - return -4705; - - if (XMEMCMP(plain, vector, sizeof(plain))) - return -4706; - - if (XMEMCMP(cipher, verify3, sizeof(cipher))) - return -4707; - - - wc_Des3Free(&enc); - wc_Des3Free(&dec); - - return 0; -} -#endif /* NO_DES */ - -int main(void) -{ - int ret = 0; - - wolfCrypt_Init(); - - /* The following is called in wolfCrypt_Init(). - wc_CryptoCb_InitOcteon(); - */ - -#ifndef NO_DES3 - des3_test(); -#endif - - wolfCrypt_Cleanup(); - - return ret; -} -#endif /* !NO_MAIN_DRIVER */ - #endif /* HAVE_CAVIUM_OCTEON_SYNC */ diff --git a/wolfcrypt/src/port/intel/quickassist_sync.c b/wolfcrypt/src/port/intel/quickassist_sync.c index 1b4679996..a148d0b83 100644 --- a/wolfcrypt/src/port/intel/quickassist_sync.c +++ b/wolfcrypt/src/port/intel/quickassist_sync.c @@ -42,6 +42,18 @@ #include #include +#include "cpa.h" +#include "cpa_cy_im.h" +#include "cpa_cy_sym.h" +#include "cpa_cy_rsa.h" +#include "cpa_cy_ln.h" +#include "cpa_cy_ecdh.h" +#include "cpa_cy_ecdsa.h" +#include "cpa_cy_dh.h" +#include "cpa_cy_drbg.h" +#include "cpa_cy_nrbg.h" +#include "cpa_cy_prime.h" + #include "icp_sal_user.h" #include "icp_sal_poll.h" @@ -53,6 +65,164 @@ #include #endif +/* User space utils */ +#include +#include +#include +#include + +#if 0 + /* Optional feature for partial QAT hashing support */ + /* This will process updates through hardware instead of caching them */ + #define QAT_HASH_ENABLE_PARTIAL +#endif +#ifdef QAT_HASH_ENABLE_PARTIAL + #define MAX_QAT_HASH_BUFFERS 2 +#endif + +/* Detect QAT driver version */ +#if defined(CPA_CY_API_VERSION_NUM_MAJOR) && CPA_CY_API_VERSION_NUM_MAJOR > 1 + #define QAT_V2 +#endif + +#ifdef QAT_V2 + /* quickassist/utilities/libusdm_drv/qae_mem.h */ + /* Provides user-space API's for accessing NUMA allocated memory through usdm_drv */ + #include "qae_mem.h" +#include "linux/include/qae_mem_utils.h" +#endif + +#ifdef QAT_USE_POLLING_THREAD + #include +#endif + +/* Tunable parameters */ +#ifndef QAT_PROCESS_NAME + #define QAT_PROCESS_NAME "SSL" +#endif +#ifndef QAT_LIMIT_DEV_ACCESS + #define QAT_LIMIT_DEV_ACCESS CPA_FALSE +#endif +#ifndef QAT_MAX_DEVICES + #define QAT_MAX_DEVICES (1) /* maximum number of QAT cards */ +#endif + +#ifndef QAT_RETRY_LIMIT + #define QAT_RETRY_LIMIT (100) +#endif +#ifndef QAT_POLL_RESP_QUOTA + #define QAT_POLL_RESP_QUOTA (0) /* all pending */ +#endif + +#if !defined(NO_AES) || !defined(NO_DES3) + #define QAT_ENABLE_CRYPTO +#endif + +/* Pre-declarations */ +struct IntelQaDev; +struct wc_CryptoInfo; +struct WC_BIGINT; +struct WC_RNG; + + +#if defined(QAT_ENABLE_HASH) || defined(QAT_ENABLE_CRYPTO) +/* symmetric context */ +typedef struct IntelQaSymCtx { + CpaCySymOpData opData; + CpaCySymSessionCtx symCtxSrc; + CpaCySymSessionCtx symCtx; + word32 symCtxSize; + + /* flags */ + word32 isOpen:1; + word32 isCopy:1; +} IntelQaSymCtx; +#endif + +typedef void (*IntelQaFreeFunc)(struct IntelQaDev*); + + +/* QuickAssist device */ +typedef struct IntelQaDev { + CpaInstanceHandle handle; + int devId; + void* heap; + + /* callback return info */ + int ret; + byte* out; + union { + word32* outLenPtr; + word32 outLen; + }; + + /* operations */ + IntelQaFreeFunc freeFunc; + union { + #ifdef QAT_ENABLE_CRYPTO + struct { + IntelQaSymCtx ctx; + CpaBufferList bufferList; + CpaFlatBuffer flatBuffer; + byte* authTag; + word32 authTagSz; + } cipher; + #endif + } op; + +#ifdef QAT_USE_POLLING_THREAD + pthread_t pollingThread; + byte pollingCy; +#endif +} IntelQaDev; + + +/* Interface */ +static int IntelQaHardwareStart(const char*, int); +static void IntelQaHardwareStop(void); +static int IntelQaInit(void*); +static void IntelQaDeInit(int); +static int IntelQaNumInstances(void); +static int IntelQaOpen(IntelQaDev*, int); +static void IntelQaClose(IntelQaDev*); +static int IntelQaDevCopy(IntelQaDev*, IntelQaDev*); +static int IntelQaPoll(IntelQaDev*); +static int IntelQaGetCyInstanceCount(void); + +#ifndef NO_AES + #ifdef HAVE_AES_CBC + static int IntelQaSymAesCbcEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32); + #ifdef HAVE_AES_DECRYPT + static int IntelQaSymAesCbcDecrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32); + #endif /* HAVE_AES_DECRYPT */ + #endif /* HAVE_AES_CBC */ + + #ifdef HAVE_AESGCM + static int IntelQaSymAesGcmEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32, + byte*, word32, const byte*, word32); + #ifdef HAVE_AES_DECRYPT + static int IntelQaSymAesGcmDecrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32, + const byte*, word32, const byte*, word32); + #endif /* HAVE_AES_DECRYPT */ + #endif /* HAVE_AESGCM */ +#endif /* !NO_AES */ + +#ifndef NO_DES3 + static int IntelQaSymDes3CbcEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte* iv, word32); + static int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, byte*, + const byte*, word32, const byte*, word32, const byte* iv, word32); +#endif /*! NO_DES3 */ + +#ifdef WOLF_CRYPTO_CB + static int IntelQaSymSync_CryptoDevCb(int, struct wc_CryptoInfo*, + void*); +#endif /* WOLF_CRYPTO_CB */ + #ifdef QAT_DEBUG #define QLOG(...) do { printf(__VA_ARGS__); } while (0) @@ -94,6 +264,9 @@ static qatCapabilities_t g_qatCapabilities = {0}; extern Cpa32U osalLogLevelSet(Cpa32U level); +static IntelQaDev qaDev; + + /* -------------------------------------------------------------------------- */ /* Polling */ /* -------------------------------------------------------------------------- */ @@ -1171,9 +1344,8 @@ int IntelQaSymSync_CryptoDevCb(int devId, struct wc_CryptoInfo* info, void* ctx) /* Public API */ /* -------------------------------------------------------------------------- */ -int wc_CryptoCb_InitIntelQa(void* dev) +int wc_CryptoCb_InitIntelQa(void) { - IntelQaDev* qaDev = (IntelQaDev*)dev; int devId, rc; devId = IntelQaInit(NULL); @@ -1182,7 +1354,7 @@ int wc_CryptoCb_InitIntelQa(void* dev) devId = INVALID_DEVID; } else { - rc = IntelQaOpen(qaDev, devId); + rc = IntelQaOpen(&qaDev, devId); if (rc != 0) { QLOG("Couldn't open the device\n"); IntelQaDeInit(devId); @@ -1190,10 +1362,10 @@ int wc_CryptoCb_InitIntelQa(void* dev) } else { rc = wc_CryptoCb_RegisterDevice(devId, - IntelQaSymSync_CryptoDevCb, qaDev); + IntelQaSymSync_CryptoDevCb, &qaDev); if (rc != 0) { QLOG("Couldn't register the device\n"); - IntelQaClose(qaDev); + IntelQaClose(&qaDev); IntelQaDeInit(devId); devId = INVALID_DEVID; } @@ -1204,13 +1376,11 @@ int wc_CryptoCb_InitIntelQa(void* dev) } -void wc_CryptoCb_CleanupIntelQa(int* id, void* dev) +void wc_CryptoCb_CleanupIntelQa(int* id) { - IntelQaDev* qaDev = (IntelQaDev*)dev; - if (INVALID_DEVID != *id) { wc_CryptoCb_UnRegisterDevice(*id); - IntelQaClose(qaDev); + IntelQaClose(&qaDev); IntelQaDeInit(*id); *id = INVALID_DEVID; } @@ -1524,7 +1694,7 @@ static void* _qaeMemAlloc(size_t size, void* heap, int type } /* Public Functions */ -void* IntelQaMalloc(size_t size, void* heap, int type +void* wc_CryptoCb_IntelQaMalloc(size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif @@ -1553,7 +1723,7 @@ void* IntelQaMalloc(size_t size, void* heap, int type return ptr; } -void IntelQaFree(void *ptr, void* heap, int type +void wc_CryptoCb_IntelQaFree(void *ptr, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif @@ -1578,7 +1748,7 @@ void IntelQaFree(void *ptr, void* heap, int type #endif } -void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type +void* wc_CryptoCb_IntelQaRealloc(void *ptr, size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif diff --git a/wolfcrypt/src/wc_port.c b/wolfcrypt/src/wc_port.c index 18ca54180..d24a21569 100644 --- a/wolfcrypt/src/wc_port.c +++ b/wolfcrypt/src/wc_port.c @@ -133,10 +133,6 @@ int wolfCrypt_Init(void) } #endif - #ifdef HAVE_INTEL_QA_SYNC - ret = IntelQaHardwareStart(QAT_PROCESS_NAME, QAT_LIMIT_DEV_ACCESS); - #endif - #if defined(WOLFSSL_TRACK_MEMORY) && !defined(WOLFSSL_STATIC_MEMORY) ret = InitMemoryTracker(); if (ret != 0) { @@ -270,10 +266,6 @@ int wolfCrypt_Cleanup(void) wolfAsync_HardwareStop(); #endif - #ifdef HAVE_INTEL_QA_SYNC - IntelQaHardwareStop(); - #endif - #if defined(WOLFSSL_IMX6_CAAM) || defined(WOLFSSL_IMX6_CAAM_RNG) || \ defined(WOLFSSL_IMX6_CAAM_BLOB) wc_caamFree(); diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 8973eb909..db7602d4d 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -258,10 +258,6 @@ static void initDefaultName(void); /* for async devices */ static int devId = INVALID_DEVID; -#if defined(WOLF_CRYPTO_CB) && defined(HAVE_INTEL_QA_SYNC) - static THREAD_LS_T IntelQaDev devQat; -#endif - #ifdef HAVE_WNR const char* wnrConfigFile = "wnr-example.conf"; #endif @@ -572,13 +568,13 @@ initDefaultName(); #ifdef WOLF_CRYPTO_CB #ifdef HAVE_INTEL_QA_SYNC - devId = wc_CryptoCb_InitIntelQa(&devQat); + devId = wc_CryptoCb_InitIntelQa(); if (INVALID_DEVID == devId) { printf("Couldn't init the Intel QA\n"); } #endif #ifdef HAVE_CAVIUM_OCTEON_SYNC - devId = wc_CryptoCb_InitOcteon(NULL); + devId = wc_CryptoCb_InitOcteon(); if (INVALID_DEVID == devId) { printf("Couldn't init the Cavium Octeon\n"); } @@ -1150,10 +1146,10 @@ initDefaultName(); #ifdef WOLF_CRYPTO_CB #ifdef HAVE_INTEL_QA_SYNC - wc_CryptoCb_CleanupIntelQa(&devId, &devQat); + wc_CryptoCb_CleanupIntelQa(&devId); #endif #ifdef HAVE_CAVIUM_OCTEON_SYNC - wc_CryptoCb_CleanupOcteon(&devId, NULL); + wc_CryptoCb_CleanupOcteon(&devId); #endif #endif diff --git a/wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h b/wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h index 0da957ccb..8e28477d0 100644 --- a/wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h +++ b/wolfssl/wolfcrypt/port/cavium/cavium_octeon_sync.h @@ -24,63 +24,8 @@ #ifdef HAVE_CAVIUM_OCTEON_SYNC -#ifndef NO_DES3 - #include -#endif -#ifndef NO_AES - #include -#endif - -#include -#include -#include - -#include "cvmx.h" -#include "cvmx-asm.h" -#include "cvmx-key.h" -#include "cvmx-swap.h" - - -#ifdef WOLF_CRYPTO_CB -WOLFSSL_API int wc_CryptoCb_InitOcteon(void* unused); -WOLFSSL_API void wc_CryptoCb_CleanupOcteon(int* id, void* unused); -#endif /* WOLF_CRYPTO_CB */ - - -#ifndef NO_DES3 -WOLFSSL_LOCAL int Octeon_Des3_CbcEncrypt(Des3 *key, - uint64_t *inp64, uint64_t *outp64, size_t inl); -WOLFSSL_LOCAL int Octeon_Des3_CbcDecrypt(Des3 *key, - uint64_t *inp64, uint64_t *outp64, size_t inl); -#endif /* !NO_DES3 */ - - -#ifndef NO_AES - -#ifdef WOLFSSL_AES_DIRECT -WOLFSSL_LOCAL int Octeon_AesEcb_Encrypt(Aes *aes, - const unsigned char *in, unsigned char *out); -WOLFSSL_LOCAL int Octeon_AesEcb_Decrypt(Aes *aes, - const unsigned char *in, unsigned char *out); -#endif - -#ifdef HAVE_AES_CBC -WOLFSSL_LOCAL int Octeon_AesCbc_Encrypt(Aes *aes, - uint64_t *inp64, uint64_t *outp64, size_t inl); -WOLFSSL_LOCAL int Octeon_AesCbc_Decrypt(Aes *aes, - uint64_t *inp64, uint64_t *outp64, size_t inl); -#endif - -#ifdef HAVE_AESGCM -WOLFSSL_LOCAL int Octeon_AesGcm_Encrypt(Aes* aes, - byte* in, byte* out, word32 inSz, - byte* iv, word32 ivSz, byte* aad, word32 aadSz, byte* tag); -WOLFSSL_LOCAL int Octeon_AesGcm_Decrypt(Aes* aes, - byte* in, byte* out, word32 inSz, - byte* iv, word32 ivSz, byte* aad, word32 aadSz, byte* tag); -#endif /* HAVE_AESGCM */ - -#endif /* !NO_AES */ +WOLFSSL_API int wc_CryptoCb_InitOcteon(void); +WOLFSSL_API void wc_CryptoCb_CleanupOcteon(int* id); #endif /* HAVE_CAVIUM_OCTEON_SYNC */ #endif /* _CAVIUM_OCTEON_SYNC_H_ */ diff --git a/wolfssl/wolfcrypt/port/intel/quickassist_sync.h b/wolfssl/wolfcrypt/port/intel/quickassist_sync.h index ed591bb0d..ff87b2ac3 100644 --- a/wolfssl/wolfcrypt/port/intel/quickassist_sync.h +++ b/wolfssl/wolfcrypt/port/intel/quickassist_sync.h @@ -24,210 +24,30 @@ #ifdef HAVE_INTEL_QA_SYNC -#include "cpa.h" -#include "cpa_cy_im.h" -#include "cpa_cy_sym.h" -#include "cpa_cy_rsa.h" -#include "cpa_cy_ln.h" -#include "cpa_cy_ecdh.h" -#include "cpa_cy_ecdsa.h" -#include "cpa_cy_dh.h" -#include "cpa_cy_drbg.h" -#include "cpa_cy_nrbg.h" -#include "cpa_cy_prime.h" +WOLFSSL_API int wc_CryptoCb_InitIntelQa(void); +WOLFSSL_API void wc_CryptoCb_CleanupIntelQa(int* id); -/* User space utils */ -#include -#include -#include -#include - - -#if 0 - /* Optional feature for partial QAT hashing support */ - /* This will process updates through hardware instead of caching them */ - #define QAT_HASH_ENABLE_PARTIAL -#endif -#ifdef QAT_HASH_ENABLE_PARTIAL - #define MAX_QAT_HASH_BUFFERS 2 -#endif - -/* Detect QAT driver version */ -#if defined(CPA_CY_API_VERSION_NUM_MAJOR) && CPA_CY_API_VERSION_NUM_MAJOR > 1 - #define QAT_V2 -#endif - -#ifdef QAT_V2 - /* quickassist/utilities/libusdm_drv/qae_mem.h */ - /* Provides user-space API's for accessing NUMA allocated memory through usdm_drv */ - #include "qae_mem.h" -#include "linux/include/qae_mem_utils.h" -#endif - -#ifdef QAT_USE_POLLING_THREAD - #include -#endif -#ifdef QA_DEMO_MAIN - #include -#endif - - -/* Tunable parameters */ -#ifndef QAT_PROCESS_NAME - #define QAT_PROCESS_NAME "SSL" -#endif -#ifndef QAT_LIMIT_DEV_ACCESS - #define QAT_LIMIT_DEV_ACCESS CPA_FALSE -#endif -#ifndef QAT_MAX_DEVICES - #define QAT_MAX_DEVICES (1) /* maximum number of QAT cards */ -#endif - -#ifndef QAT_RETRY_LIMIT - #define QAT_RETRY_LIMIT (100) -#endif -#ifndef QAT_POLL_RESP_QUOTA - #define QAT_POLL_RESP_QUOTA (0) /* all pending */ -#endif - -#if !defined(NO_AES) || !defined(NO_DES3) - #define QAT_ENABLE_CRYPTO -#endif - - -/* Pre-declarations */ -struct IntelQaDev; -struct wc_CryptoInfo; -struct WC_BIGINT; -struct WC_RNG; - - -#if defined(QAT_ENABLE_HASH) || defined(QAT_ENABLE_CRYPTO) -/* symmetric context */ -typedef struct IntelQaSymCtx { - CpaCySymOpData opData; - CpaCySymSessionCtx symCtxSrc; - CpaCySymSessionCtx symCtx; - word32 symCtxSize; - - /* flags */ - word32 isOpen:1; - word32 isCopy:1; -} IntelQaSymCtx; -#endif - -typedef void (*IntelQaFreeFunc)(struct IntelQaDev*); - - -/* QuickAssist device */ -typedef struct IntelQaDev { - CpaInstanceHandle handle; - int devId; - void* heap; - - /* callback return info */ - int ret; - byte* out; - union { - word32* outLenPtr; - word32 outLen; - }; - - /* operations */ - IntelQaFreeFunc freeFunc; - union { - #ifdef QAT_ENABLE_CRYPTO - struct { - IntelQaSymCtx ctx; - CpaBufferList bufferList; - CpaFlatBuffer flatBuffer; - byte* authTag; - word32 authTagSz; - } cipher; - #endif - } op; - -#ifdef QAT_USE_POLLING_THREAD - pthread_t pollingThread; - byte pollingCy; -#endif -} IntelQaDev; - - -WOLFSSL_API int wc_CryptoCb_InitIntelQa(void* dev); -WOLFSSL_API void wc_CryptoCb_CleanupIntelQa(int* id, void* dev); - - -/* Interface */ -WOLFSSL_LOCAL int IntelQaHardwareStart(const char*, int); -WOLFSSL_LOCAL void IntelQaHardwareStop(void); -WOLFSSL_LOCAL int IntelQaInit(void*); -WOLFSSL_LOCAL void IntelQaDeInit(int); -WOLFSSL_LOCAL int IntelQaNumInstances(void); -WOLFSSL_LOCAL int IntelQaOpen(IntelQaDev*, int); -WOLFSSL_LOCAL void IntelQaClose(IntelQaDev*); -WOLFSSL_LOCAL int IntelQaDevCopy(IntelQaDev*, IntelQaDev*); -WOLFSSL_LOCAL int IntelQaPoll(IntelQaDev*); -WOLFSSL_LOCAL int IntelQaGetCyInstanceCount(void); - -#ifndef NO_AES - #ifdef HAVE_AES_CBC - WOLFSSL_LOCAL int IntelQaSymAesCbcEncrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte*, word32); - #ifdef HAVE_AES_DECRYPT - WOLFSSL_LOCAL int IntelQaSymAesCbcDecrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte*, word32); - #endif /* HAVE_AES_DECRYPT */ - #endif /* HAVE_AES_CBC */ - - #ifdef HAVE_AESGCM - WOLFSSL_LOCAL int IntelQaSymAesGcmEncrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte*, word32, - byte*, word32, const byte*, word32); - #ifdef HAVE_AES_DECRYPT - WOLFSSL_LOCAL int IntelQaSymAesGcmDecrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte*, word32, - const byte*, word32, const byte*, word32); - #endif /* HAVE_AES_DECRYPT */ - #endif /* HAVE_AESGCM */ -#endif /* !NO_AES */ - -#ifndef NO_DES3 - WOLFSSL_LOCAL int IntelQaSymDes3CbcEncrypt(IntelQaDev*, byte*, - const byte*, word32, const byte*, word32, const byte* iv, word32); - WOLFSSL_LOCAL int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, byte*, - const byte*, word32, const byte*, word32, const byte* iv, word32); -#endif /*! NO_DES3 */ - -#ifdef WOLF_CRYPTO_CB - WOLFSSL_LOCAL int IntelQaSymSync_CryptoDevCb(int, struct wc_CryptoInfo*, - void*); -#endif /* WOLF_CRYPTO_CB */ - - -#ifdef WOLFSSL_TRACK_MEMORY - WOLFSSL_LOCAL int InitMemoryTracker(void); - WOLFSSL_LOCAL void ShowMemoryTracker(void); -#endif - - -WOLFSSL_LOCAL void* IntelQaMalloc(size_t size, void* heap, int type +WOLFSSL_API void* wc_CryptoCb_IntelQaMalloc(size_t size, + void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ); -WOLFSSL_LOCAL void IntelQaFree(void *ptr, void* heap, int type +WOLFSSL_API void wc_CryptoCb_IntelQaFree(void *ptr, + void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ); -WOLFSSL_LOCAL void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type +WOLFSSL_API void* wc_CryptoCb_IntelQaRealloc(void *ptr, + size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ); + #endif /* HAVE_INTEL_QA_SYNC */ #endif /* _INTEL_QUICKASSIST_SYNC_H_ */ diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 7c8cc1d7d..efd94b75e 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -236,20 +236,30 @@ #elif (defined(WOLFSSL_ASYNC_CRYPT) && defined(HAVE_INTEL_QA)) || \ defined(HAVE_INTEL_QA_SYNC) #ifndef HAVE_INTEL_QA_SYNC - #include + #include + #undef USE_WOLFSSL_MEMORY + #ifdef WOLFSSL_DEBUG_MEMORY + #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t), __func__, __LINE__) + #define XFREE(p, h, t) IntelQaFree((p), (h), (t), __func__, __LINE__) + #define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__) + #else + #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t)) + #define XFREE(p, h, t) IntelQaFree((p), (h), (t)) + #define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t)) + #endif /* WOLFSSL_DEBUG_MEMORY */ #else #include + #undef USE_WOLFSSL_MEMORY + #ifdef WOLFSSL_DEBUG_MEMORY + #define XMALLOC(s, h, t) wc_CryptoCb_IntelQaMalloc((s), (h), (t), __func__, __LINE__) + #define XFREE(p, h, t) wc_CryptoCb_IntelQaFree((p), (h), (t), __func__, __LINE__) + #define XREALLOC(p, n, h, t) wc_CryptoCb_IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__) + #else + #define XMALLOC(s, h, t) wc_CryptoCb_IntelQaMalloc((s), (h), (t)) + #define XFREE(p, h, t) wc_CryptoCb_IntelQaFree((p), (h), (t)) + #define XREALLOC(p, n, h, t) wc_CryptoCb_IntelQaRealloc((p), (n), (h), (t)) + #endif /* WOLFSSL_DEBUG_MEMORY */ #endif - #undef USE_WOLFSSL_MEMORY - #ifdef WOLFSSL_DEBUG_MEMORY - #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t), __func__, __LINE__) - #define XFREE(p, h, t) IntelQaFree((p), (h), (t), __func__, __LINE__) - #define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t), __func__, __LINE__) - #else - #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t)) - #define XFREE(p, h, t) IntelQaFree((p), (h), (t)) - #define XREALLOC(p, n, h, t) IntelQaRealloc((p), (n), (h), (t)) - #endif /* WOLFSSL_DEBUG_MEMORY */ #elif defined(XMALLOC_USER) /* prototypes for user heap override functions */ #include /* for size_t */