diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index a3c98370b..3ce0fbcde 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -1094,17 +1094,22 @@ void AES_CBC_encrypt(const unsigned char* in, unsigned char* out, XASM_LINK("AES_CBC_encrypt"); #ifdef HAVE_AES_DECRYPT -#ifndef HAVE_AES_DECRYPT_EX +#if defined(HAVE_AES_DECRYPT_BY8) +void AES_CBC_decrypt(const unsigned char* in, unsigned char* out, + unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) + XASM_LINK("AES_CBC_decrypt_by8"); +#elif defined(HAVE_AES_DECRYPT_BY6) +void AES_CBC_decrypt(const unsigned char* in, unsigned char* out, + unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) + XASM_LINK("AES_CBC_decrypt_by6"); +#else void AES_CBC_decrypt(const unsigned char* in, unsigned char* out, unsigned char* ivec, unsigned long length, const unsigned char* KS, int nr) XASM_LINK("AES_CBC_decrypt"); -#else /* HAVE_AES_DECRYPT_EX */ -void AES_CBC_decrypt_ex(const unsigned char* in, unsigned char* out, - unsigned char* ivec, unsigned long length, - const unsigned char* KS, int nr) - XASM_LINK("AES_CBC_decrypt_ex"); -#endif /* HAVE_AES_DECRYPT_EX */ +#endif /* HAVE_AES_DECRYPT_BYX */ #endif /* HAVE_AES_DECRYPT */ #endif /* HAVE_AES_CBC */ @@ -2556,13 +2561,8 @@ int wc_AesSetIV(Aes* aes, const byte* iv) /* if input and output same will overwrite input iv */ XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE); -#ifndef HAVE_AES_DECRYPT_EX AES_CBC_decrypt(in, out, (byte*)aes->reg, sz, (byte*)aes->key, aes->rounds); -#else /* HAVE_AES_DECRYPT_EX */ - AES_CBC_decrypt_ex(in, out, (byte*)aes->reg, sz, (byte*)aes->key, - aes->rounds); -#endif /* HAVE_AES_DECRYPT_EX */ /* store iv for next call */ XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE); return 0; diff --git a/wolfcrypt/src/aes_asm.s b/wolfcrypt/src/aes_asm.s index c9ae40a18..3120aef22 100644 --- a/wolfcrypt/src/aes_asm.s +++ b/wolfcrypt/src/aes_asm.s @@ -267,15 +267,237 @@ ret /* -AES_CBC_decrypt_ex (const unsigned char *in, +AES_CBC_decrypt_by6 (const unsigned char *in, unsigned char *out, unsigned char ivec[16], unsigned long length, const unsigned char *KS, int nr) */ -.globl AES_CBC_decrypt_ex -AES_CBC_decrypt_ex: +.globl AES_CBC_decrypt_by6 +AES_CBC_decrypt_by6: +# parameter 1: %rdi - in +# parameter 2: %rsi - out +# parameter 3: %rdx - ivec +# parameter 4: %rcx - length +# parameter 5: %r8 - KS +# parameter 6: %r9d - nr + + movq %rcx, %r10 + shrq $4, %rcx + shlq $60, %r10 + je ENO_PARTS_6 + addq $1, %rcx +ENO_PARTS_6: + movq %rax, %r12 + movq %rdx, %r13 + movq %rbx, %r14 + movq $0, %rdx + movq %rcx, %rax + movq $6, %rbx + div %rbx + movq %rax, %rcx + movq %rdx, %r10 + movq %r12, %rax + movq %r13, %rdx + movq %r14, %rbx + cmpq $0, %rcx + movdqu (%rdx), %xmm7 + je EREMAINDER_6 + subq $96, %rsi +ELOOP_6: + movdqu (%rdi), %xmm1 + movdqu 16(%rdi), %xmm2 + movdqu 32(%rdi), %xmm3 + movdqu 48(%rdi), %xmm4 + movdqu 64(%rdi), %xmm5 + movdqu 80(%rdi), %xmm6 + movdqa (%r8), %xmm8 + movdqa 16(%r8), %xmm9 + movdqa 32(%r8), %xmm10 + movdqa 48(%r8), %xmm11 + pxor %xmm8, %xmm1 + pxor %xmm8, %xmm2 + pxor %xmm8, %xmm3 + pxor %xmm8, %xmm4 + pxor %xmm8, %xmm5 + pxor %xmm8, %xmm6 + aesdec %xmm9, %xmm1 + aesdec %xmm9, %xmm2 + aesdec %xmm9, %xmm3 + aesdec %xmm9, %xmm4 + aesdec %xmm9, %xmm5 + aesdec %xmm9, %xmm6 + aesdec %xmm10, %xmm1 + aesdec %xmm10, %xmm2 + aesdec %xmm10, %xmm3 + aesdec %xmm10, %xmm4 + aesdec %xmm10, %xmm5 + aesdec %xmm10, %xmm6 + aesdec %xmm11, %xmm1 + aesdec %xmm11, %xmm2 + aesdec %xmm11, %xmm3 + aesdec %xmm11, %xmm4 + aesdec %xmm11, %xmm5 + aesdec %xmm11, %xmm6 + movdqa 64(%r8), %xmm8 + movdqa 80(%r8), %xmm9 + movdqa 96(%r8), %xmm10 + movdqa 112(%r8), %xmm11 + aesdec %xmm8, %xmm1 + aesdec %xmm8, %xmm2 + aesdec %xmm8, %xmm3 + aesdec %xmm8, %xmm4 + aesdec %xmm8, %xmm5 + aesdec %xmm8, %xmm6 + aesdec %xmm9, %xmm1 + aesdec %xmm9, %xmm2 + aesdec %xmm9, %xmm3 + aesdec %xmm9, %xmm4 + aesdec %xmm9, %xmm5 + aesdec %xmm9, %xmm6 + aesdec %xmm10, %xmm1 + aesdec %xmm10, %xmm2 + aesdec %xmm10, %xmm3 + aesdec %xmm10, %xmm4 + aesdec %xmm10, %xmm5 + aesdec %xmm10, %xmm6 + aesdec %xmm11, %xmm1 + aesdec %xmm11, %xmm2 + aesdec %xmm11, %xmm3 + aesdec %xmm11, %xmm4 + aesdec %xmm11, %xmm5 + aesdec %xmm11, %xmm6 + movdqa 128(%r8), %xmm8 + movdqa 144(%r8), %xmm9 + movdqa 160(%r8), %xmm10 + cmpl $12, %r9d + aesdec %xmm8, %xmm1 + aesdec %xmm8, %xmm2 + aesdec %xmm8, %xmm3 + aesdec %xmm8, %xmm4 + aesdec %xmm8, %xmm5 + aesdec %xmm8, %xmm6 + aesdec %xmm9, %xmm1 + aesdec %xmm9, %xmm2 + aesdec %xmm9, %xmm3 + aesdec %xmm9, %xmm4 + aesdec %xmm9, %xmm5 + aesdec %xmm9, %xmm6 + jb ELAST_6 + movdqa 160(%r8), %xmm8 + movdqa 176(%r8), %xmm9 + movdqa 192(%r8), %xmm10 + cmpl $14, %r9d + aesdec %xmm8, %xmm1 + aesdec %xmm8, %xmm2 + aesdec %xmm8, %xmm3 + aesdec %xmm8, %xmm4 + aesdec %xmm8, %xmm5 + aesdec %xmm8, %xmm6 + aesdec %xmm9, %xmm1 + aesdec %xmm9, %xmm2 + aesdec %xmm9, %xmm3 + aesdec %xmm9, %xmm4 + aesdec %xmm9, %xmm5 + aesdec %xmm9, %xmm6 + jb ELAST_6 + movdqa 192(%r8), %xmm8 + movdqa 208(%r8), %xmm9 + movdqa 224(%r8), %xmm10 + aesdec %xmm8, %xmm1 + aesdec %xmm8, %xmm2 + aesdec %xmm8, %xmm3 + aesdec %xmm8, %xmm4 + aesdec %xmm8, %xmm5 + aesdec %xmm8, %xmm6 + aesdec %xmm9, %xmm1 + aesdec %xmm9, %xmm2 + aesdec %xmm9, %xmm3 + aesdec %xmm9, %xmm4 + aesdec %xmm9, %xmm5 + aesdec %xmm9, %xmm6 +ELAST_6: + addq $96, %rsi + aesdeclast %xmm10, %xmm1 + aesdeclast %xmm10, %xmm2 + aesdeclast %xmm10, %xmm3 + aesdeclast %xmm10, %xmm4 + aesdeclast %xmm10, %xmm5 + aesdeclast %xmm10, %xmm6 + movdqu (%rdi), %xmm8 + movdqu 16(%rdi), %xmm9 + movdqu 32(%rdi), %xmm10 + movdqu 48(%rdi), %xmm11 + movdqu 64(%rdi), %xmm12 + movdqu 80(%rdi), %xmm13 + pxor %xmm7, %xmm1 + pxor %xmm8, %xmm2 + pxor %xmm9, %xmm3 + pxor %xmm10, %xmm4 + pxor %xmm11, %xmm5 + pxor %xmm12, %xmm6 + movdqu %xmm13, %xmm7 + movdqu %xmm1, (%rsi) + movdqu %xmm2, 16(%rsi) + movdqu %xmm3, 32(%rsi) + movdqu %xmm4, 48(%rsi) + movdqu %xmm5, 64(%rsi) + movdqu %xmm6, 80(%rsi) + addq $96, %rdi + decq %rcx + jne ELOOP_6 + addq $96, %rsi +EREMAINDER_6: + cmpq $0, %r10 + je EEND_6 +ELOOP_6_2: + movdqu (%rdi), %xmm1 + movdqa %xmm1 ,%xmm10 + addq $16, %rdi + pxor (%r8), %xmm1 + movdqu 160(%r8), %xmm2 + cmpl $12, %r9d + aesdec 16(%r8), %xmm1 + aesdec 32(%r8), %xmm1 + aesdec 48(%r8), %xmm1 + aesdec 64(%r8), %xmm1 + aesdec 80(%r8), %xmm1 + aesdec 96(%r8), %xmm1 + aesdec 112(%r8), %xmm1 + aesdec 128(%r8), %xmm1 + aesdec 144(%r8), %xmm1 + jb ELAST_6_2 + movdqu 192(%r8), %xmm2 + cmpl $14, %r9d + aesdec 160(%r8), %xmm1 + aesdec 176(%r8), %xmm1 + jb ELAST_6_2 + movdqu 224(%r8), %xmm2 + aesdec 192(%r8), %xmm1 + aesdec 208(%r8), %xmm1 +ELAST_6_2: + aesdeclast %xmm2, %xmm1 + pxor %xmm7, %xmm1 + movdqa %xmm10, %xmm7 + movdqu %xmm1, (%rsi) + addq $16, %rsi + decq %r10 + jne ELOOP_6_2 +EEND_6: + ret + + +/* +AES_CBC_decrypt_by8 (const unsigned char *in, + unsigned char *out, + unsigned char ivec[16], + unsigned long length, + const unsigned char *KS, + int nr) +*/ +.globl AES_CBC_decrypt_by8 +AES_CBC_decrypt_by8: # parameter 1: %rdi - in # parameter 2: %rsi - out # parameter 3: %rdx - ivec diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index c55dc3921..6d97aa0e1 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -2669,8 +2669,7 @@ int aes_test(void) if (memcmp(cipher, verify, AES_BLOCK_SIZE)) return -61; -#if defined(WOLFSSL_AESNI) && \ - defined(HAVE_AES_DECRYPT) && defined(HAVE_AES_DECRYPT_EX) +#if defined(WOLFSSL_AESNI) && defined(HAVE_AES_DECRYPT) { const byte bigMsg[] = { /* "All work and no play makes Jack a dull boy. " */ @@ -2725,32 +2724,30 @@ int aes_test(void) }; byte bigCipher[sizeof(bigMsg)]; byte bigPlain[sizeof(bigMsg)]; + word32 i; - ret = wc_AesSetKey(&enc, key, AES_BLOCK_SIZE, iv, AES_ENCRYPTION); - if (ret != 0) - return -1030; - ret = wc_AesSetKey(&dec, key, AES_BLOCK_SIZE, iv, AES_DECRYPTION); - if (ret != 0) - return -1031; + for (i = AES_BLOCK_SIZE; i <= sizeof(bigMsg); i += AES_BLOCK_SIZE) { + memset(bigCipher, 0, sizeof(bigCipher)); + memset(bigPlain, 0, sizeof(bigPlain)); + ret = wc_AesSetKey(&enc, key, AES_BLOCK_SIZE, iv, AES_ENCRYPTION); + if (ret != 0) + return -1030; + ret = wc_AesSetKey(&dec, key, AES_BLOCK_SIZE, iv, AES_DECRYPTION); + if (ret != 0) + return -1031; - #define AESNI_DECRYPT_SIZE (AES_BLOCK_SIZE*24) + ret = wc_AesCbcEncrypt(&enc, bigCipher, bigMsg, i); + if (ret != 0) + return -1032; + ret = wc_AesCbcDecrypt(&dec, bigPlain, bigCipher, i); + if (ret != 0) + return -1033; - if ((sizeof(bigMsg) < AESNI_DECRYPT_SIZE) || - (AESNI_DECRYPT_SIZE == 0) || - (AESNI_DECRYPT_SIZE % AES_BLOCK_SIZE != 0)) - return -1032; - - ret = wc_AesCbcEncrypt(&enc, bigCipher, bigMsg, AESNI_DECRYPT_SIZE); - if (ret != 0) - return -1033; - ret = wc_AesCbcDecrypt(&dec, bigPlain, bigCipher, AESNI_DECRYPT_SIZE); - if (ret != 0) - return -1034; - - if (memcmp(bigPlain, bigMsg, AESNI_DECRYPT_SIZE)) - return -1035; + if (memcmp(bigPlain, bigMsg, i)) + return -1034; + } } -#endif /* WOLFSSL_AESNI HAVE_AES_DECRYPT HAVE_AES_DECRYPT_EX */ +#endif /* WOLFSSL_AESNI HAVE_AES_DECRYPT */ #ifdef HAVE_CAVIUM wc_AesFreeCavium(&enc);