1. Rename routine AES_CBC_decrypt_ex as AES_CBC_decrypt_by8

2. Added routine AES_CBC_decrypt_by6 that does six at a time.
3. Setting HAVE_AES_DECRYPT_BY6 or _BY8 (or not setting it) selects
   the 6, 8, or 4 way version of the assembly routine.
4. Modified AES-NI decrypt test to loop checking against the test
   bolus from 1 AES block to the whole 24 blocks.

wolfcrypt/src/aes.c

@@ -1094,17 +1094,22 @@ void AES_CBC_encrypt(const unsigned char* in, unsigned char* out,
                      XASM_LINK("AES_CBC_encrypt");
 
 #ifdef HAVE_AES_DECRYPT
-#ifndef HAVE_AES_DECRYPT_EX
+#if defined(HAVE_AES_DECRYPT_BY8)
+void AES_CBC_decrypt(const unsigned char* in, unsigned char* out,
+                     unsigned char* ivec, unsigned long length,
+                     const unsigned char* KS, int nr)
+                     XASM_LINK("AES_CBC_decrypt_by8");
+#elif defined(HAVE_AES_DECRYPT_BY6)
+void AES_CBC_decrypt(const unsigned char* in, unsigned char* out,
+                     unsigned char* ivec, unsigned long length,
+                     const unsigned char* KS, int nr)
+                     XASM_LINK("AES_CBC_decrypt_by6");
+#else
 void AES_CBC_decrypt(const unsigned char* in, unsigned char* out,
                      unsigned char* ivec, unsigned long length,
                      const unsigned char* KS, int nr)
                      XASM_LINK("AES_CBC_decrypt");
-#else /* HAVE_AES_DECRYPT_EX */
+#endif /* HAVE_AES_DECRYPT_BYX */
-void AES_CBC_decrypt_ex(const unsigned char* in, unsigned char* out,
-                        unsigned char* ivec, unsigned long length,
-                        const unsigned char* KS, int nr)
-                        XASM_LINK("AES_CBC_decrypt_ex");
-#endif /* HAVE_AES_DECRYPT_EX */
 #endif /* HAVE_AES_DECRYPT */
 #endif /* HAVE_AES_CBC */
 
@@ -2556,13 +2561,8 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
 
             /* if input and output same will overwrite input iv */
             XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
-#ifndef HAVE_AES_DECRYPT_EX
             AES_CBC_decrypt(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
                             aes->rounds);
-#else /* HAVE_AES_DECRYPT_EX */
-            AES_CBC_decrypt_ex(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
-                               aes->rounds);
-#endif /* HAVE_AES_DECRYPT_EX */
             /* store iv for next call */
             XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
             return 0;

wolfcrypt/src/aes_asm.s

@@ -267,15 +267,237 @@ ret
 
 
 /*
-AES_CBC_decrypt_ex (const unsigned char *in,
+AES_CBC_decrypt_by6 (const unsigned char *in,
   unsigned char *out,
   unsigned char ivec[16],
   unsigned long length,
   const unsigned char *KS,
   int nr)
 */
-.globl AES_CBC_decrypt_ex
+.globl AES_CBC_decrypt_by6
-AES_CBC_decrypt_ex:
+AES_CBC_decrypt_by6:
+# parameter 1: %rdi - in
+# parameter 2: %rsi - out
+# parameter 3: %rdx - ivec
+# parameter 4: %rcx - length
+# parameter 5: %r8  - KS
+# parameter 6: %r9d - nr
+
+        movq        %rcx, %r10
+        shrq        $4, %rcx
+        shlq        $60, %r10
+        je          ENO_PARTS_6
+        addq        $1, %rcx
+ENO_PARTS_6:
+        movq        %rax, %r12
+        movq        %rdx, %r13
+        movq        %rbx, %r14
+        movq        $0, %rdx
+        movq        %rcx, %rax
+        movq        $6, %rbx
+        div         %rbx
+        movq        %rax, %rcx
+        movq        %rdx, %r10
+        movq        %r12, %rax
+        movq        %r13, %rdx
+        movq        %r14, %rbx
+        cmpq        $0, %rcx
+        movdqu      (%rdx), %xmm7
+        je          EREMAINDER_6
+        subq        $96, %rsi
+ELOOP_6:
+        movdqu      (%rdi), %xmm1
+        movdqu      16(%rdi), %xmm2
+        movdqu      32(%rdi), %xmm3
+        movdqu      48(%rdi), %xmm4
+        movdqu      64(%rdi), %xmm5
+        movdqu      80(%rdi), %xmm6
+        movdqa      (%r8), %xmm8
+        movdqa      16(%r8), %xmm9
+        movdqa      32(%r8), %xmm10
+        movdqa      48(%r8), %xmm11
+        pxor        %xmm8, %xmm1
+        pxor        %xmm8, %xmm2
+        pxor        %xmm8, %xmm3
+        pxor        %xmm8, %xmm4
+        pxor        %xmm8, %xmm5
+        pxor        %xmm8, %xmm6
+        aesdec      %xmm9, %xmm1
+        aesdec      %xmm9, %xmm2
+        aesdec      %xmm9, %xmm3
+        aesdec      %xmm9, %xmm4
+        aesdec      %xmm9, %xmm5
+        aesdec      %xmm9, %xmm6
+        aesdec      %xmm10, %xmm1
+        aesdec      %xmm10, %xmm2
+        aesdec      %xmm10, %xmm3
+        aesdec      %xmm10, %xmm4
+        aesdec      %xmm10, %xmm5
+        aesdec      %xmm10, %xmm6
+        aesdec      %xmm11, %xmm1
+        aesdec      %xmm11, %xmm2
+        aesdec      %xmm11, %xmm3
+        aesdec      %xmm11, %xmm4
+        aesdec      %xmm11, %xmm5
+        aesdec      %xmm11, %xmm6
+        movdqa      64(%r8), %xmm8
+        movdqa      80(%r8), %xmm9
+        movdqa      96(%r8), %xmm10
+        movdqa      112(%r8), %xmm11
+        aesdec      %xmm8, %xmm1
+        aesdec      %xmm8, %xmm2
+        aesdec      %xmm8, %xmm3
+        aesdec      %xmm8, %xmm4
+        aesdec      %xmm8, %xmm5
+        aesdec      %xmm8, %xmm6
+        aesdec      %xmm9, %xmm1
+        aesdec      %xmm9, %xmm2
+        aesdec      %xmm9, %xmm3
+        aesdec      %xmm9, %xmm4
+        aesdec      %xmm9, %xmm5
+        aesdec      %xmm9, %xmm6
+        aesdec      %xmm10, %xmm1
+        aesdec      %xmm10, %xmm2
+        aesdec      %xmm10, %xmm3
+        aesdec      %xmm10, %xmm4
+        aesdec      %xmm10, %xmm5
+        aesdec      %xmm10, %xmm6
+        aesdec      %xmm11, %xmm1
+        aesdec      %xmm11, %xmm2
+        aesdec      %xmm11, %xmm3
+        aesdec      %xmm11, %xmm4
+        aesdec      %xmm11, %xmm5
+        aesdec      %xmm11, %xmm6
+        movdqa      128(%r8), %xmm8
+        movdqa      144(%r8), %xmm9
+        movdqa      160(%r8), %xmm10
+        cmpl        $12, %r9d
+        aesdec      %xmm8, %xmm1
+        aesdec      %xmm8, %xmm2
+        aesdec      %xmm8, %xmm3
+        aesdec      %xmm8, %xmm4
+        aesdec      %xmm8, %xmm5
+        aesdec      %xmm8, %xmm6
+        aesdec      %xmm9, %xmm1
+        aesdec      %xmm9, %xmm2
+        aesdec      %xmm9, %xmm3
+        aesdec      %xmm9, %xmm4
+        aesdec      %xmm9, %xmm5
+        aesdec      %xmm9, %xmm6
+        jb          ELAST_6
+        movdqa      160(%r8), %xmm8
+        movdqa      176(%r8), %xmm9
+        movdqa      192(%r8), %xmm10
+        cmpl        $14, %r9d
+        aesdec      %xmm8, %xmm1
+        aesdec      %xmm8, %xmm2
+        aesdec      %xmm8, %xmm3
+        aesdec      %xmm8, %xmm4
+        aesdec      %xmm8, %xmm5
+        aesdec      %xmm8, %xmm6
+        aesdec      %xmm9, %xmm1
+        aesdec      %xmm9, %xmm2
+        aesdec      %xmm9, %xmm3
+        aesdec      %xmm9, %xmm4
+        aesdec      %xmm9, %xmm5
+        aesdec      %xmm9, %xmm6
+        jb          ELAST_6
+        movdqa      192(%r8), %xmm8
+        movdqa      208(%r8), %xmm9
+        movdqa      224(%r8), %xmm10
+        aesdec      %xmm8, %xmm1
+        aesdec      %xmm8, %xmm2
+        aesdec      %xmm8, %xmm3
+        aesdec      %xmm8, %xmm4
+        aesdec      %xmm8, %xmm5
+        aesdec      %xmm8, %xmm6
+        aesdec      %xmm9, %xmm1
+        aesdec      %xmm9, %xmm2
+        aesdec      %xmm9, %xmm3
+        aesdec      %xmm9, %xmm4
+        aesdec      %xmm9, %xmm5
+        aesdec      %xmm9, %xmm6
+ELAST_6:
+        addq        $96, %rsi
+        aesdeclast  %xmm10, %xmm1
+        aesdeclast  %xmm10, %xmm2
+        aesdeclast  %xmm10, %xmm3
+        aesdeclast  %xmm10, %xmm4
+        aesdeclast  %xmm10, %xmm5
+        aesdeclast  %xmm10, %xmm6
+        movdqu      (%rdi), %xmm8
+        movdqu      16(%rdi), %xmm9
+        movdqu      32(%rdi), %xmm10
+        movdqu      48(%rdi), %xmm11
+        movdqu      64(%rdi), %xmm12
+        movdqu      80(%rdi), %xmm13
+        pxor        %xmm7, %xmm1
+        pxor        %xmm8, %xmm2
+        pxor        %xmm9, %xmm3
+        pxor        %xmm10, %xmm4
+        pxor        %xmm11, %xmm5
+        pxor        %xmm12, %xmm6
+        movdqu      %xmm13, %xmm7
+        movdqu      %xmm1, (%rsi)
+        movdqu      %xmm2, 16(%rsi)
+        movdqu      %xmm3, 32(%rsi)
+        movdqu      %xmm4, 48(%rsi)
+        movdqu      %xmm5, 64(%rsi)
+        movdqu      %xmm6, 80(%rsi)
+        addq        $96, %rdi
+        decq        %rcx
+        jne         ELOOP_6
+        addq        $96, %rsi
+EREMAINDER_6:
+        cmpq        $0, %r10
+        je          EEND_6
+ELOOP_6_2:
+        movdqu      (%rdi), %xmm1
+        movdqa      %xmm1 ,%xmm10
+        addq        $16, %rdi
+        pxor        (%r8), %xmm1
+        movdqu      160(%r8), %xmm2
+        cmpl        $12, %r9d
+        aesdec      16(%r8), %xmm1
+        aesdec      32(%r8), %xmm1
+        aesdec      48(%r8), %xmm1
+        aesdec      64(%r8), %xmm1
+        aesdec      80(%r8), %xmm1
+        aesdec      96(%r8), %xmm1
+        aesdec      112(%r8), %xmm1
+        aesdec      128(%r8), %xmm1
+        aesdec      144(%r8), %xmm1
+        jb          ELAST_6_2
+        movdqu      192(%r8), %xmm2
+        cmpl        $14, %r9d
+        aesdec      160(%r8), %xmm1
+        aesdec      176(%r8), %xmm1
+        jb          ELAST_6_2
+        movdqu      224(%r8), %xmm2
+        aesdec      192(%r8), %xmm1
+        aesdec      208(%r8), %xmm1
+ELAST_6_2:
+        aesdeclast  %xmm2, %xmm1
+        pxor        %xmm7, %xmm1
+        movdqa      %xmm10, %xmm7
+        movdqu      %xmm1, (%rsi)
+        addq        $16, %rsi
+        decq        %r10
+        jne         ELOOP_6_2
+EEND_6:
+        ret
+
+
+/*
+AES_CBC_decrypt_by8 (const unsigned char *in,
+  unsigned char *out,
+  unsigned char ivec[16],
+  unsigned long length,
+  const unsigned char *KS,
+  int nr)
+*/
+.globl AES_CBC_decrypt_by8
+AES_CBC_decrypt_by8:
 # parameter 1: %rdi - in
 # parameter 2: %rsi - out
 # parameter 3: %rdx - ivec

wolfcrypt/test/test.c

@@ -2669,8 +2669,7 @@ int aes_test(void)
     if (memcmp(cipher, verify, AES_BLOCK_SIZE))
         return -61;
 
-#if defined(WOLFSSL_AESNI) && \
+#if defined(WOLFSSL_AESNI) && defined(HAVE_AES_DECRYPT)
-    defined(HAVE_AES_DECRYPT) && defined(HAVE_AES_DECRYPT_EX)
     {
         const byte bigMsg[] = {
             /* "All work and no play makes Jack a dull boy. " */
@@ -2725,32 +2724,30 @@ int aes_test(void)
         };
         byte bigCipher[sizeof(bigMsg)];
         byte bigPlain[sizeof(bigMsg)];
+        word32 i;
 
-        ret = wc_AesSetKey(&enc, key, AES_BLOCK_SIZE, iv, AES_ENCRYPTION);
+        for (i = AES_BLOCK_SIZE; i <= sizeof(bigMsg); i += AES_BLOCK_SIZE) {
-        if (ret != 0)
+            memset(bigCipher, 0, sizeof(bigCipher));
-            return -1030;
+            memset(bigPlain, 0, sizeof(bigPlain));
-        ret = wc_AesSetKey(&dec, key, AES_BLOCK_SIZE, iv, AES_DECRYPTION);
+            ret = wc_AesSetKey(&enc, key, AES_BLOCK_SIZE, iv, AES_ENCRYPTION);
-        if (ret != 0)
+            if (ret != 0)
-            return -1031;
+                return -1030;
+            ret = wc_AesSetKey(&dec, key, AES_BLOCK_SIZE, iv, AES_DECRYPTION);
+            if (ret != 0)
+                return -1031;
 
-        #define AESNI_DECRYPT_SIZE (AES_BLOCK_SIZE*24)
+            ret = wc_AesCbcEncrypt(&enc, bigCipher, bigMsg, i);
+            if (ret != 0)
+                return -1032;
+            ret = wc_AesCbcDecrypt(&dec, bigPlain, bigCipher, i);
+            if (ret != 0)
+                return -1033;
 
-        if ((sizeof(bigMsg) < AESNI_DECRYPT_SIZE) ||
+            if (memcmp(bigPlain, bigMsg, i))
-            (AESNI_DECRYPT_SIZE == 0) ||
+                return -1034;
-            (AESNI_DECRYPT_SIZE % AES_BLOCK_SIZE != 0))
+        }
-            return -1032;
-
-        ret = wc_AesCbcEncrypt(&enc, bigCipher, bigMsg, AESNI_DECRYPT_SIZE);
-        if (ret != 0)
-            return -1033;
-        ret = wc_AesCbcDecrypt(&dec, bigPlain, bigCipher, AESNI_DECRYPT_SIZE);
-        if (ret != 0)
-            return -1034;
-
-        if (memcmp(bigPlain, bigMsg, AESNI_DECRYPT_SIZE))
-            return -1035;
     }
-#endif /* WOLFSSL_AESNI HAVE_AES_DECRYPT HAVE_AES_DECRYPT_EX */
+#endif /* WOLFSSL_AESNI HAVE_AES_DECRYPT */
 
 #ifdef HAVE_CAVIUM
         wc_AesFreeCavium(&enc);