diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 13cace7ab1..8417e84efa 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -839,9 +839,12 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits #elif WC_VAES_MIN_BLOCKS < 1 #error Invalid WC_VAES_MIN_BLOCKS #endif - /* CFB/ECB: wide ECB setup (key broadcast) doesn't pay off below this. */ + /* ECB/CBC/CTR/XTS: the wide ladder handles 2+ blocks in parallel and + * only caches round keys once it pays off (>= 32B), so the wide path + * beats the single-block AES-NI fallback from 2 blocks up; a lone block + * stays on AES-NI. (Measured +8..+58% at 2-6 blocks on Zen5.) */ #ifndef WC_VAES_ECB_MIN_BLOCKS - #define WC_VAES_ECB_MIN_BLOCKS WC_VAES_MIN_BLOCKS + #define WC_VAES_ECB_MIN_BLOCKS 2 #elif WC_VAES_ECB_MIN_BLOCKS < 1 #error Invalid WC_VAES_ECB_MIN_BLOCKS #endif @@ -15587,23 +15590,52 @@ static WARN_UNUSED_RESULT int AesCfbDecrypt_C(Aes* aes, byte* out, #ifndef WOLFSSL_SMALL_STACK ALIGN16 byte tmp[WC_AES_CFB_DEC_BUF_BLOCKS * WC_AES_BLOCK_SIZE]; #endif - while (sz >= 2 * WC_AES_BLOCK_SIZE) { - word32 blocks = sz / WC_AES_BLOCK_SIZE; - word32 nbytes; - if (blocks > WC_AES_CFB_DEC_BUF_BLOCKS) - blocks = WC_AES_CFB_DEC_BUF_BLOCKS; - nbytes = blocks * WC_AES_BLOCK_SIZE; - XMEMCPY(tmp, aes->reg, WC_AES_BLOCK_SIZE); - XMEMCPY(tmp + WC_AES_BLOCK_SIZE, in, nbytes - WC_AES_BLOCK_SIZE); - XMEMCPY(aes->reg, in + nbytes - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE); - ret = wc_AesEcbEncrypt(aes, tmp, tmp, nbytes); - if (ret != 0) { - break; + if (sz >= 2 * WC_AES_BLOCK_SIZE) { + /* CFB-decrypt keystream block i is E(C_{i-1}): block 0 uses the + * feedback register, block i>=1 uses the previous cipher block. So + * ECB the ciphertext straight out of 'in' (no shift-copy) to get + * E(C_0..C_{n-1}), XOR block i with the (i-1)th ECB output, and + * carry E(C_{n-1}) as the next chunk's block-0 keystream - E(reg) + * is computed only once here. */ + ALIGN16 byte ks[WC_AES_BLOCK_SIZE]; + ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, ks, + &did_prefetches); + while ((ret == 0) && (sz >= 2 * WC_AES_BLOCK_SIZE)) { + word32 blocks = sz / WC_AES_BLOCK_SIZE; + word32 nbytes; + if (blocks > WC_AES_CFB_DEC_BUF_BLOCKS) + blocks = WC_AES_CFB_DEC_BUF_BLOCKS; + nbytes = blocks * WC_AES_BLOCK_SIZE; + /* tmp[i] = E(C_i), read directly from the input. Already inside + * VECTOR_REGISTERS_PUSH, so use the inner ECB (no nested + * save/restore or re-dispatch) where available. */ + #if defined(WOLFSSL_AESNI) && defined(WOLFSSL_X86_64_BUILD) + if (aes->use_aesni) { + AesEcbEncryptBlocks(in, tmp, nbytes, (byte*)aes->key, + (int)aes->rounds); + } + else + #endif + { + ret = wc_AesEcbEncrypt(aes, tmp, in, nbytes); + if (ret != 0) + break; + } + /* Feedback for the tail = last cipher block; save it before the + * XOR can overwrite 'in' (in == out case). */ + XMEMCPY((byte*)aes->reg, in + nbytes - WC_AES_BLOCK_SIZE, + WC_AES_BLOCK_SIZE); + /* P_0 = C_0 ^ E(feedback); P_i = C_i ^ E(C_{i-1}) = + * C_i ^ tmp[i-1]. */ + xorbufout(out, in, ks, WC_AES_BLOCK_SIZE); + xorbufout(out + WC_AES_BLOCK_SIZE, in + WC_AES_BLOCK_SIZE, tmp, + nbytes - WC_AES_BLOCK_SIZE); + /* Carry E(last cipher block) as the next chunk's block-0 KS. */ + XMEMCPY(ks, tmp + nbytes - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE); + out += nbytes; + in += nbytes; + sz -= nbytes; } - xorbufout(out, in, tmp, nbytes); - out += nbytes; - in += nbytes; - sz -= nbytes; } } #endif @@ -16903,8 +16935,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { AES_XTS_encrypt_avx512(in, out, sz, i, (const byte*)aes->key, (const byte*)xaes->tweak.key, @@ -16914,8 +16945,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, else #endif #if defined(HAVE_INTEL_VAES) - if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { AES_XTS_encrypt_vaes(in, out, sz, i, (const byte*)aes->key, (const byte*)xaes->tweak.key, @@ -17140,8 +17170,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { AES_XTS_encrypt_update_avx512(in, out, sz, (const byte*)aes->key, stream->tweak_block, @@ -17445,8 +17474,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { AES_XTS_decrypt_avx512(in, out, sz, i, (const byte*)aes->key, (const byte*)xaes->tweak.key, @@ -17676,8 +17704,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { AES_XTS_decrypt_update_avx512(in, out, sz, (const byte*)aes->key, stream->tweak_block, @@ -17687,8 +17714,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s else #endif #if defined(HAVE_INTEL_VAES) - if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { AES_XTS_decrypt_update_vaes(in, out, sz, (const byte*)aes->key, stream->tweak_block, diff --git a/wolfcrypt/src/aes_gcm_asm.S b/wolfcrypt/src/aes_gcm_asm.S index c1c357e19b..23c61530ac 100644 --- a/wolfcrypt/src/aes_gcm_asm.S +++ b/wolfcrypt/src/aes_gcm_asm.S @@ -12353,7 +12353,7 @@ _AES_GCM_encrypt_avx2: movl 64(%rsp), %r14d movq 72(%rsp), %rsi movl 80(%rsp), %r9d - subq $0xa0, %rsp + subq $0xb8, %rsp vpxor %xmm4, %xmm4, %xmm4 vpxor %xmm6, %xmm6, %xmm6 movl %ebx, %edx @@ -12602,9 +12602,183 @@ L_AES_GCM_encrypt_avx2_iv_done: cmpl $0x00, %edx je L_AES_GCM_encrypt_avx2_calc_aad_done xorl %ecx, %ecx - cmpl $16, %edx - jl L_AES_GCM_encrypt_avx2_calc_aad_lt16 + cmpl $0x80, %edx + jl L_AES_GCM_encrypt_avx2_calc_aad_after_agg + movl %edx, %r13d + andl $0xffffffc0, %r13d + vmovdqu %xmm5, 168(%rsp) + vpsrlq $63, %xmm5, %xmm1 + vpsllq $0x01, %xmm5, %xmm0 + vpslldq $8, %xmm1, %xmm1 + vpor %xmm1, %xmm0, %xmm0 + vpshufd $0xff, %xmm5, %xmm5 + vpsrad $31, %xmm5, %xmm5 + vpand L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vmovdqu L_avx2_aes_gcm_mod2_128(%rip), %xmm3 + # H ^ 1 and H ^ 2 + vpclmulqdq $0x00, %xmm5, %xmm5, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm5, %xmm10 + vpclmulqdq $16, %xmm3, %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpclmulqdq $16, %xmm3, %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm9, %xmm10, %xmm0 + vmovdqu %xmm5, (%rsp) + vmovdqu %xmm0, 16(%rsp) + # H ^ 3 and H ^ 4 + vpclmulqdq $16, %xmm5, %xmm0, %xmm11 + vpclmulqdq $0x01, %xmm5, %xmm0, %xmm10 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm12 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm13 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm2 + vpxor %xmm9, %xmm10, %xmm1 + vmovdqu %xmm1, 32(%rsp) + vmovdqu %xmm2, 48(%rsp) + # H ^ 5 and H ^ 6 + vpclmulqdq $16, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm13 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm0 + vpxor %xmm9, %xmm10, %xmm7 + vmovdqu %xmm7, 64(%rsp) + vmovdqu %xmm0, 80(%rsp) + # H ^ 7 and H ^ 8 + vpclmulqdq $16, %xmm1, %xmm2, %xmm11 + vpclmulqdq $0x01, %xmm1, %xmm2, %xmm10 + vpclmulqdq $0x00, %xmm1, %xmm2, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm2, %xmm12 + vpclmulqdq $0x00, %xmm2, %xmm2, %xmm13 + vpclmulqdq $0x11, %xmm2, %xmm2, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm0 + vpxor %xmm9, %xmm10, %xmm7 + vmovdqu %xmm7, 96(%rsp) + vmovdqu %xmm0, 112(%rsp) +L_AES_GCM_encrypt_avx2_calc_aad_agg: + # 64 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu (%rbx), %xmm8 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vmovdqu 16(%rbx), %xmm9 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm9, %xmm9 + vmovdqu 32(%rbx), %xmm10 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm10, %xmm10 + vmovdqu 48(%rbx), %xmm11 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm11, %xmm11 + vpxor %xmm6, %xmm8, %xmm8 + vmovdqu 48(%rsp), %xmm7 + vpclmulqdq $16, %xmm8, %xmm7, %xmm13 + vpclmulqdq $0x01, %xmm8, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm8, %xmm7, %xmm12 + vpclmulqdq $0x11, %xmm8, %xmm7, %xmm6 + vpxor %xmm1, %xmm13, %xmm13 + vmovdqu 32(%rsp), %xmm7 + vpclmulqdq $16, %xmm9, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm9, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm9, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm9, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm13, %xmm13 + vpxor %xmm0, %xmm12, %xmm12 + vmovdqu 16(%rsp), %xmm7 + vpclmulqdq $16, %xmm10, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm10, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm10, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm10, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm13, %xmm13 + vpxor %xmm0, %xmm12, %xmm12 + vmovdqu (%rsp), %xmm7 + vpclmulqdq $16, %xmm11, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm11, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm11, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm11, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm13, %xmm13 + vpxor %xmm0, %xmm12, %xmm12 + vpslldq $8, %xmm13, %xmm7 + vpsrldq $8, %xmm13, %xmm13 + vpxor %xmm7, %xmm12, %xmm12 + vpxor %xmm13, %xmm6, %xmm6 + # ghash_red + vmovdqu L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm12, %xmm0 + vpshufd $0x4e, %xmm12, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm6, %xmm6 + addl $0x40, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_encrypt_avx2_calc_aad_agg + vmovdqu 168(%rsp), %xmm5 +L_AES_GCM_encrypt_avx2_calc_aad_after_agg: + movl %r11d, %edx andl $0xfffffff0, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_avx2_calc_aad_partial L_AES_GCM_encrypt_avx2_calc_aad_16_loop: vmovdqu (%r12,%rcx,1), %xmm0 vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 @@ -12642,6 +12816,7 @@ L_AES_GCM_encrypt_avx2_calc_aad_16_loop: addl $16, %ecx cmpl %edx, %ecx jl L_AES_GCM_encrypt_avx2_calc_aad_16_loop +L_AES_GCM_encrypt_avx2_calc_aad_partial: movl %r11d, %edx cmpl %edx, %ecx je L_AES_GCM_encrypt_avx2_calc_aad_done @@ -13624,7 +13799,7 @@ L_AES_GCM_encrypt_avx2_store_tag_16: vmovdqu %xmm0, (%r15) L_AES_GCM_encrypt_avx2_store_tag_done: vzeroupper - addq $0xa0, %rsp + addq $0xb8, %rsp popq %r14 popq %rbx popq %r15 @@ -13663,7 +13838,7 @@ _AES_GCM_decrypt_avx2: movq 80(%rsp), %rsi movl 88(%rsp), %r9d movq 96(%rsp), %rbp - subq $0xa8, %rsp + subq $0xb8, %rsp vpxor %xmm4, %xmm4, %xmm4 vpxor %xmm6, %xmm6, %xmm6 movl %ebx, %edx @@ -13912,9 +14087,183 @@ L_AES_GCM_decrypt_avx2_iv_done: cmpl $0x00, %edx je L_AES_GCM_decrypt_avx2_calc_aad_done xorl %ecx, %ecx - cmpl $16, %edx - jl L_AES_GCM_decrypt_avx2_calc_aad_lt16 + cmpl $0x80, %edx + jl L_AES_GCM_decrypt_avx2_calc_aad_after_agg + movl %edx, %r13d + andl $0xffffffc0, %r13d + vmovdqu %xmm5, 168(%rsp) + vpsrlq $63, %xmm5, %xmm1 + vpsllq $0x01, %xmm5, %xmm0 + vpslldq $8, %xmm1, %xmm1 + vpor %xmm1, %xmm0, %xmm0 + vpshufd $0xff, %xmm5, %xmm5 + vpsrad $31, %xmm5, %xmm5 + vpand L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vmovdqu L_avx2_aes_gcm_mod2_128(%rip), %xmm3 + # H ^ 1 and H ^ 2 + vpclmulqdq $0x00, %xmm5, %xmm5, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm5, %xmm10 + vpclmulqdq $16, %xmm3, %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpclmulqdq $16, %xmm3, %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm9, %xmm10, %xmm0 + vmovdqu %xmm5, (%rsp) + vmovdqu %xmm0, 16(%rsp) + # H ^ 3 and H ^ 4 + vpclmulqdq $16, %xmm5, %xmm0, %xmm11 + vpclmulqdq $0x01, %xmm5, %xmm0, %xmm10 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm12 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm13 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm2 + vpxor %xmm9, %xmm10, %xmm1 + vmovdqu %xmm1, 32(%rsp) + vmovdqu %xmm2, 48(%rsp) + # H ^ 5 and H ^ 6 + vpclmulqdq $16, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm13 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm0 + vpxor %xmm9, %xmm10, %xmm7 + vmovdqu %xmm7, 64(%rsp) + vmovdqu %xmm0, 80(%rsp) + # H ^ 7 and H ^ 8 + vpclmulqdq $16, %xmm1, %xmm2, %xmm11 + vpclmulqdq $0x01, %xmm1, %xmm2, %xmm10 + vpclmulqdq $0x00, %xmm1, %xmm2, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm2, %xmm12 + vpclmulqdq $0x00, %xmm2, %xmm2, %xmm13 + vpclmulqdq $0x11, %xmm2, %xmm2, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm0 + vpxor %xmm9, %xmm10, %xmm7 + vmovdqu %xmm7, 96(%rsp) + vmovdqu %xmm0, 112(%rsp) +L_AES_GCM_decrypt_avx2_calc_aad_agg: + # 64 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu (%rbx), %xmm8 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vmovdqu 16(%rbx), %xmm9 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm9, %xmm9 + vmovdqu 32(%rbx), %xmm10 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm10, %xmm10 + vmovdqu 48(%rbx), %xmm11 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm11, %xmm11 + vpxor %xmm6, %xmm8, %xmm8 + vmovdqu 48(%rsp), %xmm7 + vpclmulqdq $16, %xmm8, %xmm7, %xmm13 + vpclmulqdq $0x01, %xmm8, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm8, %xmm7, %xmm12 + vpclmulqdq $0x11, %xmm8, %xmm7, %xmm6 + vpxor %xmm1, %xmm13, %xmm13 + vmovdqu 32(%rsp), %xmm7 + vpclmulqdq $16, %xmm9, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm9, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm9, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm9, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm13, %xmm13 + vpxor %xmm0, %xmm12, %xmm12 + vmovdqu 16(%rsp), %xmm7 + vpclmulqdq $16, %xmm10, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm10, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm10, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm10, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm13, %xmm13 + vpxor %xmm0, %xmm12, %xmm12 + vmovdqu (%rsp), %xmm7 + vpclmulqdq $16, %xmm11, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm11, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm11, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm11, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm13, %xmm13 + vpxor %xmm0, %xmm12, %xmm12 + vpslldq $8, %xmm13, %xmm7 + vpsrldq $8, %xmm13, %xmm13 + vpxor %xmm7, %xmm12, %xmm12 + vpxor %xmm13, %xmm6, %xmm6 + # ghash_red + vmovdqu L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm12, %xmm0 + vpshufd $0x4e, %xmm12, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm6, %xmm6 + addl $0x40, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_decrypt_avx2_calc_aad_agg + vmovdqu 168(%rsp), %xmm5 +L_AES_GCM_decrypt_avx2_calc_aad_after_agg: + movl %r11d, %edx andl $0xfffffff0, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_avx2_calc_aad_partial L_AES_GCM_decrypt_avx2_calc_aad_16_loop: vmovdqu (%r12,%rcx,1), %xmm0 vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 @@ -13952,6 +14301,7 @@ L_AES_GCM_decrypt_avx2_calc_aad_16_loop: addl $16, %ecx cmpl %edx, %ecx jl L_AES_GCM_decrypt_avx2_calc_aad_16_loop +L_AES_GCM_decrypt_avx2_calc_aad_partial: movl %r11d, %edx cmpl %edx, %ecx je L_AES_GCM_decrypt_avx2_calc_aad_done @@ -14594,7 +14944,7 @@ L_AES_GCM_decrypt_avx2_cmp_tag_16: L_AES_GCM_decrypt_avx2_cmp_tag_done: movl %eax, (%rbp) vzeroupper - addq $0xa8, %rsp + addq $0xb8, %rsp popq %rbp popq %r15 popq %rbx @@ -17026,7 +17376,7 @@ _AES_GCM_encrypt_vaes: movl 64(%rsp), %r14d movq 72(%rsp), %r15 movl 80(%rsp), %r10d - subq $0x230, %rsp + subq $0x250, %rsp vpxor %xmm5, %xmm5, %xmm5 vpxor %xmm15, %xmm15, %xmm15 movl %ebx, %edx @@ -17320,9 +17670,527 @@ L_AES_GCM_encrypt_vaes_iv_done: cmpl $0x00, %edx je L_AES_GCM_encrypt_vaes_calc_aad_done xorl %ecx, %ecx - cmpl $16, %edx - jl L_AES_GCM_encrypt_vaes_calc_aad_lt16 + cmpl $0x80, %edx + jl L_AES_GCM_encrypt_vaes_calc_aad_ser + vmovdqu %xmm6, 560(%rsp) + vmovdqu %xmm5, 576(%rsp) + vpsrlq $63, %xmm6, %xmm8 + vpsllq $0x01, %xmm6, %xmm7 + vpslldq $8, %xmm8, %xmm8 + vpor %xmm8, %xmm7, %xmm7 + vpshufd $0xff, %xmm6, %xmm6 + vpsrad $31, %xmm6, %xmm6 + vpand L_vaes_aes_gcm_mod2_128(%rip), %xmm6, %xmm6 + vpxor %xmm7, %xmm6, %xmm6 + vmovdqa %xmm15, %xmm2 + # H ^ 1 + vmovdqu %xmm6, (%rsp) + # H ^ 2 + vpclmulqdq $0x00, %xmm6, %xmm6, %xmm7 + vpclmulqdq $0x11, %xmm6, %xmm6, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm0 + vmovdqu %xmm0, 16(%rsp) + # H ^ 3 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm6, %xmm0, %xmm7 + vpclmulqdq $0x01, %xmm6, %xmm0, %xmm8 + vpclmulqdq $16, %xmm6, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm6, %xmm0, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm1 + vmovdqu %xmm1, 32(%rsp) + # H ^ 4 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm3 + vmovdqu %xmm3, 48(%rsp) + # H ^ 5 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 64(%rsp) + # H ^ 6 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm7 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 80(%rsp) + # H ^ 7 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm1, %xmm3, %xmm7 + vpclmulqdq $0x01, %xmm1, %xmm3, %xmm8 + vpclmulqdq $16, %xmm1, %xmm3, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm3, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 96(%rsp) + # H ^ 8 + vpclmulqdq $0x00, %xmm3, %xmm3, %xmm7 + vpclmulqdq $0x11, %xmm3, %xmm3, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 112(%rsp) + cmpl $0x100, %edx + jl L_AES_GCM_encrypt_vaes_calc_aad_no_ext + # H ^ 9 + vmovdqu 48(%rsp), %xmm0 + vmovdqu 64(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 128(%rsp) + # H ^ 10 + vmovdqu 64(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 144(%rsp) + # H ^ 11 + vmovdqu 64(%rsp), %xmm0 + vmovdqu 80(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 160(%rsp) + # H ^ 12 + vmovdqu 80(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 176(%rsp) + # H ^ 13 + vmovdqu 80(%rsp), %xmm0 + vmovdqu 96(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 192(%rsp) + # H ^ 14 + vmovdqu 96(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 208(%rsp) + # H ^ 15 + vmovdqu 96(%rsp), %xmm0 + vmovdqu 112(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 224(%rsp) + # H ^ 16 + vmovdqu 112(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 240(%rsp) + vmovdqu 224(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 192(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 160(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu 128(%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vmovdqu %ymm7, 256(%rsp) + vmovdqu %ymm8, 288(%rsp) + vmovdqu %ymm9, 320(%rsp) + vmovdqu %ymm10, 352(%rsp) + vmovdqu 96(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 64(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 32(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu (%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vmovdqu %ymm7, 384(%rsp) + vmovdqu %ymm8, 416(%rsp) + vmovdqu %ymm9, 448(%rsp) + vmovdqu %ymm10, 480(%rsp) +L_AES_GCM_encrypt_vaes_calc_aad_no_ext: + vbroadcasti128 L_vaes_aes_gcm_mod2_128(%rip), %ymm14 + movl %r11d, %r13d + andl $0xffffff00, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_encrypt_vaes_calc_aad_a256 +L_AES_GCM_encrypt_vaes_calc_aad_l256: + # 256 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vbroadcasti128 L_vaes_aes_gcm_bswap_mask(%rip), %ymm6 + vpxor %ymm4, %ymm4, %ymm4 + vinserti128 $0x00, %xmm15, %ymm4, %ymm4 + vmovdqu 256(%rsp), %ymm7 + vmovdqu 288(%rsp), %ymm8 + vmovdqu 320(%rsp), %ymm9 + vmovdqu 352(%rsp), %ymm10 + vmovdqu (%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpxor %ymm4, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vmovdqa %ymm0, %ymm11 + vpxor %ymm1, %ymm2, %ymm12 + vmovdqa %ymm3, %ymm13 + vmovdqu 32(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 64(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 96(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 384(%rsp), %ymm7 + vmovdqu 416(%rsp), %ymm8 + vmovdqu 448(%rsp), %ymm9 + vmovdqu 480(%rsp), %ymm10 + vmovdqu 128(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 160(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 192(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 224(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vpclmulqdq $0x01, %ymm11, %ymm14, %ymm5 + vpshufd $0x4e, %ymm11, %ymm11 + vpxor %ymm5, %ymm12, %ymm12 + vpxor %ymm11, %ymm12, %ymm12 + vpclmulqdq $0x01, %ymm12, %ymm14, %ymm5 + vpshufd $0x4e, %ymm12, %ymm12 + vpxor %ymm5, %ymm13, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vextracti128 $0x01, %ymm13, %xmm0 + vpxor %xmm0, %xmm13, %xmm15 + addl $0x100, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_encrypt_vaes_calc_aad_l256 +L_AES_GCM_encrypt_vaes_calc_aad_a256: + movl %r11d, %r13d + andl $0xffffff80, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_encrypt_vaes_calc_aad_a128 + # 128 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu 96(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 64(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 32(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu (%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vbroadcasti128 L_vaes_aes_gcm_bswap_mask(%rip), %ymm6 + vpxor %ymm4, %ymm4, %ymm4 + vinserti128 $0x00, %xmm15, %ymm4, %ymm4 + vmovdqu (%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpxor %ymm4, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vmovdqa %ymm0, %ymm11 + vpxor %ymm1, %ymm2, %ymm12 + vmovdqa %ymm3, %ymm13 + vmovdqu 32(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 64(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 96(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vpclmulqdq $0x01, %ymm11, %ymm14, %ymm5 + vpshufd $0x4e, %ymm11, %ymm11 + vpxor %ymm5, %ymm12, %ymm12 + vpxor %ymm11, %ymm12, %ymm12 + vpclmulqdq $0x01, %ymm12, %ymm14, %ymm5 + vpshufd $0x4e, %ymm12, %ymm12 + vpxor %ymm5, %ymm13, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vextracti128 $0x01, %ymm13, %xmm0 + vpxor %xmm0, %xmm13, %xmm15 + addl $0x80, %ecx +L_AES_GCM_encrypt_vaes_calc_aad_a128: + movl %r11d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_encrypt_vaes_calc_aad_arem + vmovdqu (%rsp), %xmm6 +L_AES_GCM_encrypt_vaes_calc_aad_rem: + # 16 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu (%rbx), %xmm7 + vpshufb L_vaes_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7 + vpxor %xmm7, %xmm15, %xmm15 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm6, %xmm15, %xmm7 + vpclmulqdq $0x01, %xmm6, %xmm15, %xmm8 + vpclmulqdq $16, %xmm6, %xmm15, %xmm9 + vpclmulqdq $0x11, %xmm6, %xmm15, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm15 + addl $16, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_encrypt_vaes_calc_aad_rem +L_AES_GCM_encrypt_vaes_calc_aad_arem: + vmovdqu 560(%rsp), %xmm6 + vmovdqu 576(%rsp), %xmm5 + jmp L_AES_GCM_encrypt_vaes_calc_aad_partial +L_AES_GCM_encrypt_vaes_calc_aad_ser: + movl %r11d, %edx andl $0xfffffff0, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_vaes_calc_aad_partial L_AES_GCM_encrypt_vaes_calc_aad_16_loop: vmovdqu (%r12,%rcx,1), %xmm7 vpshufb L_vaes_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7 @@ -17373,6 +18241,7 @@ L_AES_GCM_encrypt_vaes_calc_aad_16_loop: addl $16, %ecx cmpl %edx, %ecx jl L_AES_GCM_encrypt_vaes_calc_aad_16_loop +L_AES_GCM_encrypt_vaes_calc_aad_partial: movl %r11d, %edx cmpl %edx, %ecx je L_AES_GCM_encrypt_vaes_calc_aad_done @@ -18476,7 +19345,7 @@ L_AES_GCM_encrypt_vaes_store_tag_16: vmovdqu %xmm0, (%r8) L_AES_GCM_encrypt_vaes_store_tag_done: vzeroupper - addq $0x230, %rsp + addq $0x250, %rsp popq %r15 popq %r14 popq %rbx @@ -18512,7 +19381,7 @@ _AES_GCM_decrypt_vaes: movq 80(%rsp), %r15 movl 88(%rsp), %r10d movq 96(%rsp), %rbp - subq $0x220, %rsp + subq $0x250, %rsp vpxor %xmm5, %xmm5, %xmm5 vpxor %xmm15, %xmm15, %xmm15 cmpl $12, %ebx @@ -18806,9 +19675,527 @@ L_AES_GCM_decrypt_vaes_iv_done: cmpl $0x00, %edx je L_AES_GCM_decrypt_vaes_calc_aad_done xorl %ecx, %ecx - cmpl $16, %edx - jl L_AES_GCM_decrypt_vaes_calc_aad_lt16 + cmpl $0x80, %edx + jl L_AES_GCM_decrypt_vaes_calc_aad_ser + vmovdqu %xmm6, 560(%rsp) + vmovdqu %xmm5, 576(%rsp) + vpsrlq $63, %xmm6, %xmm8 + vpsllq $0x01, %xmm6, %xmm7 + vpslldq $8, %xmm8, %xmm8 + vpor %xmm8, %xmm7, %xmm7 + vpshufd $0xff, %xmm6, %xmm6 + vpsrad $31, %xmm6, %xmm6 + vpand L_vaes_aes_gcm_mod2_128(%rip), %xmm6, %xmm6 + vpxor %xmm7, %xmm6, %xmm6 + vmovdqa %xmm15, %xmm2 + # H ^ 1 + vmovdqu %xmm6, (%rsp) + # H ^ 2 + vpclmulqdq $0x00, %xmm6, %xmm6, %xmm7 + vpclmulqdq $0x11, %xmm6, %xmm6, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm0 + vmovdqu %xmm0, 16(%rsp) + # H ^ 3 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm6, %xmm0, %xmm7 + vpclmulqdq $0x01, %xmm6, %xmm0, %xmm8 + vpclmulqdq $16, %xmm6, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm6, %xmm0, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm1 + vmovdqu %xmm1, 32(%rsp) + # H ^ 4 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm3 + vmovdqu %xmm3, 48(%rsp) + # H ^ 5 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 64(%rsp) + # H ^ 6 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm7 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 80(%rsp) + # H ^ 7 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm1, %xmm3, %xmm7 + vpclmulqdq $0x01, %xmm1, %xmm3, %xmm8 + vpclmulqdq $16, %xmm1, %xmm3, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm3, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 96(%rsp) + # H ^ 8 + vpclmulqdq $0x00, %xmm3, %xmm3, %xmm7 + vpclmulqdq $0x11, %xmm3, %xmm3, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 112(%rsp) + cmpl $0x100, %edx + jl L_AES_GCM_decrypt_vaes_calc_aad_no_ext + # H ^ 9 + vmovdqu 48(%rsp), %xmm0 + vmovdqu 64(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 128(%rsp) + # H ^ 10 + vmovdqu 64(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 144(%rsp) + # H ^ 11 + vmovdqu 64(%rsp), %xmm0 + vmovdqu 80(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 160(%rsp) + # H ^ 12 + vmovdqu 80(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 176(%rsp) + # H ^ 13 + vmovdqu 80(%rsp), %xmm0 + vmovdqu 96(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 192(%rsp) + # H ^ 14 + vmovdqu 96(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 208(%rsp) + # H ^ 15 + vmovdqu 96(%rsp), %xmm0 + vmovdqu 112(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 224(%rsp) + # H ^ 16 + vmovdqu 112(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 240(%rsp) + vmovdqu 224(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 192(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 160(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu 128(%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vmovdqu %ymm7, 256(%rsp) + vmovdqu %ymm8, 288(%rsp) + vmovdqu %ymm9, 320(%rsp) + vmovdqu %ymm10, 352(%rsp) + vmovdqu 96(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 64(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 32(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu (%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vmovdqu %ymm7, 384(%rsp) + vmovdqu %ymm8, 416(%rsp) + vmovdqu %ymm9, 448(%rsp) + vmovdqu %ymm10, 480(%rsp) +L_AES_GCM_decrypt_vaes_calc_aad_no_ext: + vbroadcasti128 L_vaes_aes_gcm_mod2_128(%rip), %ymm14 + movl %r11d, %r13d + andl $0xffffff00, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_decrypt_vaes_calc_aad_a256 +L_AES_GCM_decrypt_vaes_calc_aad_l256: + # 256 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vbroadcasti128 L_vaes_aes_gcm_bswap_mask(%rip), %ymm6 + vpxor %ymm4, %ymm4, %ymm4 + vinserti128 $0x00, %xmm15, %ymm4, %ymm4 + vmovdqu 256(%rsp), %ymm7 + vmovdqu 288(%rsp), %ymm8 + vmovdqu 320(%rsp), %ymm9 + vmovdqu 352(%rsp), %ymm10 + vmovdqu (%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpxor %ymm4, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vmovdqa %ymm0, %ymm11 + vpxor %ymm1, %ymm2, %ymm12 + vmovdqa %ymm3, %ymm13 + vmovdqu 32(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 64(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 96(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 384(%rsp), %ymm7 + vmovdqu 416(%rsp), %ymm8 + vmovdqu 448(%rsp), %ymm9 + vmovdqu 480(%rsp), %ymm10 + vmovdqu 128(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 160(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 192(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 224(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vpclmulqdq $0x01, %ymm11, %ymm14, %ymm5 + vpshufd $0x4e, %ymm11, %ymm11 + vpxor %ymm5, %ymm12, %ymm12 + vpxor %ymm11, %ymm12, %ymm12 + vpclmulqdq $0x01, %ymm12, %ymm14, %ymm5 + vpshufd $0x4e, %ymm12, %ymm12 + vpxor %ymm5, %ymm13, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vextracti128 $0x01, %ymm13, %xmm0 + vpxor %xmm0, %xmm13, %xmm15 + addl $0x100, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_decrypt_vaes_calc_aad_l256 +L_AES_GCM_decrypt_vaes_calc_aad_a256: + movl %r11d, %r13d + andl $0xffffff80, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_decrypt_vaes_calc_aad_a128 + # 128 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu 96(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 64(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 32(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu (%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vbroadcasti128 L_vaes_aes_gcm_bswap_mask(%rip), %ymm6 + vpxor %ymm4, %ymm4, %ymm4 + vinserti128 $0x00, %xmm15, %ymm4, %ymm4 + vmovdqu (%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpxor %ymm4, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vmovdqa %ymm0, %ymm11 + vpxor %ymm1, %ymm2, %ymm12 + vmovdqa %ymm3, %ymm13 + vmovdqu 32(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 64(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 96(%rbx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vpclmulqdq $0x01, %ymm11, %ymm14, %ymm5 + vpshufd $0x4e, %ymm11, %ymm11 + vpxor %ymm5, %ymm12, %ymm12 + vpxor %ymm11, %ymm12, %ymm12 + vpclmulqdq $0x01, %ymm12, %ymm14, %ymm5 + vpshufd $0x4e, %ymm12, %ymm12 + vpxor %ymm5, %ymm13, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vextracti128 $0x01, %ymm13, %xmm0 + vpxor %xmm0, %xmm13, %xmm15 + addl $0x80, %ecx +L_AES_GCM_decrypt_vaes_calc_aad_a128: + movl %r11d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_decrypt_vaes_calc_aad_arem + vmovdqu (%rsp), %xmm6 +L_AES_GCM_decrypt_vaes_calc_aad_rem: + # 16 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu (%rbx), %xmm7 + vpshufb L_vaes_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7 + vpxor %xmm7, %xmm15, %xmm15 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm6, %xmm15, %xmm7 + vpclmulqdq $0x01, %xmm6, %xmm15, %xmm8 + vpclmulqdq $16, %xmm6, %xmm15, %xmm9 + vpclmulqdq $0x11, %xmm6, %xmm15, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm15 + addl $16, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_decrypt_vaes_calc_aad_rem +L_AES_GCM_decrypt_vaes_calc_aad_arem: + vmovdqu 560(%rsp), %xmm6 + vmovdqu 576(%rsp), %xmm5 + jmp L_AES_GCM_decrypt_vaes_calc_aad_partial +L_AES_GCM_decrypt_vaes_calc_aad_ser: + movl %r11d, %edx andl $0xfffffff0, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_vaes_calc_aad_partial L_AES_GCM_decrypt_vaes_calc_aad_16_loop: vmovdqu (%r12,%rcx,1), %xmm7 vpshufb L_vaes_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7 @@ -18859,6 +20246,7 @@ L_AES_GCM_decrypt_vaes_calc_aad_16_loop: addl $16, %ecx cmpl %edx, %ecx jl L_AES_GCM_decrypt_vaes_calc_aad_16_loop +L_AES_GCM_decrypt_vaes_calc_aad_partial: movl %r11d, %edx cmpl %edx, %ecx je L_AES_GCM_decrypt_vaes_calc_aad_done @@ -19914,7 +21302,7 @@ L_AES_GCM_decrypt_vaes_cmp_tag_16: L_AES_GCM_decrypt_vaes_cmp_tag_done: movl %ebx, (%rbp) vzeroupper - addq $0x220, %rsp + addq $0x250, %rsp popq %rbp popq %r15 popq %r14 @@ -20255,61 +21643,523 @@ AES_GCM_aad_update_vaes: .p2align 4 _AES_GCM_aad_update_vaes: #endif /* __APPLE__ */ - movq %rcx, %rax - vmovdqa (%rdx), %xmm5 - vmovdqa (%rax), %xmm6 - xorl %ecx, %ecx -L_AES_GCM_aad_update_vaes_16_loop: - vmovdqu (%rdi,%rcx,1), %xmm7 + movq %rdx, %r8 + movq %rcx, %r9 + subq $0x200, %rsp + vmovdqa (%r8), %xmm15 + vmovdqa (%r9), %xmm6 + xorl %edx, %edx + vpsrlq $63, %xmm6, %xmm8 + vpsllq $0x01, %xmm6, %xmm7 + vpslldq $8, %xmm8, %xmm8 + vpor %xmm8, %xmm7, %xmm7 + vpshufd $0xff, %xmm6, %xmm6 + vpsrad $31, %xmm6, %xmm6 + vpand L_vaes_aes_gcm_mod2_128(%rip), %xmm6, %xmm6 + vpxor %xmm7, %xmm6, %xmm6 + vmovdqa %xmm15, %xmm2 + # H ^ 1 + vmovdqu %xmm6, (%rsp) + # H ^ 2 + vpclmulqdq $0x00, %xmm6, %xmm6, %xmm7 + vpclmulqdq $0x11, %xmm6, %xmm6, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm0 + vmovdqu %xmm0, 16(%rsp) + # H ^ 3 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm6, %xmm0, %xmm7 + vpclmulqdq $0x01, %xmm6, %xmm0, %xmm8 + vpclmulqdq $16, %xmm6, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm6, %xmm0, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm1 + vmovdqu %xmm1, 32(%rsp) + # H ^ 4 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm3 + vmovdqu %xmm3, 48(%rsp) + # H ^ 5 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 64(%rsp) + # H ^ 6 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm7 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 80(%rsp) + # H ^ 7 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm1, %xmm3, %xmm7 + vpclmulqdq $0x01, %xmm1, %xmm3, %xmm8 + vpclmulqdq $16, %xmm1, %xmm3, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm3, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 96(%rsp) + # H ^ 8 + vpclmulqdq $0x00, %xmm3, %xmm3, %xmm7 + vpclmulqdq $0x11, %xmm3, %xmm3, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 112(%rsp) + cmpl $0x100, %esi + jl L_AES_GCM_aad_update_vaes_no_ext + # H ^ 9 + vmovdqu 48(%rsp), %xmm0 + vmovdqu 64(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 128(%rsp) + # H ^ 10 + vmovdqu 64(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 144(%rsp) + # H ^ 11 + vmovdqu 64(%rsp), %xmm0 + vmovdqu 80(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 160(%rsp) + # H ^ 12 + vmovdqu 80(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 176(%rsp) + # H ^ 13 + vmovdqu 80(%rsp), %xmm0 + vmovdqu 96(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 192(%rsp) + # H ^ 14 + vmovdqu 96(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 208(%rsp) + # H ^ 15 + vmovdqu 96(%rsp), %xmm0 + vmovdqu 112(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm7 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 224(%rsp) + # H ^ 16 + vmovdqu 112(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm7 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm10 + vpxor %xmm8, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm4 + vmovdqu %xmm4, 240(%rsp) + vmovdqu 224(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 192(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 160(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu 128(%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vmovdqu %ymm7, 256(%rsp) + vmovdqu %ymm8, 288(%rsp) + vmovdqu %ymm9, 320(%rsp) + vmovdqu %ymm10, 352(%rsp) + vmovdqu 96(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 64(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 32(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu (%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vmovdqu %ymm7, 384(%rsp) + vmovdqu %ymm8, 416(%rsp) + vmovdqu %ymm9, 448(%rsp) + vmovdqu %ymm10, 480(%rsp) +L_AES_GCM_aad_update_vaes_no_ext: + vbroadcasti128 L_vaes_aes_gcm_mod2_128(%rip), %ymm14 + movl %esi, %eax + andl $0xffffff00, %eax + cmpl %eax, %edx + jge L_AES_GCM_aad_update_vaes_after_256 +L_AES_GCM_aad_update_vaes_loop_256: + # 256 bytes of input + leaq (%rdi,%rdx,1), %rcx + vbroadcasti128 L_vaes_aes_gcm_bswap_mask(%rip), %ymm6 + vpxor %ymm4, %ymm4, %ymm4 + vinserti128 $0x00, %xmm15, %ymm4, %ymm4 + vmovdqu 256(%rsp), %ymm7 + vmovdqu 288(%rsp), %ymm8 + vmovdqu 320(%rsp), %ymm9 + vmovdqu 352(%rsp), %ymm10 + vmovdqu (%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpxor %ymm4, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vmovdqa %ymm0, %ymm11 + vpxor %ymm1, %ymm2, %ymm12 + vmovdqa %ymm3, %ymm13 + vmovdqu 32(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 64(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 96(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 384(%rsp), %ymm7 + vmovdqu 416(%rsp), %ymm8 + vmovdqu 448(%rsp), %ymm9 + vmovdqu 480(%rsp), %ymm10 + vmovdqu 128(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 160(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 192(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 224(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vpclmulqdq $0x01, %ymm11, %ymm14, %ymm5 + vpshufd $0x4e, %ymm11, %ymm11 + vpxor %ymm5, %ymm12, %ymm12 + vpxor %ymm11, %ymm12, %ymm12 + vpclmulqdq $0x01, %ymm12, %ymm14, %ymm5 + vpshufd $0x4e, %ymm12, %ymm12 + vpxor %ymm5, %ymm13, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vextracti128 $0x01, %ymm13, %xmm0 + vpxor %xmm0, %xmm13, %xmm15 + addl $0x100, %edx + cmpl %eax, %edx + jl L_AES_GCM_aad_update_vaes_loop_256 +L_AES_GCM_aad_update_vaes_after_256: + movl %esi, %eax + andl $0xffffff80, %eax + cmpl %eax, %edx + jge L_AES_GCM_aad_update_vaes_after_128 + # 128 bytes of input + leaq (%rdi,%rdx,1), %rcx + vmovdqu 96(%rsp), %ymm7 + vpermq $0x4e, %ymm7, %ymm7 + vmovdqu 64(%rsp), %ymm8 + vpermq $0x4e, %ymm8, %ymm8 + vmovdqu 32(%rsp), %ymm9 + vpermq $0x4e, %ymm9, %ymm9 + vmovdqu (%rsp), %ymm10 + vpermq $0x4e, %ymm10, %ymm10 + vbroadcasti128 L_vaes_aes_gcm_bswap_mask(%rip), %ymm6 + vpxor %ymm4, %ymm4, %ymm4 + vinserti128 $0x00, %xmm15, %ymm4, %ymm4 + vmovdqu (%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpxor %ymm4, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm7, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm7, %ymm5, %ymm1 + vpclmulqdq $16, %ymm7, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm7, %ymm5, %ymm3 + vmovdqa %ymm0, %ymm11 + vpxor %ymm1, %ymm2, %ymm12 + vmovdqa %ymm3, %ymm13 + vmovdqu 32(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm8, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm8, %ymm5, %ymm1 + vpclmulqdq $16, %ymm8, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm8, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 64(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm9, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm9, %ymm5, %ymm1 + vpclmulqdq $16, %ymm9, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm9, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vmovdqu 96(%rcx), %ymm5 + vpshufb %ymm6, %ymm5, %ymm5 + vpclmulqdq $0x00, %ymm10, %ymm5, %ymm0 + vpclmulqdq $0x01, %ymm10, %ymm5, %ymm1 + vpclmulqdq $16, %ymm10, %ymm5, %ymm2 + vpclmulqdq $0x11, %ymm10, %ymm5, %ymm3 + vpxor %ymm0, %ymm11, %ymm11 + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vpxor %ymm3, %ymm13, %ymm13 + vpclmulqdq $0x01, %ymm11, %ymm14, %ymm5 + vpshufd $0x4e, %ymm11, %ymm11 + vpxor %ymm5, %ymm12, %ymm12 + vpxor %ymm11, %ymm12, %ymm12 + vpclmulqdq $0x01, %ymm12, %ymm14, %ymm5 + vpshufd $0x4e, %ymm12, %ymm12 + vpxor %ymm5, %ymm13, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vextracti128 $0x01, %ymm13, %xmm0 + vpxor %xmm0, %xmm13, %xmm15 + addl $0x80, %edx +L_AES_GCM_aad_update_vaes_after_128: + vmovdqu (%rsp), %xmm6 + movl %esi, %eax + andl $0xfffffff0, %eax + cmpl %eax, %edx + jge L_AES_GCM_aad_update_vaes_done +L_AES_GCM_aad_update_vaes_tail: + # 16 bytes of input + vmovdqu (%rdi,%rdx,1), %xmm7 vpshufb L_vaes_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7 - vpxor %xmm7, %xmm5, %xmm5 - # ghash_gfmul_avx - vpshufd $0x4e, %xmm5, %xmm1 - vpshufd $0x4e, %xmm6, %xmm2 - vpclmulqdq $0x11, %xmm5, %xmm6, %xmm3 - vpclmulqdq $0x00, %xmm5, %xmm6, %xmm0 - vpxor %xmm5, %xmm1, %xmm1 - vpxor %xmm6, %xmm2, %xmm2 - vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 - vpxor %xmm0, %xmm1, %xmm1 - vpxor %xmm3, %xmm1, %xmm1 - vmovdqa %xmm0, %xmm4 - vmovdqa %xmm3, %xmm5 - vpslldq $8, %xmm1, %xmm2 - vpsrldq $8, %xmm1, %xmm1 - vpxor %xmm2, %xmm4, %xmm4 - vpxor %xmm1, %xmm5, %xmm5 - vpsrld $31, %xmm4, %xmm0 - vpsrld $31, %xmm5, %xmm1 - vpslld $0x01, %xmm4, %xmm4 - vpslld $0x01, %xmm5, %xmm5 - vpsrldq $12, %xmm0, %xmm2 - vpslldq $4, %xmm0, %xmm0 - vpslldq $4, %xmm1, %xmm1 - vpor %xmm2, %xmm5, %xmm5 - vpor %xmm0, %xmm4, %xmm4 - vpor %xmm1, %xmm5, %xmm5 - vpslld $31, %xmm4, %xmm0 - vpslld $30, %xmm4, %xmm1 - vpslld $25, %xmm4, %xmm2 - vpxor %xmm1, %xmm0, %xmm0 - vpxor %xmm2, %xmm0, %xmm0 - vmovdqa %xmm0, %xmm1 - vpsrldq $4, %xmm1, %xmm1 - vpslldq $12, %xmm0, %xmm0 - vpxor %xmm0, %xmm4, %xmm4 - vpsrld $0x01, %xmm4, %xmm2 - vpsrld $2, %xmm4, %xmm3 - vpsrld $7, %xmm4, %xmm0 - vpxor %xmm3, %xmm2, %xmm2 - vpxor %xmm0, %xmm2, %xmm2 - vpxor %xmm1, %xmm2, %xmm2 - vpxor %xmm4, %xmm2, %xmm2 - vpxor %xmm2, %xmm5, %xmm5 - addl $16, %ecx - cmpl %esi, %ecx - jl L_AES_GCM_aad_update_vaes_16_loop - vmovdqa %xmm5, (%rdx) + vpxor %xmm7, %xmm15, %xmm15 + # ghash_gfmul_red_avx + vpclmulqdq $0x00, %xmm6, %xmm15, %xmm7 + vpclmulqdq $0x01, %xmm6, %xmm15, %xmm8 + vpclmulqdq $16, %xmm6, %xmm15, %xmm9 + vpclmulqdq $0x11, %xmm6, %xmm15, %xmm10 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqa L_vaes_aes_gcm_mod2_128(%rip), %xmm9 + vpclmulqdq $0x01, %xmm7, %xmm9, %xmm11 + vpshufd $0x4e, %xmm7, %xmm7 + vpxor %xmm11, %xmm8, %xmm8 + vpxor %xmm7, %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm8, %xmm9, %xmm11 + vpshufd $0x4e, %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm8, %xmm10, %xmm10 + vmovdqa %xmm10, %xmm15 + addl $16, %edx + cmpl %eax, %edx + jl L_AES_GCM_aad_update_vaes_tail +L_AES_GCM_aad_update_vaes_done: + vmovdqa %xmm15, (%r8) + vzeroupper + addq $0x200, %rsp repz retq #ifndef __APPLE__ .size AES_GCM_aad_update_vaes,.-AES_GCM_aad_update_vaes @@ -22923,7 +24773,7 @@ _AES_GCM_encrypt_avx512: movl 64(%rsp), %r14d movq 72(%rsp), %r15 movl 80(%rsp), %r10d - subq $0x440, %rsp + subq $0x460, %rsp vpxor %xmm4, %xmm4, %xmm4 vpxor %xmm6, %xmm6, %xmm6 movl %ebx, %edx @@ -23217,9 +25067,426 @@ L_AES_GCM_encrypt_avx512_iv_done: cmpl $0x00, %edx je L_AES_GCM_encrypt_avx512_calc_aad_done xorl %ecx, %ecx - cmpl $16, %edx - jl L_AES_GCM_encrypt_avx512_calc_aad_lt16 + cmpl $0x80, %edx + jl L_AES_GCM_encrypt_avx512_calc_aad_ser + vmovdqu %xmm5, 1088(%rsp) + vmovdqu %xmm4, 1104(%rsp) + vpsrlq $63, %xmm5, %xmm9 + vpsllq $0x01, %xmm5, %xmm8 + vpslldq $8, %xmm9, %xmm9 + vpor %xmm9, %xmm8, %xmm8 + vpshufd $0xff, %xmm5, %xmm5 + vpsrad $31, %xmm5, %xmm5 + vpand L_avx512_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 + vpxor %xmm8, %xmm5, %xmm5 + vmovdqa %xmm6, %xmm2 + # H ^ 1 + vmovdqu %xmm5, (%rsp) + # H ^ 2 + vpclmulqdq $0x00, %xmm5, %xmm5, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm5, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm0 + vmovdqu %xmm0, 16(%rsp) + # H ^ 3 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpxor %xmm5, %xmm9, %xmm9 + vpshufd $0x4e, %xmm0, %xmm10 + vpxor %xmm0, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm1 + vmovdqu %xmm1, 32(%rsp) + # H ^ 4 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm3 + vmovdqu %xmm3, 48(%rsp) + # H ^ 5 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 64(%rsp) + # H ^ 6 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 80(%rsp) + # H ^ 7 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm1, %xmm9 + vpxor %xmm1, %xmm9, %xmm9 + vpshufd $0x4e, %xmm3, %xmm10 + vpxor %xmm3, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm1, %xmm3, %xmm8 + vpclmulqdq $0x11, %xmm1, %xmm3, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 96(%rsp) + # H ^ 8 + vpclmulqdq $0x00, %xmm3, %xmm3, %xmm8 + vpclmulqdq $0x11, %xmm3, %xmm3, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 112(%rsp) + # H ^ 9 + vmovdqu 48(%rsp), %xmm0 + vmovdqu 64(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 128(%rsp) + # H ^ 10 + vmovdqu 64(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 144(%rsp) + # H ^ 11 + vmovdqu 64(%rsp), %xmm0 + vmovdqu 80(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 160(%rsp) + # H ^ 12 + vmovdqu 80(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 176(%rsp) + # H ^ 13 + vmovdqu 80(%rsp), %xmm0 + vmovdqu 96(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 192(%rsp) + # H ^ 14 + vmovdqu 96(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 208(%rsp) + # H ^ 15 + vmovdqu 96(%rsp), %xmm0 + vmovdqu 112(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 224(%rsp) + # H ^ 16 + vmovdqu 112(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 240(%rsp) + vbroadcasti32x4 L_avx512_aes_gcm_bswap_mask(%rip), %zmm30 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 + vmovdqu64 192(%rsp), %zmm23 + vshufi64x2 $27, %zmm23, %zmm23, %zmm23 + vmovdqu64 128(%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vmovdqu64 64(%rsp), %zmm25 + vshufi64x2 $27, %zmm25, %zmm25, %zmm25 + vmovdqu64 (%rsp), %zmm26 + vshufi64x2 $27, %zmm26, %zmm26, %zmm26 + movl %r11d, %r13d + andl $0xffffff00, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_encrypt_avx512_calc_aad_a256 +L_AES_GCM_encrypt_avx512_calc_aad_l256: + # 256 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm23, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm23, %zmm21, %zmm17 + vpclmulqdq $16, %zmm23, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm23, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm27 + vpxorq %zmm17, %zmm18, %zmm28 + vmovdqa64 %zmm19, %zmm29 + vmovdqu64 64(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vmovdqu64 128(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm25, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm25, %zmm21, %zmm17 + vpclmulqdq $16, %zmm25, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm25, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vmovdqu64 192(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm26, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm26, %zmm21, %zmm17 + vpclmulqdq $16, %zmm26, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm26, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vpclmulqdq $0x01, %zmm27, %zmm31, %zmm21 + vpshufd $0x4e, %zmm27, %zmm27 + vpternlogq $0x96, %zmm21, %zmm27, %zmm28 + vpclmulqdq $0x01, %zmm28, %zmm31, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vextracti32x4 $0x01, %zmm29, %xmm0 + vextracti32x4 $2, %zmm29, %xmm4 + vextracti32x4 $3, %zmm29, %xmm5 + vpxorq %xmm0, %xmm29, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + addl $0x100, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_encrypt_avx512_calc_aad_l256 +L_AES_GCM_encrypt_avx512_calc_aad_a256: + movl %r11d, %r13d + andl $0xffffff80, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_encrypt_avx512_calc_aad_a128 + # 128 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu64 64(%rsp), %zmm23 + vshufi64x2 $27, %zmm23, %zmm23, %zmm23 + vmovdqu64 (%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm23, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm23, %zmm21, %zmm17 + vpclmulqdq $16, %zmm23, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm23, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm27 + vpxorq %zmm17, %zmm18, %zmm28 + vmovdqa64 %zmm19, %zmm29 + vmovdqu64 64(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 + vpclmulqdq $0x01, %zmm27, %zmm31, %zmm21 + vpshufd $0x4e, %zmm27, %zmm27 + vpternlogq $0x96, %zmm21, %zmm27, %zmm28 + vpclmulqdq $0x01, %zmm28, %zmm31, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vextracti32x4 $0x01, %zmm29, %xmm0 + vextracti32x4 $2, %zmm29, %xmm4 + vextracti32x4 $3, %zmm29, %xmm5 + vpxorq %xmm0, %xmm29, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + addl $0x80, %ecx +L_AES_GCM_encrypt_avx512_calc_aad_a128: + movl %r11d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_encrypt_avx512_calc_aad_arem + vmovdqu (%rsp), %xmm5 +L_AES_GCM_encrypt_avx512_calc_aad_rem: + # 16 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu (%rbx), %xmm8 + vpshufb L_avx512_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm6, %xmm6 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpxor %xmm5, %xmm9, %xmm9 + vpshufd $0x4e, %xmm6, %xmm10 + vpxor %xmm6, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm6 + addl $16, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_encrypt_avx512_calc_aad_rem +L_AES_GCM_encrypt_avx512_calc_aad_arem: + vmovdqu 1088(%rsp), %xmm5 + vmovdqu 1104(%rsp), %xmm4 + jmp L_AES_GCM_encrypt_avx512_calc_aad_partial +L_AES_GCM_encrypt_avx512_calc_aad_ser: + movl %r11d, %edx andl $0xfffffff0, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_avx512_calc_aad_partial L_AES_GCM_encrypt_avx512_calc_aad_16_loop: vmovdqu (%r12,%rcx,1), %xmm8 vpshufb L_avx512_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 @@ -23270,6 +25537,7 @@ L_AES_GCM_encrypt_avx512_calc_aad_16_loop: addl $16, %ecx cmpl %edx, %ecx jl L_AES_GCM_encrypt_avx512_calc_aad_16_loop +L_AES_GCM_encrypt_avx512_calc_aad_partial: movl %r11d, %edx cmpl %edx, %ecx je L_AES_GCM_encrypt_avx512_calc_aad_done @@ -23346,7 +25614,7 @@ L_AES_GCM_encrypt_avx512_calc_aad_done: vpxor %xmm8, %xmm5, %xmm5 vmovdqu %xmm4, 1024(%rsp) xorl %ebx, %ebx - cmpl $0x100, %r9d + cmpl $0x80, %r9d jl L_AES_GCM_encrypt_avx512_done_128 vmovdqa %xmm6, %xmm2 # H ^ 1 @@ -23460,6 +25728,8 @@ L_AES_GCM_encrypt_avx512_calc_aad_done: vpternlogq $0x96, %xmm12, %xmm9, %xmm11 vmovdqa %xmm11, %xmm7 vmovdqu %xmm7, 112(%rsp) + cmpl $0x100, %r9d + jl L_AES_GCM_encrypt_avx512_no_ext2 # H ^ 9 vmovdqu 48(%rsp), %xmm0 vmovdqu 64(%rsp), %xmm1 @@ -23883,6 +26153,7 @@ L_AES_GCM_encrypt_avx512_calc_aad_done: vmovdqa %xmm11, %xmm7 vmovdqu %xmm7, 496(%rsp) L_AES_GCM_encrypt_avx512_no_ext: +L_AES_GCM_encrypt_avx512_no_ext2: vbroadcasti32x4 L_avx512_aes_gcm_bswap_epi64(%rip), %zmm22 vbroadcasti32x4 L_avx512_aes_gcm_bswap_mask(%rip), %zmm30 vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 @@ -24827,6 +27098,116 @@ L_AES_GCM_encrypt_avx512_last_ghash: vpxorq %xmm0, %xmm29, %xmm6 vpternlogq $0x96, %xmm4, %xmm5, %xmm6 L_AES_GCM_encrypt_avx512_after_256: + movl %r9d, %r13d + andl $0xffffff80, %r13d + cmpl %r13d, %ebx + jge L_AES_GCM_encrypt_avx512_after_128 + # 128 bytes of input + vbroadcasti32x4 (%r15), %zmm9 + vbroadcasti32x4 16(%r15), %zmm10 + vbroadcasti32x4 32(%r15), %zmm11 + vbroadcasti32x4 48(%r15), %zmm12 + vbroadcasti32x4 64(%r15), %zmm13 + vbroadcasti32x4 80(%r15), %zmm14 + vbroadcasti32x4 96(%r15), %zmm15 + vbroadcasti32x4 112(%r15), %zmm1 + vbroadcasti32x4 128(%r15), %zmm2 + vbroadcasti32x4 144(%r15), %zmm3 + vbroadcasti32x4 1024(%rsp), %zmm20 + vpaddd L_avx512_aes_gcm_inc_z0(%rip), %zmm20, %zmm16 + vpshufb %zmm22, %zmm16, %zmm16 + vpaddd L_avx512_aes_gcm_inc_z1(%rip), %zmm20, %zmm17 + vpshufb %zmm22, %zmm17, %zmm17 + vmovdqu 1024(%rsp), %xmm8 + vpaddd L_avx512_aes_gcm_eight(%rip), %xmm8, %xmm8 + vmovdqu %xmm8, 1024(%rsp) + vpxorq %zmm9, %zmm16, %zmm16 + vpxorq %zmm9, %zmm17, %zmm17 + vaesenc %zmm10, %zmm16, %zmm16 + vaesenc %zmm10, %zmm17, %zmm17 + vaesenc %zmm11, %zmm16, %zmm16 + vaesenc %zmm11, %zmm17, %zmm17 + vaesenc %zmm12, %zmm16, %zmm16 + vaesenc %zmm12, %zmm17, %zmm17 + vaesenc %zmm13, %zmm16, %zmm16 + vaesenc %zmm13, %zmm17, %zmm17 + vaesenc %zmm14, %zmm16, %zmm16 + vaesenc %zmm14, %zmm17, %zmm17 + vaesenc %zmm15, %zmm16, %zmm16 + vaesenc %zmm15, %zmm17, %zmm17 + vaesenc %zmm1, %zmm16, %zmm16 + vaesenc %zmm1, %zmm17, %zmm17 + vaesenc %zmm2, %zmm16, %zmm16 + vaesenc %zmm2, %zmm17, %zmm17 + vaesenc %zmm3, %zmm16, %zmm16 + vaesenc %zmm3, %zmm17, %zmm17 + cmpl $11, %r10d + vbroadcasti32x4 160(%r15), %zmm20 + jl L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 176(%r15), %zmm20 + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + cmpl $13, %r10d + vbroadcasti32x4 192(%r15), %zmm20 + jl L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 208(%r15), %zmm20 + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 224(%r15), %zmm20 +L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last: + vaesenclast %zmm20, %zmm16, %zmm16 + vaesenclast %zmm20, %zmm17, %zmm17 + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + vmovdqu64 (%rcx), %zmm21 + vpxorq %zmm21, %zmm16, %zmm16 + vmovdqu64 %zmm16, (%rdx) + vmovdqu64 64(%rcx), %zmm21 + vpxorq %zmm21, %zmm17, %zmm17 + vmovdqu64 %zmm17, 64(%rdx) + vmovdqu64 64(%rsp), %zmm23 + vshufi64x2 $27, %zmm23, %zmm23, %zmm23 + vmovdqu64 (%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rdx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm23, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm23, %zmm21, %zmm17 + vpclmulqdq $16, %zmm23, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm23, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm27 + vpxorq %zmm17, %zmm18, %zmm28 + vmovdqa64 %zmm19, %zmm29 + vmovdqu64 64(%rdx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 + vpclmulqdq $0x01, %zmm27, %zmm31, %zmm21 + vpshufd $0x4e, %zmm27, %zmm27 + vpternlogq $0x96, %zmm21, %zmm27, %zmm28 + vpclmulqdq $0x01, %zmm28, %zmm31, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vextracti32x4 $0x01, %zmm29, %xmm0 + vextracti32x4 $2, %zmm29, %xmm4 + vextracti32x4 $3, %zmm29, %xmm5 + vpxorq %xmm0, %xmm29, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + addl $0x80, %ebx +L_AES_GCM_encrypt_avx512_after_128: vmovdqu (%rsp), %xmm5 L_AES_GCM_encrypt_avx512_done_128: movl %r9d, %edx @@ -25060,7 +27441,7 @@ L_AES_GCM_encrypt_avx512_store_tag_16: vmovdqu %xmm0, (%r8) L_AES_GCM_encrypt_avx512_store_tag_done: vzeroupper - addq $0x440, %rsp + addq $0x460, %rsp popq %r15 popq %r14 popq %rbx @@ -25096,7 +27477,7 @@ _AES_GCM_decrypt_avx512: movq 80(%rsp), %r15 movl 88(%rsp), %r10d movq 96(%rsp), %rbp - subq $0x420, %rsp + subq $0x460, %rsp vpxor %xmm4, %xmm4, %xmm4 vpxor %xmm6, %xmm6, %xmm6 cmpl $12, %ebx @@ -25390,9 +27771,426 @@ L_AES_GCM_decrypt_avx512_iv_done: cmpl $0x00, %edx je L_AES_GCM_decrypt_avx512_calc_aad_done xorl %ecx, %ecx - cmpl $16, %edx - jl L_AES_GCM_decrypt_avx512_calc_aad_lt16 + cmpl $0x80, %edx + jl L_AES_GCM_decrypt_avx512_calc_aad_ser + vmovdqu %xmm5, 1088(%rsp) + vmovdqu %xmm4, 1104(%rsp) + vpsrlq $63, %xmm5, %xmm9 + vpsllq $0x01, %xmm5, %xmm8 + vpslldq $8, %xmm9, %xmm9 + vpor %xmm9, %xmm8, %xmm8 + vpshufd $0xff, %xmm5, %xmm5 + vpsrad $31, %xmm5, %xmm5 + vpand L_avx512_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 + vpxor %xmm8, %xmm5, %xmm5 + vmovdqa %xmm6, %xmm2 + # H ^ 1 + vmovdqu %xmm5, (%rsp) + # H ^ 2 + vpclmulqdq $0x00, %xmm5, %xmm5, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm5, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm0 + vmovdqu %xmm0, 16(%rsp) + # H ^ 3 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpxor %xmm5, %xmm9, %xmm9 + vpshufd $0x4e, %xmm0, %xmm10 + vpxor %xmm0, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm1 + vmovdqu %xmm1, 32(%rsp) + # H ^ 4 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm3 + vmovdqu %xmm3, 48(%rsp) + # H ^ 5 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 64(%rsp) + # H ^ 6 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 80(%rsp) + # H ^ 7 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm1, %xmm9 + vpxor %xmm1, %xmm9, %xmm9 + vpshufd $0x4e, %xmm3, %xmm10 + vpxor %xmm3, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm1, %xmm3, %xmm8 + vpclmulqdq $0x11, %xmm1, %xmm3, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 96(%rsp) + # H ^ 8 + vpclmulqdq $0x00, %xmm3, %xmm3, %xmm8 + vpclmulqdq $0x11, %xmm3, %xmm3, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 112(%rsp) + # H ^ 9 + vmovdqu 48(%rsp), %xmm0 + vmovdqu 64(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 128(%rsp) + # H ^ 10 + vmovdqu 64(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 144(%rsp) + # H ^ 11 + vmovdqu 64(%rsp), %xmm0 + vmovdqu 80(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 160(%rsp) + # H ^ 12 + vmovdqu 80(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 176(%rsp) + # H ^ 13 + vmovdqu 80(%rsp), %xmm0 + vmovdqu 96(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 192(%rsp) + # H ^ 14 + vmovdqu 96(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 208(%rsp) + # H ^ 15 + vmovdqu 96(%rsp), %xmm0 + vmovdqu 112(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpxor %xmm0, %xmm9, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 224(%rsp) + # H ^ 16 + vmovdqu 112(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm11 + vpxor %xmm9, %xmm9, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm7 + vmovdqu %xmm7, 240(%rsp) + vbroadcasti32x4 L_avx512_aes_gcm_bswap_mask(%rip), %zmm30 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 + vmovdqu64 192(%rsp), %zmm23 + vshufi64x2 $27, %zmm23, %zmm23, %zmm23 + vmovdqu64 128(%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vmovdqu64 64(%rsp), %zmm25 + vshufi64x2 $27, %zmm25, %zmm25, %zmm25 + vmovdqu64 (%rsp), %zmm26 + vshufi64x2 $27, %zmm26, %zmm26, %zmm26 + movl %r11d, %r13d + andl $0xffffff00, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_decrypt_avx512_calc_aad_a256 +L_AES_GCM_decrypt_avx512_calc_aad_l256: + # 256 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm23, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm23, %zmm21, %zmm17 + vpclmulqdq $16, %zmm23, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm23, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm27 + vpxorq %zmm17, %zmm18, %zmm28 + vmovdqa64 %zmm19, %zmm29 + vmovdqu64 64(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vmovdqu64 128(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm25, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm25, %zmm21, %zmm17 + vpclmulqdq $16, %zmm25, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm25, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vmovdqu64 192(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm26, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm26, %zmm21, %zmm17 + vpclmulqdq $16, %zmm26, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm26, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vpclmulqdq $0x01, %zmm27, %zmm31, %zmm21 + vpshufd $0x4e, %zmm27, %zmm27 + vpternlogq $0x96, %zmm21, %zmm27, %zmm28 + vpclmulqdq $0x01, %zmm28, %zmm31, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vextracti32x4 $0x01, %zmm29, %xmm0 + vextracti32x4 $2, %zmm29, %xmm4 + vextracti32x4 $3, %zmm29, %xmm5 + vpxorq %xmm0, %xmm29, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + addl $0x100, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_decrypt_avx512_calc_aad_l256 +L_AES_GCM_decrypt_avx512_calc_aad_a256: + movl %r11d, %r13d + andl $0xffffff80, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_decrypt_avx512_calc_aad_a128 + # 128 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu64 64(%rsp), %zmm23 + vshufi64x2 $27, %zmm23, %zmm23, %zmm23 + vmovdqu64 (%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm23, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm23, %zmm21, %zmm17 + vpclmulqdq $16, %zmm23, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm23, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm27 + vpxorq %zmm17, %zmm18, %zmm28 + vmovdqa64 %zmm19, %zmm29 + vmovdqu64 64(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 + vpclmulqdq $0x01, %zmm27, %zmm31, %zmm21 + vpshufd $0x4e, %zmm27, %zmm27 + vpternlogq $0x96, %zmm21, %zmm27, %zmm28 + vpclmulqdq $0x01, %zmm28, %zmm31, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vextracti32x4 $0x01, %zmm29, %xmm0 + vextracti32x4 $2, %zmm29, %xmm4 + vextracti32x4 $3, %zmm29, %xmm5 + vpxorq %xmm0, %xmm29, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + addl $0x80, %ecx +L_AES_GCM_decrypt_avx512_calc_aad_a128: + movl %r11d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ecx + jge L_AES_GCM_decrypt_avx512_calc_aad_arem + vmovdqu (%rsp), %xmm5 +L_AES_GCM_decrypt_avx512_calc_aad_rem: + # 16 bytes of AAD + leaq (%r12,%rcx,1), %rbx + vmovdqu (%rbx), %xmm8 + vpshufb L_avx512_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm6, %xmm6 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpxor %xmm5, %xmm9, %xmm9 + vpshufd $0x4e, %xmm6, %xmm10 + vpxor %xmm6, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm11 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpternlogq $0x96, %xmm8, %xmm11, %xmm9 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm10 + vpclmulqdq $0x01, %xmm8, %xmm10, %xmm12 + vpshufd $0x4e, %xmm8, %xmm8 + vpternlogq $0x96, %xmm12, %xmm8, %xmm9 + vpclmulqdq $0x01, %xmm9, %xmm10, %xmm12 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm12, %xmm9, %xmm11 + vmovdqa %xmm11, %xmm6 + addl $16, %ecx + cmpl %r13d, %ecx + jl L_AES_GCM_decrypt_avx512_calc_aad_rem +L_AES_GCM_decrypt_avx512_calc_aad_arem: + vmovdqu 1088(%rsp), %xmm5 + vmovdqu 1104(%rsp), %xmm4 + jmp L_AES_GCM_decrypt_avx512_calc_aad_partial +L_AES_GCM_decrypt_avx512_calc_aad_ser: + movl %r11d, %edx andl $0xfffffff0, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_avx512_calc_aad_partial L_AES_GCM_decrypt_avx512_calc_aad_16_loop: vmovdqu (%r12,%rcx,1), %xmm8 vpshufb L_avx512_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 @@ -25443,6 +28241,7 @@ L_AES_GCM_decrypt_avx512_calc_aad_16_loop: addl $16, %ecx cmpl %edx, %ecx jl L_AES_GCM_decrypt_avx512_calc_aad_16_loop +L_AES_GCM_decrypt_avx512_calc_aad_partial: movl %r11d, %edx cmpl %edx, %ecx je L_AES_GCM_decrypt_avx512_calc_aad_done @@ -25519,7 +28318,7 @@ L_AES_GCM_decrypt_avx512_calc_aad_done: vpxor %xmm8, %xmm5, %xmm5 vmovdqu %xmm4, 1024(%rsp) xorl %ebx, %ebx - cmpl $0x100, %r9d + cmpl $0x80, %r9d jl L_AES_GCM_decrypt_avx512_done_128 vmovdqa %xmm6, %xmm2 # H ^ 1 @@ -25633,6 +28432,8 @@ L_AES_GCM_decrypt_avx512_calc_aad_done: vpternlogq $0x96, %xmm12, %xmm9, %xmm11 vmovdqa %xmm11, %xmm7 vmovdqu %xmm7, 112(%rsp) + cmpl $0x100, %r9d + jl L_AES_GCM_decrypt_avx512_no_ext2 # H ^ 9 vmovdqu 48(%rsp), %xmm0 vmovdqu 64(%rsp), %xmm1 @@ -26056,6 +28857,7 @@ L_AES_GCM_decrypt_avx512_calc_aad_done: vmovdqa %xmm11, %xmm7 vmovdqu %xmm7, 496(%rsp) L_AES_GCM_decrypt_avx512_no_ext: +L_AES_GCM_decrypt_avx512_no_ext2: vbroadcasti32x4 L_avx512_aes_gcm_bswap_epi64(%rip), %zmm22 vbroadcasti32x4 L_avx512_aes_gcm_bswap_mask(%rip), %zmm30 vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 @@ -26841,6 +29643,117 @@ L_AES_GCM_decrypt_avx512_t_avx512_ctr16_last: vmovdqu64 %zmm19, 192(%rdx) addl $0x100, %ebx L_AES_GCM_decrypt_avx512_after_256: + movl %r9d, %r13d + andl $0xffffff80, %r13d + cmpl %r13d, %ebx + jge L_AES_GCM_decrypt_avx512_after_128 + # 128 bytes of input + leaq (%rdi,%rbx,1), %rax + vmovdqu64 64(%rsp), %zmm23 + vshufi64x2 $27, %zmm23, %zmm23, %zmm23 + vmovdqu64 (%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rax), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm23, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm23, %zmm21, %zmm17 + vpclmulqdq $16, %zmm23, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm23, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm27 + vpxorq %zmm17, %zmm18, %zmm28 + vmovdqa64 %zmm19, %zmm29 + vmovdqu64 64(%rax), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 + vpclmulqdq $0x01, %zmm27, %zmm31, %zmm21 + vpshufd $0x4e, %zmm27, %zmm27 + vpternlogq $0x96, %zmm21, %zmm27, %zmm28 + vpclmulqdq $0x01, %zmm28, %zmm31, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vextracti32x4 $0x01, %zmm29, %xmm0 + vextracti32x4 $2, %zmm29, %xmm4 + vextracti32x4 $3, %zmm29, %xmm5 + vpxorq %xmm0, %xmm29, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + vbroadcasti32x4 (%r15), %zmm9 + vbroadcasti32x4 16(%r15), %zmm10 + vbroadcasti32x4 32(%r15), %zmm11 + vbroadcasti32x4 48(%r15), %zmm12 + vbroadcasti32x4 64(%r15), %zmm13 + vbroadcasti32x4 80(%r15), %zmm14 + vbroadcasti32x4 96(%r15), %zmm15 + vbroadcasti32x4 112(%r15), %zmm1 + vbroadcasti32x4 128(%r15), %zmm2 + vbroadcasti32x4 144(%r15), %zmm3 + vbroadcasti32x4 1024(%rsp), %zmm20 + vpaddd L_avx512_aes_gcm_inc_z0(%rip), %zmm20, %zmm16 + vpshufb %zmm22, %zmm16, %zmm16 + vpaddd L_avx512_aes_gcm_inc_z1(%rip), %zmm20, %zmm17 + vpshufb %zmm22, %zmm17, %zmm17 + vmovdqu 1024(%rsp), %xmm8 + vpaddd L_avx512_aes_gcm_eight(%rip), %xmm8, %xmm8 + vmovdqu %xmm8, 1024(%rsp) + vpxorq %zmm9, %zmm16, %zmm16 + vpxorq %zmm9, %zmm17, %zmm17 + vaesenc %zmm10, %zmm16, %zmm16 + vaesenc %zmm10, %zmm17, %zmm17 + vaesenc %zmm11, %zmm16, %zmm16 + vaesenc %zmm11, %zmm17, %zmm17 + vaesenc %zmm12, %zmm16, %zmm16 + vaesenc %zmm12, %zmm17, %zmm17 + vaesenc %zmm13, %zmm16, %zmm16 + vaesenc %zmm13, %zmm17, %zmm17 + vaesenc %zmm14, %zmm16, %zmm16 + vaesenc %zmm14, %zmm17, %zmm17 + vaesenc %zmm15, %zmm16, %zmm16 + vaesenc %zmm15, %zmm17, %zmm17 + vaesenc %zmm1, %zmm16, %zmm16 + vaesenc %zmm1, %zmm17, %zmm17 + vaesenc %zmm2, %zmm16, %zmm16 + vaesenc %zmm2, %zmm17, %zmm17 + vaesenc %zmm3, %zmm16, %zmm16 + vaesenc %zmm3, %zmm17, %zmm17 + cmpl $11, %r10d + vbroadcasti32x4 160(%r15), %zmm20 + jl L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 176(%r15), %zmm20 + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + cmpl $13, %r10d + vbroadcasti32x4 192(%r15), %zmm20 + jl L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 208(%r15), %zmm20 + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 224(%r15), %zmm20 +L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last: + vaesenclast %zmm20, %zmm16, %zmm16 + vaesenclast %zmm20, %zmm17, %zmm17 + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + vmovdqu64 (%rcx), %zmm21 + vpxorq %zmm21, %zmm16, %zmm16 + vmovdqu64 %zmm16, (%rdx) + vmovdqu64 64(%rcx), %zmm21 + vpxorq %zmm21, %zmm17, %zmm17 + vmovdqu64 %zmm17, 64(%rdx) + addl $0x80, %ebx +L_AES_GCM_decrypt_avx512_after_128: vmovdqu (%rsp), %xmm5 L_AES_GCM_decrypt_avx512_done_128: movl %r9d, %edx @@ -27029,7 +29942,7 @@ L_AES_GCM_decrypt_avx512_cmp_tag_16: L_AES_GCM_decrypt_avx512_cmp_tag_done: movl %ebx, (%rbp) vzeroupper - addq $0x420, %rsp + addq $0x460, %rsp popq %rbp popq %r15 popq %r14 @@ -27370,61 +30283,422 @@ AES_GCM_aad_update_avx512: .p2align 4 _AES_GCM_aad_update_avx512: #endif /* __APPLE__ */ - movq %rcx, %rax - vmovdqa (%rdx), %xmm5 - vmovdqa (%rax), %xmm6 - xorl %ecx, %ecx -L_AES_GCM_aad_update_avx512_16_loop: - vmovdqu (%rdi,%rcx,1), %xmm7 - vpshufb L_avx512_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7 - vpxor %xmm7, %xmm5, %xmm5 - # ghash_gfmul_avx - vpshufd $0x4e, %xmm5, %xmm1 - vpshufd $0x4e, %xmm6, %xmm2 - vpclmulqdq $0x11, %xmm5, %xmm6, %xmm3 - vpclmulqdq $0x00, %xmm5, %xmm6, %xmm0 - vpxor %xmm5, %xmm1, %xmm1 - vpxor %xmm6, %xmm2, %xmm2 - vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 - vpxor %xmm0, %xmm1, %xmm1 - vpxor %xmm3, %xmm1, %xmm1 - vmovdqa %xmm0, %xmm4 - vmovdqa %xmm3, %xmm5 - vpslldq $8, %xmm1, %xmm2 - vpsrldq $8, %xmm1, %xmm1 - vpxor %xmm2, %xmm4, %xmm4 - vpxor %xmm1, %xmm5, %xmm5 - vpsrld $31, %xmm4, %xmm0 - vpsrld $31, %xmm5, %xmm1 - vpslld $0x01, %xmm4, %xmm4 - vpslld $0x01, %xmm5, %xmm5 - vpsrldq $12, %xmm0, %xmm2 - vpslldq $4, %xmm0, %xmm0 - vpslldq $4, %xmm1, %xmm1 - vpor %xmm2, %xmm5, %xmm5 - vpor %xmm0, %xmm4, %xmm4 - vpor %xmm1, %xmm5, %xmm5 - vpslld $31, %xmm4, %xmm0 - vpslld $30, %xmm4, %xmm1 - vpslld $25, %xmm4, %xmm2 - vpxor %xmm1, %xmm0, %xmm0 - vpxor %xmm2, %xmm0, %xmm0 - vmovdqa %xmm0, %xmm1 - vpsrldq $4, %xmm1, %xmm1 - vpslldq $12, %xmm0, %xmm0 - vpxor %xmm0, %xmm4, %xmm4 - vpsrld $0x01, %xmm4, %xmm2 - vpsrld $2, %xmm4, %xmm3 - vpsrld $7, %xmm4, %xmm0 - vpxor %xmm3, %xmm2, %xmm2 - vpxor %xmm0, %xmm2, %xmm2 - vpxor %xmm1, %xmm2, %xmm2 - vpxor %xmm4, %xmm2, %xmm2 - vpxor %xmm2, %xmm5, %xmm5 - addl $16, %ecx - cmpl %esi, %ecx - jl L_AES_GCM_aad_update_avx512_16_loop - vmovdqa %xmm5, (%rdx) + movq %rdx, %r8 + movq %rcx, %r9 + subq $0x100, %rsp + vmovdqa (%r8), %xmm6 + vmovdqa (%r9), %xmm7 + xorl %edx, %edx + vpsrlq $63, %xmm7, %xmm10 + vpsllq $0x01, %xmm7, %xmm9 + vpslldq $8, %xmm10, %xmm10 + vpor %xmm10, %xmm9, %xmm9 + vpshufd $0xff, %xmm7, %xmm7 + vpsrad $31, %xmm7, %xmm7 + vpand L_avx512_aes_gcm_mod2_128(%rip), %xmm7, %xmm7 + vpxor %xmm9, %xmm7, %xmm7 + vmovdqa %xmm6, %xmm2 + # H ^ 1 + vmovdqu %xmm7, (%rsp) + # H ^ 2 + vpclmulqdq $0x00, %xmm7, %xmm7, %xmm9 + vpclmulqdq $0x11, %xmm7, %xmm7, %xmm12 + vpxor %xmm10, %xmm10, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm0 + vmovdqu %xmm0, 16(%rsp) + # H ^ 3 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm7, %xmm10 + vpxor %xmm7, %xmm10, %xmm10 + vpshufd $0x4e, %xmm0, %xmm11 + vpxor %xmm0, %xmm11, %xmm11 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm12 + vpclmulqdq $0x00, %xmm11, %xmm10, %xmm10 + vpternlogq $0x96, %xmm9, %xmm12, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm1 + vmovdqu %xmm1, 32(%rsp) + # H ^ 4 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm12 + vpxor %xmm10, %xmm10, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm3 + vmovdqu %xmm3, 48(%rsp) + # H ^ 5 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm10 + vpxor %xmm0, %xmm10, %xmm10 + vpshufd $0x4e, %xmm1, %xmm11 + vpxor %xmm1, %xmm11, %xmm11 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm11, %xmm10, %xmm10 + vpternlogq $0x96, %xmm9, %xmm12, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 64(%rsp) + # H ^ 6 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm12 + vpxor %xmm10, %xmm10, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 80(%rsp) + # H ^ 7 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm1, %xmm10 + vpxor %xmm1, %xmm10, %xmm10 + vpshufd $0x4e, %xmm3, %xmm11 + vpxor %xmm3, %xmm11, %xmm11 + vpclmulqdq $0x00, %xmm1, %xmm3, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm3, %xmm12 + vpclmulqdq $0x00, %xmm11, %xmm10, %xmm10 + vpternlogq $0x96, %xmm9, %xmm12, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 96(%rsp) + # H ^ 8 + vpclmulqdq $0x00, %xmm3, %xmm3, %xmm9 + vpclmulqdq $0x11, %xmm3, %xmm3, %xmm12 + vpxor %xmm10, %xmm10, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 112(%rsp) + # H ^ 9 + vmovdqu 48(%rsp), %xmm0 + vmovdqu 64(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm10 + vpxor %xmm0, %xmm10, %xmm10 + vpshufd $0x4e, %xmm1, %xmm11 + vpxor %xmm1, %xmm11, %xmm11 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm11, %xmm10, %xmm10 + vpternlogq $0x96, %xmm9, %xmm12, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 128(%rsp) + # H ^ 10 + vmovdqu 64(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm12 + vpxor %xmm10, %xmm10, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 144(%rsp) + # H ^ 11 + vmovdqu 64(%rsp), %xmm0 + vmovdqu 80(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm10 + vpxor %xmm0, %xmm10, %xmm10 + vpshufd $0x4e, %xmm1, %xmm11 + vpxor %xmm1, %xmm11, %xmm11 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm11, %xmm10, %xmm10 + vpternlogq $0x96, %xmm9, %xmm12, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 160(%rsp) + # H ^ 12 + vmovdqu 80(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm12 + vpxor %xmm10, %xmm10, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 176(%rsp) + # H ^ 13 + vmovdqu 80(%rsp), %xmm0 + vmovdqu 96(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm10 + vpxor %xmm0, %xmm10, %xmm10 + vpshufd $0x4e, %xmm1, %xmm11 + vpxor %xmm1, %xmm11, %xmm11 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm11, %xmm10, %xmm10 + vpternlogq $0x96, %xmm9, %xmm12, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 192(%rsp) + # H ^ 14 + vmovdqu 96(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm12 + vpxor %xmm10, %xmm10, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 208(%rsp) + # H ^ 15 + vmovdqu 96(%rsp), %xmm0 + vmovdqu 112(%rsp), %xmm1 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm10 + vpxor %xmm0, %xmm10, %xmm10 + vpshufd $0x4e, %xmm1, %xmm11 + vpxor %xmm1, %xmm11, %xmm11 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm11, %xmm10, %xmm10 + vpternlogq $0x96, %xmm9, %xmm12, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 224(%rsp) + # H ^ 16 + vmovdqu 112(%rsp), %xmm0 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm12 + vpxor %xmm10, %xmm10, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm8 + vmovdqu %xmm8, 240(%rsp) + vbroadcasti32x4 L_avx512_aes_gcm_bswap_mask(%rip), %zmm22 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm23 + vmovdqu64 192(%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vmovdqu64 128(%rsp), %zmm25 + vshufi64x2 $27, %zmm25, %zmm25, %zmm25 + vmovdqu64 64(%rsp), %zmm26 + vshufi64x2 $27, %zmm26, %zmm26, %zmm26 + vmovdqu64 (%rsp), %zmm27 + vshufi64x2 $27, %zmm27, %zmm27, %zmm27 + movl %esi, %eax + andl $0xffffff00, %eax + cmpl %eax, %edx + jge L_AES_GCM_aad_update_avx512_after_256 +L_AES_GCM_aad_update_avx512_loop_256: + # 256 bytes of input + leaq (%rdi,%rdx,1), %rcx + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rcx), %zmm21 + vpshufb %zmm22, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm28 + vpxorq %zmm17, %zmm18, %zmm29 + vmovdqa64 %zmm19, %zmm30 + vmovdqu64 64(%rcx), %zmm21 + vpshufb %zmm22, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm25, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm25, %zmm21, %zmm17 + vpclmulqdq $16, %zmm25, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm25, %zmm21, %zmm19 + vpxorq %zmm16, %zmm28, %zmm28 + vpternlogq $0x96, %zmm17, %zmm18, %zmm29 + vpxorq %zmm19, %zmm30, %zmm30 + vmovdqu64 128(%rcx), %zmm21 + vpshufb %zmm22, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm26, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm26, %zmm21, %zmm17 + vpclmulqdq $16, %zmm26, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm26, %zmm21, %zmm19 + vpxorq %zmm16, %zmm28, %zmm28 + vpternlogq $0x96, %zmm17, %zmm18, %zmm29 + vpxorq %zmm19, %zmm30, %zmm30 + vmovdqu64 192(%rcx), %zmm21 + vpshufb %zmm22, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm27, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm27, %zmm21, %zmm17 + vpclmulqdq $16, %zmm27, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm27, %zmm21, %zmm19 + vpxorq %zmm16, %zmm28, %zmm28 + vpternlogq $0x96, %zmm17, %zmm18, %zmm29 + vpxorq %zmm19, %zmm30, %zmm30 + vpclmulqdq $0x01, %zmm28, %zmm23, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vpclmulqdq $0x01, %zmm29, %zmm23, %zmm21 + vpshufd $0x4e, %zmm29, %zmm29 + vpternlogq $0x96, %zmm21, %zmm29, %zmm30 + vextracti32x4 $0x01, %zmm30, %xmm0 + vextracti32x4 $2, %zmm30, %xmm4 + vextracti32x4 $3, %zmm30, %xmm5 + vpxorq %xmm0, %xmm30, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + addl $0x100, %edx + cmpl %eax, %edx + jl L_AES_GCM_aad_update_avx512_loop_256 +L_AES_GCM_aad_update_avx512_after_256: + movl %esi, %eax + andl $0xffffff80, %eax + cmpl %eax, %edx + jge L_AES_GCM_aad_update_avx512_after_128 + # 128 bytes of input + leaq (%rdi,%rdx,1), %rcx + vmovdqu64 64(%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vmovdqu64 (%rsp), %zmm25 + vshufi64x2 $27, %zmm25, %zmm25, %zmm25 + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rcx), %zmm21 + vpshufb %zmm22, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm28 + vpxorq %zmm17, %zmm18, %zmm29 + vmovdqa64 %zmm19, %zmm30 + vmovdqu64 64(%rcx), %zmm21 + vpshufb %zmm22, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm25, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm25, %zmm21, %zmm17 + vpclmulqdq $16, %zmm25, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm25, %zmm21, %zmm19 + vpxorq %zmm16, %zmm28, %zmm28 + vpternlogq $0x96, %zmm17, %zmm18, %zmm29 + vpxorq %zmm19, %zmm30, %zmm30 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm23 + vpclmulqdq $0x01, %zmm28, %zmm23, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vpclmulqdq $0x01, %zmm29, %zmm23, %zmm21 + vpshufd $0x4e, %zmm29, %zmm29 + vpternlogq $0x96, %zmm21, %zmm29, %zmm30 + vextracti32x4 $0x01, %zmm30, %xmm0 + vextracti32x4 $2, %zmm30, %xmm4 + vextracti32x4 $3, %zmm30, %xmm5 + vpxorq %xmm0, %xmm30, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + addl $0x80, %edx +L_AES_GCM_aad_update_avx512_after_128: + movl %esi, %eax + andl $0xfffffff0, %eax + cmpl %eax, %edx + jge L_AES_GCM_aad_update_avx512_done + vmovdqu (%rsp), %xmm7 +L_AES_GCM_aad_update_avx512_tail: + # 16 bytes of input + vmovdqu (%rdi,%rdx,1), %xmm9 + vpshufb L_avx512_aes_gcm_bswap_mask(%rip), %xmm9, %xmm9 + vpxor %xmm9, %xmm6, %xmm6 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm7, %xmm10 + vpxor %xmm7, %xmm10, %xmm10 + vpshufd $0x4e, %xmm6, %xmm11 + vpxor %xmm6, %xmm11, %xmm11 + vpclmulqdq $0x00, %xmm7, %xmm6, %xmm9 + vpclmulqdq $0x11, %xmm7, %xmm6, %xmm12 + vpclmulqdq $0x00, %xmm11, %xmm10, %xmm10 + vpternlogq $0x96, %xmm9, %xmm12, %xmm10 + vmovdqa L_avx512_aes_gcm_mod2_128(%rip), %xmm11 + vpclmulqdq $0x01, %xmm9, %xmm11, %xmm13 + vpshufd $0x4e, %xmm9, %xmm9 + vpternlogq $0x96, %xmm13, %xmm9, %xmm10 + vpclmulqdq $0x01, %xmm10, %xmm11, %xmm13 + vpshufd $0x4e, %xmm10, %xmm10 + vpternlogq $0x96, %xmm13, %xmm10, %xmm12 + vmovdqa %xmm12, %xmm6 + addl $16, %edx + cmpl %eax, %edx + jl L_AES_GCM_aad_update_avx512_tail +L_AES_GCM_aad_update_avx512_done: + vmovdqa %xmm6, (%r8) + vzeroupper + addq $0x100, %rsp repz retq #ifndef __APPLE__ .size AES_GCM_aad_update_avx512,.-AES_GCM_aad_update_avx512 @@ -27579,7 +30853,7 @@ _AES_GCM_encrypt_update_avx512: vpand L_avx512_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 vpxor %xmm8, %xmm5, %xmm5 xorl %r14d, %r14d - cmpl $0x100, %r8d + cmpl $0x80, %r8d jl L_AES_GCM_encrypt_update_avx512_done_128 vmovdqa %xmm6, %xmm2 # H ^ 1 @@ -27693,6 +30967,8 @@ _AES_GCM_encrypt_update_avx512: vpternlogq $0x96, %xmm12, %xmm9, %xmm11 vmovdqa %xmm11, %xmm7 vmovdqu %xmm7, 112(%rsp) + cmpl $0x100, %r8d + jl L_AES_GCM_encrypt_update_avx512_no_ext2 # H ^ 9 vmovdqu 48(%rsp), %xmm0 vmovdqu 64(%rsp), %xmm1 @@ -28116,6 +31392,7 @@ _AES_GCM_encrypt_update_avx512: vmovdqa %xmm11, %xmm7 vmovdqu %xmm7, 496(%rsp) L_AES_GCM_encrypt_update_avx512_no_ext: +L_AES_GCM_encrypt_update_avx512_no_ext2: vbroadcasti32x4 L_avx512_aes_gcm_bswap_epi64(%rip), %zmm22 vbroadcasti32x4 L_avx512_aes_gcm_bswap_mask(%rip), %zmm30 vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 @@ -29053,6 +32330,117 @@ L_AES_GCM_encrypt_update_avx512_last_ghash: vpxorq %xmm0, %xmm29, %xmm6 vpternlogq $0x96, %xmm4, %xmm5, %xmm6 L_AES_GCM_encrypt_update_avx512_after_256: + movl %r8d, %r13d + andl $0xffffff80, %r13d + cmpl %r13d, %r14d + jge L_AES_GCM_encrypt_update_avx512_after_128 + # 128 bytes of input + vbroadcasti32x4 (%rdi), %zmm9 + vbroadcasti32x4 16(%rdi), %zmm10 + vbroadcasti32x4 32(%rdi), %zmm11 + vbroadcasti32x4 48(%rdi), %zmm12 + vbroadcasti32x4 64(%rdi), %zmm13 + vbroadcasti32x4 80(%rdi), %zmm14 + vbroadcasti32x4 96(%rdi), %zmm15 + vbroadcasti32x4 112(%rdi), %zmm1 + vbroadcasti32x4 128(%rdi), %zmm2 + vbroadcasti32x4 144(%rdi), %zmm3 + leaq (%r10,%r14,1), %r15 + vbroadcasti32x4 (%r12), %zmm20 + vpaddd L_avx512_aes_gcm_inc_z0(%rip), %zmm20, %zmm16 + vpshufb %zmm22, %zmm16, %zmm16 + vpaddd L_avx512_aes_gcm_inc_z1(%rip), %zmm20, %zmm17 + vpshufb %zmm22, %zmm17, %zmm17 + vmovdqu (%r12), %xmm8 + vpaddd L_avx512_aes_gcm_eight(%rip), %xmm8, %xmm8 + vmovdqu %xmm8, (%r12) + vpxorq %zmm9, %zmm16, %zmm16 + vpxorq %zmm9, %zmm17, %zmm17 + vaesenc %zmm10, %zmm16, %zmm16 + vaesenc %zmm10, %zmm17, %zmm17 + vaesenc %zmm11, %zmm16, %zmm16 + vaesenc %zmm11, %zmm17, %zmm17 + vaesenc %zmm12, %zmm16, %zmm16 + vaesenc %zmm12, %zmm17, %zmm17 + vaesenc %zmm13, %zmm16, %zmm16 + vaesenc %zmm13, %zmm17, %zmm17 + vaesenc %zmm14, %zmm16, %zmm16 + vaesenc %zmm14, %zmm17, %zmm17 + vaesenc %zmm15, %zmm16, %zmm16 + vaesenc %zmm15, %zmm17, %zmm17 + vaesenc %zmm1, %zmm16, %zmm16 + vaesenc %zmm1, %zmm17, %zmm17 + vaesenc %zmm2, %zmm16, %zmm16 + vaesenc %zmm2, %zmm17, %zmm17 + vaesenc %zmm3, %zmm16, %zmm16 + vaesenc %zmm3, %zmm17, %zmm17 + cmpl $11, %esi + vbroadcasti32x4 160(%rdi), %zmm20 + jl L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 176(%rdi), %zmm20 + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + cmpl $13, %esi + vbroadcasti32x4 192(%rdi), %zmm20 + jl L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 208(%rdi), %zmm20 + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 224(%rdi), %zmm20 +L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last: + vaesenclast %zmm20, %zmm16, %zmm16 + vaesenclast %zmm20, %zmm17, %zmm17 + leaq (%r11,%r14,1), %rcx + leaq (%r10,%r14,1), %rdx + vmovdqu64 (%rcx), %zmm21 + vpxorq %zmm21, %zmm16, %zmm16 + vmovdqu64 %zmm16, (%rdx) + vmovdqu64 64(%rcx), %zmm21 + vpxorq %zmm21, %zmm17, %zmm17 + vmovdqu64 %zmm17, 64(%rdx) + addl $0x80, %r14d + vmovdqu64 64(%rsp), %zmm23 + vshufi64x2 $27, %zmm23, %zmm23, %zmm23 + vmovdqu64 (%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%r15), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm23, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm23, %zmm21, %zmm17 + vpclmulqdq $16, %zmm23, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm23, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm27 + vpxorq %zmm17, %zmm18, %zmm28 + vmovdqa64 %zmm19, %zmm29 + vmovdqu64 64(%r15), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 + vpclmulqdq $0x01, %zmm27, %zmm31, %zmm21 + vpshufd $0x4e, %zmm27, %zmm27 + vpternlogq $0x96, %zmm21, %zmm27, %zmm28 + vpclmulqdq $0x01, %zmm28, %zmm31, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vextracti32x4 $0x01, %zmm29, %xmm0 + vextracti32x4 $2, %zmm29, %xmm4 + vextracti32x4 $3, %zmm29, %xmm5 + vpxorq %xmm0, %xmm29, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 +L_AES_GCM_encrypt_update_avx512_after_128: vmovdqu (%rsp), %xmm5 L_AES_GCM_encrypt_update_avx512_done_128: movl %r8d, %edx @@ -29294,7 +32682,7 @@ _AES_GCM_decrypt_update_avx512: vpand L_avx512_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 vpxor %xmm8, %xmm5, %xmm5 xorl %r14d, %r14d - cmpl $0x100, %r8d + cmpl $0x80, %r8d jl L_AES_GCM_decrypt_update_avx512_done_128 vmovdqa %xmm6, %xmm2 # H ^ 1 @@ -29408,6 +32796,8 @@ _AES_GCM_decrypt_update_avx512: vpternlogq $0x96, %xmm12, %xmm9, %xmm11 vmovdqa %xmm11, %xmm7 vmovdqu %xmm7, 112(%rsp) + cmpl $0x100, %r8d + jl L_AES_GCM_decrypt_update_avx512_no_ext2 # H ^ 9 vmovdqu 48(%rsp), %xmm0 vmovdqu 64(%rsp), %xmm1 @@ -29831,6 +33221,7 @@ _AES_GCM_decrypt_update_avx512: vmovdqa %xmm11, %xmm7 vmovdqu %xmm7, 496(%rsp) L_AES_GCM_decrypt_update_avx512_no_ext: +L_AES_GCM_decrypt_update_avx512_no_ext2: vbroadcasti32x4 L_avx512_aes_gcm_bswap_epi64(%rip), %zmm22 vbroadcasti32x4 L_avx512_aes_gcm_bswap_mask(%rip), %zmm30 vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 @@ -30616,6 +34007,117 @@ L_AES_GCM_decrypt_update_avx512_t_avx512_ctr16_last: vmovdqu64 %zmm19, 192(%rdx) addl $0x100, %r14d L_AES_GCM_decrypt_update_avx512_after_256: + movl %r8d, %r13d + andl $0xffffff80, %r13d + cmpl %r13d, %r14d + jge L_AES_GCM_decrypt_update_avx512_after_128 + # 128 bytes of input + leaq (%r11,%r14,1), %rbx + vmovdqu64 64(%rsp), %zmm23 + vshufi64x2 $27, %zmm23, %zmm23, %zmm23 + vmovdqu64 (%rsp), %zmm24 + vshufi64x2 $27, %zmm24, %zmm24, %zmm24 + vpxorq %zmm20, %zmm20, %zmm20 + vinserti32x4 $0x00, %xmm6, %zmm20, %zmm20 + vmovdqu64 (%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpxorq %zmm20, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm23, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm23, %zmm21, %zmm17 + vpclmulqdq $16, %zmm23, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm23, %zmm21, %zmm19 + vmovdqa64 %zmm16, %zmm27 + vpxorq %zmm17, %zmm18, %zmm28 + vmovdqa64 %zmm19, %zmm29 + vmovdqu64 64(%rbx), %zmm21 + vpshufb %zmm30, %zmm21, %zmm21 + vpclmulqdq $0x00, %zmm24, %zmm21, %zmm16 + vpclmulqdq $0x01, %zmm24, %zmm21, %zmm17 + vpclmulqdq $16, %zmm24, %zmm21, %zmm18 + vpclmulqdq $0x11, %zmm24, %zmm21, %zmm19 + vpxorq %zmm16, %zmm27, %zmm27 + vpternlogq $0x96, %zmm17, %zmm18, %zmm28 + vpxorq %zmm19, %zmm29, %zmm29 + vbroadcasti32x4 L_avx512_aes_gcm_mod2_128(%rip), %zmm31 + vpclmulqdq $0x01, %zmm27, %zmm31, %zmm21 + vpshufd $0x4e, %zmm27, %zmm27 + vpternlogq $0x96, %zmm21, %zmm27, %zmm28 + vpclmulqdq $0x01, %zmm28, %zmm31, %zmm21 + vpshufd $0x4e, %zmm28, %zmm28 + vpternlogq $0x96, %zmm21, %zmm28, %zmm29 + vextracti32x4 $0x01, %zmm29, %xmm0 + vextracti32x4 $2, %zmm29, %xmm4 + vextracti32x4 $3, %zmm29, %xmm5 + vpxorq %xmm0, %xmm29, %xmm6 + vpternlogq $0x96, %xmm4, %xmm5, %xmm6 + vbroadcasti32x4 (%rdi), %zmm9 + vbroadcasti32x4 16(%rdi), %zmm10 + vbroadcasti32x4 32(%rdi), %zmm11 + vbroadcasti32x4 48(%rdi), %zmm12 + vbroadcasti32x4 64(%rdi), %zmm13 + vbroadcasti32x4 80(%rdi), %zmm14 + vbroadcasti32x4 96(%rdi), %zmm15 + vbroadcasti32x4 112(%rdi), %zmm1 + vbroadcasti32x4 128(%rdi), %zmm2 + vbroadcasti32x4 144(%rdi), %zmm3 + vbroadcasti32x4 (%r12), %zmm20 + vpaddd L_avx512_aes_gcm_inc_z0(%rip), %zmm20, %zmm16 + vpshufb %zmm22, %zmm16, %zmm16 + vpaddd L_avx512_aes_gcm_inc_z1(%rip), %zmm20, %zmm17 + vpshufb %zmm22, %zmm17, %zmm17 + vmovdqu (%r12), %xmm8 + vpaddd L_avx512_aes_gcm_eight(%rip), %xmm8, %xmm8 + vmovdqu %xmm8, (%r12) + vpxorq %zmm9, %zmm16, %zmm16 + vpxorq %zmm9, %zmm17, %zmm17 + vaesenc %zmm10, %zmm16, %zmm16 + vaesenc %zmm10, %zmm17, %zmm17 + vaesenc %zmm11, %zmm16, %zmm16 + vaesenc %zmm11, %zmm17, %zmm17 + vaesenc %zmm12, %zmm16, %zmm16 + vaesenc %zmm12, %zmm17, %zmm17 + vaesenc %zmm13, %zmm16, %zmm16 + vaesenc %zmm13, %zmm17, %zmm17 + vaesenc %zmm14, %zmm16, %zmm16 + vaesenc %zmm14, %zmm17, %zmm17 + vaesenc %zmm15, %zmm16, %zmm16 + vaesenc %zmm15, %zmm17, %zmm17 + vaesenc %zmm1, %zmm16, %zmm16 + vaesenc %zmm1, %zmm17, %zmm17 + vaesenc %zmm2, %zmm16, %zmm16 + vaesenc %zmm2, %zmm17, %zmm17 + vaesenc %zmm3, %zmm16, %zmm16 + vaesenc %zmm3, %zmm17, %zmm17 + cmpl $11, %esi + vbroadcasti32x4 160(%rdi), %zmm20 + jl L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 176(%rdi), %zmm20 + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + cmpl $13, %esi + vbroadcasti32x4 192(%rdi), %zmm20 + jl L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 208(%rdi), %zmm20 + vaesenc %zmm20, %zmm16, %zmm16 + vaesenc %zmm20, %zmm17, %zmm17 + vbroadcasti32x4 224(%rdi), %zmm20 +L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last: + vaesenclast %zmm20, %zmm16, %zmm16 + vaesenclast %zmm20, %zmm17, %zmm17 + leaq (%r11,%r14,1), %rcx + leaq (%r10,%r14,1), %rdx + vmovdqu64 (%rcx), %zmm21 + vpxorq %zmm21, %zmm16, %zmm16 + vmovdqu64 %zmm16, (%rdx) + vmovdqu64 64(%rcx), %zmm21 + vpxorq %zmm21, %zmm17, %zmm17 + vmovdqu64 %zmm17, 64(%rdx) + addl $0x80, %r14d +L_AES_GCM_decrypt_update_avx512_after_128: vmovdqu (%rsp), %xmm5 L_AES_GCM_decrypt_update_avx512_done_128: movl %r8d, %edx diff --git a/wolfcrypt/src/aes_gcm_asm.asm b/wolfcrypt/src/aes_gcm_asm.asm index 34f6847631..ad352e11cb 100644 --- a/wolfcrypt/src/aes_gcm_asm.asm +++ b/wolfcrypt/src/aes_gcm_asm.asm @@ -12244,17 +12244,17 @@ AES_GCM_encrypt_avx2 PROC mov r14d, DWORD PTR [rsp+128] mov rsi, QWORD PTR [rsp+136] mov r9d, DWORD PTR [rsp+144] - sub rsp, 320 - vmovdqu OWORD PTR [rsp+160], xmm6 - vmovdqu OWORD PTR [rsp+176], xmm7 - vmovdqu OWORD PTR [rsp+192], xmm8 - vmovdqu OWORD PTR [rsp+208], xmm9 - vmovdqu OWORD PTR [rsp+224], xmm10 - vmovdqu OWORD PTR [rsp+240], xmm11 - vmovdqu OWORD PTR [rsp+256], xmm12 - vmovdqu OWORD PTR [rsp+272], xmm13 - vmovdqu OWORD PTR [rsp+288], xmm14 - vmovdqu OWORD PTR [rsp+304], xmm15 + sub rsp, 344 + vmovdqu OWORD PTR [rsp+184], xmm6 + vmovdqu OWORD PTR [rsp+200], xmm7 + vmovdqu OWORD PTR [rsp+216], xmm8 + vmovdqu OWORD PTR [rsp+232], xmm9 + vmovdqu OWORD PTR [rsp+248], xmm10 + vmovdqu OWORD PTR [rsp+264], xmm11 + vmovdqu OWORD PTR [rsp+280], xmm12 + vmovdqu OWORD PTR [rsp+296], xmm13 + vmovdqu OWORD PTR [rsp+312], xmm14 + vmovdqu OWORD PTR [rsp+328], xmm15 vpxor xmm4, xmm4, xmm4 vpxor xmm6, xmm6, xmm6 mov edx, ebx @@ -12503,9 +12503,183 @@ L_AES_GCM_encrypt_avx2_iv_done: cmp edx, 0 je L_AES_GCM_encrypt_avx2_calc_aad_done xor ecx, ecx - cmp edx, 16 - jl L_AES_GCM_encrypt_avx2_calc_aad_lt16 + cmp edx, 128 + jl L_AES_GCM_encrypt_avx2_calc_aad_after_agg + mov r13d, edx + and r13d, 4294967232 + vmovdqu OWORD PTR [rsp+168], xmm5 + vpsrlq xmm1, xmm5, 63 + vpsllq xmm0, xmm5, 1 + vpslldq xmm1, xmm1, 8 + vpor xmm0, xmm0, xmm1 + vpshufd xmm5, xmm5, 255 + vpsrad xmm5, xmm5, 31 + vpand xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_mod2_128 + vpxor xmm5, xmm5, xmm0 + vmovdqu xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128 + ; H ^ 1 and H ^ 2 + vpclmulqdq xmm9, xmm5, xmm5, 0 + vpclmulqdq xmm10, xmm5, xmm5, 17 + vpclmulqdq xmm8, xmm9, xmm3, 16 + vpshufd xmm9, xmm9, 78 + vpxor xmm9, xmm9, xmm8 + vpclmulqdq xmm8, xmm9, xmm3, 16 + vpshufd xmm9, xmm9, 78 + vpxor xmm9, xmm9, xmm8 + vpxor xmm0, xmm10, xmm9 + vmovdqu OWORD PTR [rsp], xmm5 + vmovdqu OWORD PTR [rsp+16], xmm0 + ; H ^ 3 and H ^ 4 + vpclmulqdq xmm11, xmm0, xmm5, 16 + vpclmulqdq xmm10, xmm0, xmm5, 1 + vpclmulqdq xmm9, xmm0, xmm5, 0 + vpclmulqdq xmm12, xmm0, xmm5, 17 + vpclmulqdq xmm13, xmm0, xmm0, 0 + vpclmulqdq xmm14, xmm0, xmm0, 17 + vpxor xmm11, xmm11, xmm10 + vpslldq xmm10, xmm11, 8 + vpsrldq xmm11, xmm11, 8 + vpxor xmm10, xmm10, xmm9 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm10, xmm10, xmm9 + vpxor xmm13, xmm13, xmm8 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm12, xmm12, xmm11 + vpxor xmm13, xmm13, xmm8 + vpxor xmm10, xmm10, xmm12 + vpxor xmm2, xmm13, xmm14 + vpxor xmm1, xmm10, xmm9 + vmovdqu OWORD PTR [rsp+32], xmm1 + vmovdqu OWORD PTR [rsp+48], xmm2 + ; H ^ 5 and H ^ 6 + vpclmulqdq xmm11, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 0 + vpclmulqdq xmm12, xmm1, xmm0, 17 + vpclmulqdq xmm13, xmm1, xmm1, 0 + vpclmulqdq xmm14, xmm1, xmm1, 17 + vpxor xmm11, xmm11, xmm10 + vpslldq xmm10, xmm11, 8 + vpsrldq xmm11, xmm11, 8 + vpxor xmm10, xmm10, xmm9 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm10, xmm10, xmm9 + vpxor xmm13, xmm13, xmm8 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm12, xmm12, xmm11 + vpxor xmm13, xmm13, xmm8 + vpxor xmm10, xmm10, xmm12 + vpxor xmm0, xmm13, xmm14 + vpxor xmm7, xmm10, xmm9 + vmovdqu OWORD PTR [rsp+64], xmm7 + vmovdqu OWORD PTR [rsp+80], xmm0 + ; H ^ 7 and H ^ 8 + vpclmulqdq xmm11, xmm2, xmm1, 16 + vpclmulqdq xmm10, xmm2, xmm1, 1 + vpclmulqdq xmm9, xmm2, xmm1, 0 + vpclmulqdq xmm12, xmm2, xmm1, 17 + vpclmulqdq xmm13, xmm2, xmm2, 0 + vpclmulqdq xmm14, xmm2, xmm2, 17 + vpxor xmm11, xmm11, xmm10 + vpslldq xmm10, xmm11, 8 + vpsrldq xmm11, xmm11, 8 + vpxor xmm10, xmm10, xmm9 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm10, xmm10, xmm9 + vpxor xmm13, xmm13, xmm8 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm12, xmm12, xmm11 + vpxor xmm13, xmm13, xmm8 + vpxor xmm10, xmm10, xmm12 + vpxor xmm0, xmm13, xmm14 + vpxor xmm7, xmm10, xmm9 + vmovdqu OWORD PTR [rsp+96], xmm7 + vmovdqu OWORD PTR [rsp+112], xmm0 +L_AES_GCM_encrypt_avx2_calc_aad_agg: + ; 64 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu xmm8, OWORD PTR [rbx] + vpshufb xmm8, xmm8, OWORD PTR L_avx2_aes_gcm_bswap_mask + vmovdqu xmm9, OWORD PTR [rbx+16] + vpshufb xmm9, xmm9, OWORD PTR L_avx2_aes_gcm_bswap_mask + vmovdqu xmm10, OWORD PTR [rbx+32] + vpshufb xmm10, xmm10, OWORD PTR L_avx2_aes_gcm_bswap_mask + vmovdqu xmm11, OWORD PTR [rbx+48] + vpshufb xmm11, xmm11, OWORD PTR L_avx2_aes_gcm_bswap_mask + vpxor xmm8, xmm8, xmm6 + vmovdqu xmm7, OWORD PTR [rsp+48] + vpclmulqdq xmm13, xmm7, xmm8, 16 + vpclmulqdq xmm1, xmm7, xmm8, 1 + vpclmulqdq xmm12, xmm7, xmm8, 0 + vpclmulqdq xmm6, xmm7, xmm8, 17 + vpxor xmm13, xmm13, xmm1 + vmovdqu xmm7, OWORD PTR [rsp+32] + vpclmulqdq xmm2, xmm7, xmm9, 16 + vpclmulqdq xmm1, xmm7, xmm9, 1 + vpclmulqdq xmm0, xmm7, xmm9, 0 + vpclmulqdq xmm3, xmm7, xmm9, 17 + vpxor xmm2, xmm2, xmm1 + vpxor xmm6, xmm6, xmm3 + vpxor xmm13, xmm13, xmm2 + vpxor xmm12, xmm12, xmm0 + vmovdqu xmm7, OWORD PTR [rsp+16] + vpclmulqdq xmm2, xmm7, xmm10, 16 + vpclmulqdq xmm1, xmm7, xmm10, 1 + vpclmulqdq xmm0, xmm7, xmm10, 0 + vpclmulqdq xmm3, xmm7, xmm10, 17 + vpxor xmm2, xmm2, xmm1 + vpxor xmm6, xmm6, xmm3 + vpxor xmm13, xmm13, xmm2 + vpxor xmm12, xmm12, xmm0 + vmovdqu xmm7, OWORD PTR [rsp] + vpclmulqdq xmm2, xmm7, xmm11, 16 + vpclmulqdq xmm1, xmm7, xmm11, 1 + vpclmulqdq xmm0, xmm7, xmm11, 0 + vpclmulqdq xmm3, xmm7, xmm11, 17 + vpxor xmm2, xmm2, xmm1 + vpxor xmm6, xmm6, xmm3 + vpxor xmm13, xmm13, xmm2 + vpxor xmm12, xmm12, xmm0 + vpslldq xmm7, xmm13, 8 + vpsrldq xmm13, xmm13, 8 + vpxor xmm12, xmm12, xmm7 + vpxor xmm6, xmm6, xmm13 + ; ghash_red + vmovdqu xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128 + vpclmulqdq xmm0, xmm12, xmm2, 16 + vpshufd xmm1, xmm12, 78 + vpxor xmm1, xmm1, xmm0 + vpclmulqdq xmm0, xmm1, xmm2, 16 + vpshufd xmm1, xmm1, 78 + vpxor xmm1, xmm1, xmm0 + vpxor xmm6, xmm6, xmm1 + add ecx, 64 + cmp ecx, r13d + jl L_AES_GCM_encrypt_avx2_calc_aad_agg + vmovdqu xmm5, OWORD PTR [rsp+168] +L_AES_GCM_encrypt_avx2_calc_aad_after_agg: + mov edx, r11d and edx, 4294967280 + cmp ecx, edx + je L_AES_GCM_encrypt_avx2_calc_aad_partial L_AES_GCM_encrypt_avx2_calc_aad_16_loop: vmovdqu xmm0, OWORD PTR [r12+rcx] vpshufb xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask @@ -12543,6 +12717,7 @@ L_AES_GCM_encrypt_avx2_calc_aad_16_loop: add ecx, 16 cmp ecx, edx jl L_AES_GCM_encrypt_avx2_calc_aad_16_loop +L_AES_GCM_encrypt_avx2_calc_aad_partial: mov edx, r11d cmp ecx, edx je L_AES_GCM_encrypt_avx2_calc_aad_done @@ -13525,17 +13700,17 @@ L_AES_GCM_encrypt_avx2_store_tag_16: vmovdqu OWORD PTR [r15], xmm0 L_AES_GCM_encrypt_avx2_store_tag_done: vzeroupper - vmovdqu xmm6, OWORD PTR [rsp+160] - vmovdqu xmm7, OWORD PTR [rsp+176] - vmovdqu xmm8, OWORD PTR [rsp+192] - vmovdqu xmm9, OWORD PTR [rsp+208] - vmovdqu xmm10, OWORD PTR [rsp+224] - vmovdqu xmm11, OWORD PTR [rsp+240] - vmovdqu xmm12, OWORD PTR [rsp+256] - vmovdqu xmm13, OWORD PTR [rsp+272] - vmovdqu xmm14, OWORD PTR [rsp+288] - vmovdqu xmm15, OWORD PTR [rsp+304] - add rsp, 320 + vmovdqu xmm6, OWORD PTR [rsp+184] + vmovdqu xmm7, OWORD PTR [rsp+200] + vmovdqu xmm8, OWORD PTR [rsp+216] + vmovdqu xmm9, OWORD PTR [rsp+232] + vmovdqu xmm10, OWORD PTR [rsp+248] + vmovdqu xmm11, OWORD PTR [rsp+264] + vmovdqu xmm12, OWORD PTR [rsp+280] + vmovdqu xmm13, OWORD PTR [rsp+296] + vmovdqu xmm14, OWORD PTR [rsp+312] + vmovdqu xmm15, OWORD PTR [rsp+328] + add rsp, 344 pop rsi pop r14 pop rbx @@ -13568,17 +13743,17 @@ AES_GCM_decrypt_avx2 PROC mov rsi, QWORD PTR [rsp+144] mov r9d, DWORD PTR [rsp+152] mov rbp, QWORD PTR [rsp+160] - sub rsp, 328 - vmovdqu OWORD PTR [rsp+168], xmm6 - vmovdqu OWORD PTR [rsp+184], xmm7 - vmovdqu OWORD PTR [rsp+200], xmm8 - vmovdqu OWORD PTR [rsp+216], xmm9 - vmovdqu OWORD PTR [rsp+232], xmm10 - vmovdqu OWORD PTR [rsp+248], xmm11 - vmovdqu OWORD PTR [rsp+264], xmm12 - vmovdqu OWORD PTR [rsp+280], xmm13 - vmovdqu OWORD PTR [rsp+296], xmm14 - vmovdqu OWORD PTR [rsp+312], xmm15 + sub rsp, 344 + vmovdqu OWORD PTR [rsp+184], xmm6 + vmovdqu OWORD PTR [rsp+200], xmm7 + vmovdqu OWORD PTR [rsp+216], xmm8 + vmovdqu OWORD PTR [rsp+232], xmm9 + vmovdqu OWORD PTR [rsp+248], xmm10 + vmovdqu OWORD PTR [rsp+264], xmm11 + vmovdqu OWORD PTR [rsp+280], xmm12 + vmovdqu OWORD PTR [rsp+296], xmm13 + vmovdqu OWORD PTR [rsp+312], xmm14 + vmovdqu OWORD PTR [rsp+328], xmm15 vpxor xmm4, xmm4, xmm4 vpxor xmm6, xmm6, xmm6 mov edx, ebx @@ -13827,9 +14002,183 @@ L_AES_GCM_decrypt_avx2_iv_done: cmp edx, 0 je L_AES_GCM_decrypt_avx2_calc_aad_done xor ecx, ecx - cmp edx, 16 - jl L_AES_GCM_decrypt_avx2_calc_aad_lt16 + cmp edx, 128 + jl L_AES_GCM_decrypt_avx2_calc_aad_after_agg + mov r13d, edx + and r13d, 4294967232 + vmovdqu OWORD PTR [rsp+168], xmm5 + vpsrlq xmm1, xmm5, 63 + vpsllq xmm0, xmm5, 1 + vpslldq xmm1, xmm1, 8 + vpor xmm0, xmm0, xmm1 + vpshufd xmm5, xmm5, 255 + vpsrad xmm5, xmm5, 31 + vpand xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_mod2_128 + vpxor xmm5, xmm5, xmm0 + vmovdqu xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128 + ; H ^ 1 and H ^ 2 + vpclmulqdq xmm9, xmm5, xmm5, 0 + vpclmulqdq xmm10, xmm5, xmm5, 17 + vpclmulqdq xmm8, xmm9, xmm3, 16 + vpshufd xmm9, xmm9, 78 + vpxor xmm9, xmm9, xmm8 + vpclmulqdq xmm8, xmm9, xmm3, 16 + vpshufd xmm9, xmm9, 78 + vpxor xmm9, xmm9, xmm8 + vpxor xmm0, xmm10, xmm9 + vmovdqu OWORD PTR [rsp], xmm5 + vmovdqu OWORD PTR [rsp+16], xmm0 + ; H ^ 3 and H ^ 4 + vpclmulqdq xmm11, xmm0, xmm5, 16 + vpclmulqdq xmm10, xmm0, xmm5, 1 + vpclmulqdq xmm9, xmm0, xmm5, 0 + vpclmulqdq xmm12, xmm0, xmm5, 17 + vpclmulqdq xmm13, xmm0, xmm0, 0 + vpclmulqdq xmm14, xmm0, xmm0, 17 + vpxor xmm11, xmm11, xmm10 + vpslldq xmm10, xmm11, 8 + vpsrldq xmm11, xmm11, 8 + vpxor xmm10, xmm10, xmm9 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm10, xmm10, xmm9 + vpxor xmm13, xmm13, xmm8 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm12, xmm12, xmm11 + vpxor xmm13, xmm13, xmm8 + vpxor xmm10, xmm10, xmm12 + vpxor xmm2, xmm13, xmm14 + vpxor xmm1, xmm10, xmm9 + vmovdqu OWORD PTR [rsp+32], xmm1 + vmovdqu OWORD PTR [rsp+48], xmm2 + ; H ^ 5 and H ^ 6 + vpclmulqdq xmm11, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 0 + vpclmulqdq xmm12, xmm1, xmm0, 17 + vpclmulqdq xmm13, xmm1, xmm1, 0 + vpclmulqdq xmm14, xmm1, xmm1, 17 + vpxor xmm11, xmm11, xmm10 + vpslldq xmm10, xmm11, 8 + vpsrldq xmm11, xmm11, 8 + vpxor xmm10, xmm10, xmm9 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm10, xmm10, xmm9 + vpxor xmm13, xmm13, xmm8 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm12, xmm12, xmm11 + vpxor xmm13, xmm13, xmm8 + vpxor xmm10, xmm10, xmm12 + vpxor xmm0, xmm13, xmm14 + vpxor xmm7, xmm10, xmm9 + vmovdqu OWORD PTR [rsp+64], xmm7 + vmovdqu OWORD PTR [rsp+80], xmm0 + ; H ^ 7 and H ^ 8 + vpclmulqdq xmm11, xmm2, xmm1, 16 + vpclmulqdq xmm10, xmm2, xmm1, 1 + vpclmulqdq xmm9, xmm2, xmm1, 0 + vpclmulqdq xmm12, xmm2, xmm1, 17 + vpclmulqdq xmm13, xmm2, xmm2, 0 + vpclmulqdq xmm14, xmm2, xmm2, 17 + vpxor xmm11, xmm11, xmm10 + vpslldq xmm10, xmm11, 8 + vpsrldq xmm11, xmm11, 8 + vpxor xmm10, xmm10, xmm9 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm10, xmm10, xmm9 + vpxor xmm13, xmm13, xmm8 + vpclmulqdq xmm9, xmm10, xmm3, 16 + vpclmulqdq xmm8, xmm13, xmm3, 16 + vpshufd xmm10, xmm10, 78 + vpshufd xmm13, xmm13, 78 + vpxor xmm12, xmm12, xmm11 + vpxor xmm13, xmm13, xmm8 + vpxor xmm10, xmm10, xmm12 + vpxor xmm0, xmm13, xmm14 + vpxor xmm7, xmm10, xmm9 + vmovdqu OWORD PTR [rsp+96], xmm7 + vmovdqu OWORD PTR [rsp+112], xmm0 +L_AES_GCM_decrypt_avx2_calc_aad_agg: + ; 64 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu xmm8, OWORD PTR [rbx] + vpshufb xmm8, xmm8, OWORD PTR L_avx2_aes_gcm_bswap_mask + vmovdqu xmm9, OWORD PTR [rbx+16] + vpshufb xmm9, xmm9, OWORD PTR L_avx2_aes_gcm_bswap_mask + vmovdqu xmm10, OWORD PTR [rbx+32] + vpshufb xmm10, xmm10, OWORD PTR L_avx2_aes_gcm_bswap_mask + vmovdqu xmm11, OWORD PTR [rbx+48] + vpshufb xmm11, xmm11, OWORD PTR L_avx2_aes_gcm_bswap_mask + vpxor xmm8, xmm8, xmm6 + vmovdqu xmm7, OWORD PTR [rsp+48] + vpclmulqdq xmm13, xmm7, xmm8, 16 + vpclmulqdq xmm1, xmm7, xmm8, 1 + vpclmulqdq xmm12, xmm7, xmm8, 0 + vpclmulqdq xmm6, xmm7, xmm8, 17 + vpxor xmm13, xmm13, xmm1 + vmovdqu xmm7, OWORD PTR [rsp+32] + vpclmulqdq xmm2, xmm7, xmm9, 16 + vpclmulqdq xmm1, xmm7, xmm9, 1 + vpclmulqdq xmm0, xmm7, xmm9, 0 + vpclmulqdq xmm3, xmm7, xmm9, 17 + vpxor xmm2, xmm2, xmm1 + vpxor xmm6, xmm6, xmm3 + vpxor xmm13, xmm13, xmm2 + vpxor xmm12, xmm12, xmm0 + vmovdqu xmm7, OWORD PTR [rsp+16] + vpclmulqdq xmm2, xmm7, xmm10, 16 + vpclmulqdq xmm1, xmm7, xmm10, 1 + vpclmulqdq xmm0, xmm7, xmm10, 0 + vpclmulqdq xmm3, xmm7, xmm10, 17 + vpxor xmm2, xmm2, xmm1 + vpxor xmm6, xmm6, xmm3 + vpxor xmm13, xmm13, xmm2 + vpxor xmm12, xmm12, xmm0 + vmovdqu xmm7, OWORD PTR [rsp] + vpclmulqdq xmm2, xmm7, xmm11, 16 + vpclmulqdq xmm1, xmm7, xmm11, 1 + vpclmulqdq xmm0, xmm7, xmm11, 0 + vpclmulqdq xmm3, xmm7, xmm11, 17 + vpxor xmm2, xmm2, xmm1 + vpxor xmm6, xmm6, xmm3 + vpxor xmm13, xmm13, xmm2 + vpxor xmm12, xmm12, xmm0 + vpslldq xmm7, xmm13, 8 + vpsrldq xmm13, xmm13, 8 + vpxor xmm12, xmm12, xmm7 + vpxor xmm6, xmm6, xmm13 + ; ghash_red + vmovdqu xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128 + vpclmulqdq xmm0, xmm12, xmm2, 16 + vpshufd xmm1, xmm12, 78 + vpxor xmm1, xmm1, xmm0 + vpclmulqdq xmm0, xmm1, xmm2, 16 + vpshufd xmm1, xmm1, 78 + vpxor xmm1, xmm1, xmm0 + vpxor xmm6, xmm6, xmm1 + add ecx, 64 + cmp ecx, r13d + jl L_AES_GCM_decrypt_avx2_calc_aad_agg + vmovdqu xmm5, OWORD PTR [rsp+168] +L_AES_GCM_decrypt_avx2_calc_aad_after_agg: + mov edx, r11d and edx, 4294967280 + cmp ecx, edx + je L_AES_GCM_decrypt_avx2_calc_aad_partial L_AES_GCM_decrypt_avx2_calc_aad_16_loop: vmovdqu xmm0, OWORD PTR [r12+rcx] vpshufb xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask @@ -13867,6 +14216,7 @@ L_AES_GCM_decrypt_avx2_calc_aad_16_loop: add ecx, 16 cmp ecx, edx jl L_AES_GCM_decrypt_avx2_calc_aad_16_loop +L_AES_GCM_decrypt_avx2_calc_aad_partial: mov edx, r11d cmp ecx, edx je L_AES_GCM_decrypt_avx2_calc_aad_done @@ -14509,17 +14859,17 @@ L_AES_GCM_decrypt_avx2_cmp_tag_16: L_AES_GCM_decrypt_avx2_cmp_tag_done: mov DWORD PTR [rbp], eax vzeroupper - vmovdqu xmm6, OWORD PTR [rsp+168] - vmovdqu xmm7, OWORD PTR [rsp+184] - vmovdqu xmm8, OWORD PTR [rsp+200] - vmovdqu xmm9, OWORD PTR [rsp+216] - vmovdqu xmm10, OWORD PTR [rsp+232] - vmovdqu xmm11, OWORD PTR [rsp+248] - vmovdqu xmm12, OWORD PTR [rsp+264] - vmovdqu xmm13, OWORD PTR [rsp+280] - vmovdqu xmm14, OWORD PTR [rsp+296] - vmovdqu xmm15, OWORD PTR [rsp+312] - add rsp, 328 + vmovdqu xmm6, OWORD PTR [rsp+184] + vmovdqu xmm7, OWORD PTR [rsp+200] + vmovdqu xmm8, OWORD PTR [rsp+216] + vmovdqu xmm9, OWORD PTR [rsp+232] + vmovdqu xmm10, OWORD PTR [rsp+248] + vmovdqu xmm11, OWORD PTR [rsp+264] + vmovdqu xmm12, OWORD PTR [rsp+280] + vmovdqu xmm13, OWORD PTR [rsp+296] + vmovdqu xmm14, OWORD PTR [rsp+312] + vmovdqu xmm15, OWORD PTR [rsp+328] + add rsp, 344 pop rbp pop rsi pop r15 @@ -16854,17 +17204,17 @@ AES_GCM_encrypt_vaes PROC mov r14d, DWORD PTR [rsp+128] mov r15, QWORD PTR [rsp+136] mov r10d, DWORD PTR [rsp+144] - sub rsp, 720 - vmovdqu OWORD PTR [rsp+560], xmm6 - vmovdqu OWORD PTR [rsp+576], xmm7 - vmovdqu OWORD PTR [rsp+592], xmm8 - vmovdqu OWORD PTR [rsp+608], xmm9 - vmovdqu OWORD PTR [rsp+624], xmm10 - vmovdqu OWORD PTR [rsp+640], xmm11 - vmovdqu OWORD PTR [rsp+656], xmm12 - vmovdqu OWORD PTR [rsp+672], xmm13 - vmovdqu OWORD PTR [rsp+688], xmm14 - vmovdqu OWORD PTR [rsp+704], xmm15 + sub rsp, 752 + vmovdqu OWORD PTR [rsp+592], xmm6 + vmovdqu OWORD PTR [rsp+608], xmm7 + vmovdqu OWORD PTR [rsp+624], xmm8 + vmovdqu OWORD PTR [rsp+640], xmm9 + vmovdqu OWORD PTR [rsp+656], xmm10 + vmovdqu OWORD PTR [rsp+672], xmm11 + vmovdqu OWORD PTR [rsp+688], xmm12 + vmovdqu OWORD PTR [rsp+704], xmm13 + vmovdqu OWORD PTR [rsp+720], xmm14 + vmovdqu OWORD PTR [rsp+736], xmm15 vpxor xmm5, xmm5, xmm5 vpxor xmm15, xmm15, xmm15 mov edx, ebx @@ -17158,9 +17508,527 @@ L_AES_GCM_encrypt_vaes_iv_done: cmp edx, 0 je L_AES_GCM_encrypt_vaes_calc_aad_done xor ecx, ecx - cmp edx, 16 - jl L_AES_GCM_encrypt_vaes_calc_aad_lt16 + cmp edx, 128 + jl L_AES_GCM_encrypt_vaes_calc_aad_ser + vmovdqu OWORD PTR [rsp+560], xmm6 + vmovdqu OWORD PTR [rsp+576], xmm5 + vpsrlq xmm8, xmm6, 63 + vpsllq xmm7, xmm6, 1 + vpslldq xmm8, xmm8, 8 + vpor xmm7, xmm7, xmm8 + vpshufd xmm6, xmm6, 255 + vpsrad xmm6, xmm6, 31 + vpand xmm6, xmm6, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpxor xmm6, xmm6, xmm7 + vmovdqa xmm2, xmm15 + ; H ^ 1 + vmovdqu OWORD PTR [rsp], xmm6 + ; H ^ 2 + vpclmulqdq xmm7, xmm6, xmm6, 0 + vpclmulqdq xmm10, xmm6, xmm6, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm0, xmm10 + vmovdqu OWORD PTR [rsp+16], xmm0 + ; H ^ 3 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm0, xmm6, 0 + vpclmulqdq xmm8, xmm0, xmm6, 1 + vpclmulqdq xmm9, xmm0, xmm6, 16 + vpclmulqdq xmm10, xmm0, xmm6, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm1, xmm10 + vmovdqu OWORD PTR [rsp+32], xmm1 + ; H ^ 4 + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm3, xmm10 + vmovdqu OWORD PTR [rsp+48], xmm3 + ; H ^ 5 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+64], xmm4 + ; H ^ 6 + vpclmulqdq xmm7, xmm1, xmm1, 0 + vpclmulqdq xmm10, xmm1, xmm1, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+80], xmm4 + ; H ^ 7 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm3, xmm1, 0 + vpclmulqdq xmm8, xmm3, xmm1, 1 + vpclmulqdq xmm9, xmm3, xmm1, 16 + vpclmulqdq xmm10, xmm3, xmm1, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+96], xmm4 + ; H ^ 8 + vpclmulqdq xmm7, xmm3, xmm3, 0 + vpclmulqdq xmm10, xmm3, xmm3, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+112], xmm4 + cmp edx, 256 + jl L_AES_GCM_encrypt_vaes_calc_aad_no_ext + ; H ^ 9 + vmovdqu xmm0, OWORD PTR [rsp+48] + vmovdqu xmm1, OWORD PTR [rsp+64] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+128], xmm4 + ; H ^ 10 + vmovdqu xmm0, OWORD PTR [rsp+64] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+144], xmm4 + ; H ^ 11 + vmovdqu xmm0, OWORD PTR [rsp+64] + vmovdqu xmm1, OWORD PTR [rsp+80] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+160], xmm4 + ; H ^ 12 + vmovdqu xmm0, OWORD PTR [rsp+80] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+176], xmm4 + ; H ^ 13 + vmovdqu xmm0, OWORD PTR [rsp+80] + vmovdqu xmm1, OWORD PTR [rsp+96] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+192], xmm4 + ; H ^ 14 + vmovdqu xmm0, OWORD PTR [rsp+96] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+208], xmm4 + ; H ^ 15 + vmovdqu xmm0, OWORD PTR [rsp+96] + vmovdqu xmm1, OWORD PTR [rsp+112] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+224], xmm4 + ; H ^ 16 + vmovdqu xmm0, OWORD PTR [rsp+112] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+240], xmm4 + vmovdqu ymm7, YMMWORD PTR [rsp+224] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+192] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+160] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp+128] + vpermq ymm10, ymm10, 78 + vmovdqu YMMWORD PTR [rsp+256], ymm7 + vmovdqu YMMWORD PTR [rsp+288], ymm8 + vmovdqu YMMWORD PTR [rsp+320], ymm9 + vmovdqu YMMWORD PTR [rsp+352], ymm10 + vmovdqu ymm7, YMMWORD PTR [rsp+96] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+64] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+32] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp] + vpermq ymm10, ymm10, 78 + vmovdqu YMMWORD PTR [rsp+384], ymm7 + vmovdqu YMMWORD PTR [rsp+416], ymm8 + vmovdqu YMMWORD PTR [rsp+448], ymm9 + vmovdqu YMMWORD PTR [rsp+480], ymm10 +L_AES_GCM_encrypt_vaes_calc_aad_no_ext: + vbroadcasti128 ymm14, ptr_L_vaes_aes_gcm_mod2_128 + mov r13d, r11d + and r13d, 4294967040 + cmp ecx, r13d + jge L_AES_GCM_encrypt_vaes_calc_aad_a256 +L_AES_GCM_encrypt_vaes_calc_aad_l256: + ; 256 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vbroadcasti128 ymm6, ptr_L_vaes_aes_gcm_bswap_mask + vpxor ymm4, ymm4, ymm4 + vinserti128 ymm4, ymm4, xmm15, 0 + vmovdqu ymm7, YMMWORD PTR [rsp+256] + vmovdqu ymm8, YMMWORD PTR [rsp+288] + vmovdqu ymm9, YMMWORD PTR [rsp+320] + vmovdqu ymm10, YMMWORD PTR [rsp+352] + vmovdqu ymm5, YMMWORD PTR [rbx] + vpshufb ymm5, ymm5, ymm6 + vpxor ymm5, ymm5, ymm4 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vmovdqa ymm11, ymm0 + vpxor ymm12, ymm2, ymm1 + vmovdqa ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+32] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+64] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+96] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm7, YMMWORD PTR [rsp+384] + vmovdqu ymm8, YMMWORD PTR [rsp+416] + vmovdqu ymm9, YMMWORD PTR [rsp+448] + vmovdqu ymm10, YMMWORD PTR [rsp+480] + vmovdqu ymm5, YMMWORD PTR [rbx+128] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+160] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+192] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+224] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vpclmulqdq ymm5, ymm14, ymm11, 1 + vpshufd ymm11, ymm11, 78 + vpxor ymm12, ymm12, ymm5 + vpxor ymm12, ymm12, ymm11 + vpclmulqdq ymm5, ymm14, ymm12, 1 + vpshufd ymm12, ymm12, 78 + vpxor ymm13, ymm13, ymm5 + vpxor ymm13, ymm13, ymm12 + vextracti128 xmm0, ymm13, 1 + vpxor xmm15, xmm13, xmm0 + add ecx, 256 + cmp ecx, r13d + jl L_AES_GCM_encrypt_vaes_calc_aad_l256 +L_AES_GCM_encrypt_vaes_calc_aad_a256: + mov r13d, r11d + and r13d, 4294967168 + cmp ecx, r13d + jge L_AES_GCM_encrypt_vaes_calc_aad_a128 + ; 128 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu ymm7, YMMWORD PTR [rsp+96] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+64] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+32] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp] + vpermq ymm10, ymm10, 78 + vbroadcasti128 ymm6, ptr_L_vaes_aes_gcm_bswap_mask + vpxor ymm4, ymm4, ymm4 + vinserti128 ymm4, ymm4, xmm15, 0 + vmovdqu ymm5, YMMWORD PTR [rbx] + vpshufb ymm5, ymm5, ymm6 + vpxor ymm5, ymm5, ymm4 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vmovdqa ymm11, ymm0 + vpxor ymm12, ymm2, ymm1 + vmovdqa ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+32] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+64] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+96] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vpclmulqdq ymm5, ymm14, ymm11, 1 + vpshufd ymm11, ymm11, 78 + vpxor ymm12, ymm12, ymm5 + vpxor ymm12, ymm12, ymm11 + vpclmulqdq ymm5, ymm14, ymm12, 1 + vpshufd ymm12, ymm12, 78 + vpxor ymm13, ymm13, ymm5 + vpxor ymm13, ymm13, ymm12 + vextracti128 xmm0, ymm13, 1 + vpxor xmm15, xmm13, xmm0 + add ecx, 128 +L_AES_GCM_encrypt_vaes_calc_aad_a128: + mov r13d, r11d + and r13d, 4294967280 + cmp ecx, r13d + jge L_AES_GCM_encrypt_vaes_calc_aad_arem + vmovdqu xmm6, OWORD PTR [rsp] +L_AES_GCM_encrypt_vaes_calc_aad_rem: + ; 16 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu xmm7, OWORD PTR [rbx] + vpshufb xmm7, xmm7, OWORD PTR L_vaes_aes_gcm_bswap_mask + vpxor xmm15, xmm15, xmm7 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm15, xmm6, 0 + vpclmulqdq xmm8, xmm15, xmm6, 1 + vpclmulqdq xmm9, xmm15, xmm6, 16 + vpclmulqdq xmm10, xmm15, xmm6, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm15, xmm10 + add ecx, 16 + cmp ecx, r13d + jl L_AES_GCM_encrypt_vaes_calc_aad_rem +L_AES_GCM_encrypt_vaes_calc_aad_arem: + vmovdqu xmm6, OWORD PTR [rsp+560] + vmovdqu xmm5, OWORD PTR [rsp+576] + jmp L_AES_GCM_encrypt_vaes_calc_aad_partial +L_AES_GCM_encrypt_vaes_calc_aad_ser: + mov edx, r11d and edx, 4294967280 + cmp ecx, edx + je L_AES_GCM_encrypt_vaes_calc_aad_partial L_AES_GCM_encrypt_vaes_calc_aad_16_loop: vmovdqu xmm7, OWORD PTR [r12+rcx] vpshufb xmm7, xmm7, OWORD PTR L_vaes_aes_gcm_bswap_mask @@ -17211,6 +18079,7 @@ L_AES_GCM_encrypt_vaes_calc_aad_16_loop: add ecx, 16 cmp ecx, edx jl L_AES_GCM_encrypt_vaes_calc_aad_16_loop +L_AES_GCM_encrypt_vaes_calc_aad_partial: mov edx, r11d cmp ecx, edx je L_AES_GCM_encrypt_vaes_calc_aad_done @@ -18314,17 +19183,17 @@ L_AES_GCM_encrypt_vaes_store_tag_16: vmovdqu OWORD PTR [r8], xmm0 L_AES_GCM_encrypt_vaes_store_tag_done: vzeroupper - vmovdqu xmm6, OWORD PTR [rsp+560] - vmovdqu xmm7, OWORD PTR [rsp+576] - vmovdqu xmm8, OWORD PTR [rsp+592] - vmovdqu xmm9, OWORD PTR [rsp+608] - vmovdqu xmm10, OWORD PTR [rsp+624] - vmovdqu xmm11, OWORD PTR [rsp+640] - vmovdqu xmm12, OWORD PTR [rsp+656] - vmovdqu xmm13, OWORD PTR [rsp+672] - vmovdqu xmm14, OWORD PTR [rsp+688] - vmovdqu xmm15, OWORD PTR [rsp+704] - add rsp, 720 + vmovdqu xmm6, OWORD PTR [rsp+592] + vmovdqu xmm7, OWORD PTR [rsp+608] + vmovdqu xmm8, OWORD PTR [rsp+624] + vmovdqu xmm9, OWORD PTR [rsp+640] + vmovdqu xmm10, OWORD PTR [rsp+656] + vmovdqu xmm11, OWORD PTR [rsp+672] + vmovdqu xmm12, OWORD PTR [rsp+688] + vmovdqu xmm13, OWORD PTR [rsp+704] + vmovdqu xmm14, OWORD PTR [rsp+720] + vmovdqu xmm15, OWORD PTR [rsp+736] + add rsp, 752 pop r15 pop r14 pop rbx @@ -18357,17 +19226,17 @@ AES_GCM_decrypt_vaes PROC mov r15, QWORD PTR [rsp+144] mov r10d, DWORD PTR [rsp+152] mov rbp, QWORD PTR [rsp+160] - sub rsp, 704 - vmovdqu OWORD PTR [rsp+544], xmm6 - vmovdqu OWORD PTR [rsp+560], xmm7 - vmovdqu OWORD PTR [rsp+576], xmm8 - vmovdqu OWORD PTR [rsp+592], xmm9 - vmovdqu OWORD PTR [rsp+608], xmm10 - vmovdqu OWORD PTR [rsp+624], xmm11 - vmovdqu OWORD PTR [rsp+640], xmm12 - vmovdqu OWORD PTR [rsp+656], xmm13 - vmovdqu OWORD PTR [rsp+672], xmm14 - vmovdqu OWORD PTR [rsp+688], xmm15 + sub rsp, 752 + vmovdqu OWORD PTR [rsp+592], xmm6 + vmovdqu OWORD PTR [rsp+608], xmm7 + vmovdqu OWORD PTR [rsp+624], xmm8 + vmovdqu OWORD PTR [rsp+640], xmm9 + vmovdqu OWORD PTR [rsp+656], xmm10 + vmovdqu OWORD PTR [rsp+672], xmm11 + vmovdqu OWORD PTR [rsp+688], xmm12 + vmovdqu OWORD PTR [rsp+704], xmm13 + vmovdqu OWORD PTR [rsp+720], xmm14 + vmovdqu OWORD PTR [rsp+736], xmm15 vpxor xmm5, xmm5, xmm5 vpxor xmm15, xmm15, xmm15 cmp ebx, 12 @@ -18661,9 +19530,527 @@ L_AES_GCM_decrypt_vaes_iv_done: cmp edx, 0 je L_AES_GCM_decrypt_vaes_calc_aad_done xor ecx, ecx - cmp edx, 16 - jl L_AES_GCM_decrypt_vaes_calc_aad_lt16 + cmp edx, 128 + jl L_AES_GCM_decrypt_vaes_calc_aad_ser + vmovdqu OWORD PTR [rsp+560], xmm6 + vmovdqu OWORD PTR [rsp+576], xmm5 + vpsrlq xmm8, xmm6, 63 + vpsllq xmm7, xmm6, 1 + vpslldq xmm8, xmm8, 8 + vpor xmm7, xmm7, xmm8 + vpshufd xmm6, xmm6, 255 + vpsrad xmm6, xmm6, 31 + vpand xmm6, xmm6, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpxor xmm6, xmm6, xmm7 + vmovdqa xmm2, xmm15 + ; H ^ 1 + vmovdqu OWORD PTR [rsp], xmm6 + ; H ^ 2 + vpclmulqdq xmm7, xmm6, xmm6, 0 + vpclmulqdq xmm10, xmm6, xmm6, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm0, xmm10 + vmovdqu OWORD PTR [rsp+16], xmm0 + ; H ^ 3 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm0, xmm6, 0 + vpclmulqdq xmm8, xmm0, xmm6, 1 + vpclmulqdq xmm9, xmm0, xmm6, 16 + vpclmulqdq xmm10, xmm0, xmm6, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm1, xmm10 + vmovdqu OWORD PTR [rsp+32], xmm1 + ; H ^ 4 + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm3, xmm10 + vmovdqu OWORD PTR [rsp+48], xmm3 + ; H ^ 5 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+64], xmm4 + ; H ^ 6 + vpclmulqdq xmm7, xmm1, xmm1, 0 + vpclmulqdq xmm10, xmm1, xmm1, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+80], xmm4 + ; H ^ 7 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm3, xmm1, 0 + vpclmulqdq xmm8, xmm3, xmm1, 1 + vpclmulqdq xmm9, xmm3, xmm1, 16 + vpclmulqdq xmm10, xmm3, xmm1, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+96], xmm4 + ; H ^ 8 + vpclmulqdq xmm7, xmm3, xmm3, 0 + vpclmulqdq xmm10, xmm3, xmm3, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+112], xmm4 + cmp edx, 256 + jl L_AES_GCM_decrypt_vaes_calc_aad_no_ext + ; H ^ 9 + vmovdqu xmm0, OWORD PTR [rsp+48] + vmovdqu xmm1, OWORD PTR [rsp+64] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+128], xmm4 + ; H ^ 10 + vmovdqu xmm0, OWORD PTR [rsp+64] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+144], xmm4 + ; H ^ 11 + vmovdqu xmm0, OWORD PTR [rsp+64] + vmovdqu xmm1, OWORD PTR [rsp+80] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+160], xmm4 + ; H ^ 12 + vmovdqu xmm0, OWORD PTR [rsp+80] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+176], xmm4 + ; H ^ 13 + vmovdqu xmm0, OWORD PTR [rsp+80] + vmovdqu xmm1, OWORD PTR [rsp+96] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+192], xmm4 + ; H ^ 14 + vmovdqu xmm0, OWORD PTR [rsp+96] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+208], xmm4 + ; H ^ 15 + vmovdqu xmm0, OWORD PTR [rsp+96] + vmovdqu xmm1, OWORD PTR [rsp+112] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+224], xmm4 + ; H ^ 16 + vmovdqu xmm0, OWORD PTR [rsp+112] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+240], xmm4 + vmovdqu ymm7, YMMWORD PTR [rsp+224] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+192] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+160] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp+128] + vpermq ymm10, ymm10, 78 + vmovdqu YMMWORD PTR [rsp+256], ymm7 + vmovdqu YMMWORD PTR [rsp+288], ymm8 + vmovdqu YMMWORD PTR [rsp+320], ymm9 + vmovdqu YMMWORD PTR [rsp+352], ymm10 + vmovdqu ymm7, YMMWORD PTR [rsp+96] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+64] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+32] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp] + vpermq ymm10, ymm10, 78 + vmovdqu YMMWORD PTR [rsp+384], ymm7 + vmovdqu YMMWORD PTR [rsp+416], ymm8 + vmovdqu YMMWORD PTR [rsp+448], ymm9 + vmovdqu YMMWORD PTR [rsp+480], ymm10 +L_AES_GCM_decrypt_vaes_calc_aad_no_ext: + vbroadcasti128 ymm14, ptr_L_vaes_aes_gcm_mod2_128 + mov r13d, r11d + and r13d, 4294967040 + cmp ecx, r13d + jge L_AES_GCM_decrypt_vaes_calc_aad_a256 +L_AES_GCM_decrypt_vaes_calc_aad_l256: + ; 256 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vbroadcasti128 ymm6, ptr_L_vaes_aes_gcm_bswap_mask + vpxor ymm4, ymm4, ymm4 + vinserti128 ymm4, ymm4, xmm15, 0 + vmovdqu ymm7, YMMWORD PTR [rsp+256] + vmovdqu ymm8, YMMWORD PTR [rsp+288] + vmovdqu ymm9, YMMWORD PTR [rsp+320] + vmovdqu ymm10, YMMWORD PTR [rsp+352] + vmovdqu ymm5, YMMWORD PTR [rbx] + vpshufb ymm5, ymm5, ymm6 + vpxor ymm5, ymm5, ymm4 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vmovdqa ymm11, ymm0 + vpxor ymm12, ymm2, ymm1 + vmovdqa ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+32] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+64] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+96] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm7, YMMWORD PTR [rsp+384] + vmovdqu ymm8, YMMWORD PTR [rsp+416] + vmovdqu ymm9, YMMWORD PTR [rsp+448] + vmovdqu ymm10, YMMWORD PTR [rsp+480] + vmovdqu ymm5, YMMWORD PTR [rbx+128] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+160] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+192] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+224] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vpclmulqdq ymm5, ymm14, ymm11, 1 + vpshufd ymm11, ymm11, 78 + vpxor ymm12, ymm12, ymm5 + vpxor ymm12, ymm12, ymm11 + vpclmulqdq ymm5, ymm14, ymm12, 1 + vpshufd ymm12, ymm12, 78 + vpxor ymm13, ymm13, ymm5 + vpxor ymm13, ymm13, ymm12 + vextracti128 xmm0, ymm13, 1 + vpxor xmm15, xmm13, xmm0 + add ecx, 256 + cmp ecx, r13d + jl L_AES_GCM_decrypt_vaes_calc_aad_l256 +L_AES_GCM_decrypt_vaes_calc_aad_a256: + mov r13d, r11d + and r13d, 4294967168 + cmp ecx, r13d + jge L_AES_GCM_decrypt_vaes_calc_aad_a128 + ; 128 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu ymm7, YMMWORD PTR [rsp+96] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+64] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+32] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp] + vpermq ymm10, ymm10, 78 + vbroadcasti128 ymm6, ptr_L_vaes_aes_gcm_bswap_mask + vpxor ymm4, ymm4, ymm4 + vinserti128 ymm4, ymm4, xmm15, 0 + vmovdqu ymm5, YMMWORD PTR [rbx] + vpshufb ymm5, ymm5, ymm6 + vpxor ymm5, ymm5, ymm4 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vmovdqa ymm11, ymm0 + vpxor ymm12, ymm2, ymm1 + vmovdqa ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+32] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+64] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rbx+96] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vpclmulqdq ymm5, ymm14, ymm11, 1 + vpshufd ymm11, ymm11, 78 + vpxor ymm12, ymm12, ymm5 + vpxor ymm12, ymm12, ymm11 + vpclmulqdq ymm5, ymm14, ymm12, 1 + vpshufd ymm12, ymm12, 78 + vpxor ymm13, ymm13, ymm5 + vpxor ymm13, ymm13, ymm12 + vextracti128 xmm0, ymm13, 1 + vpxor xmm15, xmm13, xmm0 + add ecx, 128 +L_AES_GCM_decrypt_vaes_calc_aad_a128: + mov r13d, r11d + and r13d, 4294967280 + cmp ecx, r13d + jge L_AES_GCM_decrypt_vaes_calc_aad_arem + vmovdqu xmm6, OWORD PTR [rsp] +L_AES_GCM_decrypt_vaes_calc_aad_rem: + ; 16 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu xmm7, OWORD PTR [rbx] + vpshufb xmm7, xmm7, OWORD PTR L_vaes_aes_gcm_bswap_mask + vpxor xmm15, xmm15, xmm7 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm15, xmm6, 0 + vpclmulqdq xmm8, xmm15, xmm6, 1 + vpclmulqdq xmm9, xmm15, xmm6, 16 + vpclmulqdq xmm10, xmm15, xmm6, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm15, xmm10 + add ecx, 16 + cmp ecx, r13d + jl L_AES_GCM_decrypt_vaes_calc_aad_rem +L_AES_GCM_decrypt_vaes_calc_aad_arem: + vmovdqu xmm6, OWORD PTR [rsp+560] + vmovdqu xmm5, OWORD PTR [rsp+576] + jmp L_AES_GCM_decrypt_vaes_calc_aad_partial +L_AES_GCM_decrypt_vaes_calc_aad_ser: + mov edx, r11d and edx, 4294967280 + cmp ecx, edx + je L_AES_GCM_decrypt_vaes_calc_aad_partial L_AES_GCM_decrypt_vaes_calc_aad_16_loop: vmovdqu xmm7, OWORD PTR [r12+rcx] vpshufb xmm7, xmm7, OWORD PTR L_vaes_aes_gcm_bswap_mask @@ -18714,6 +20101,7 @@ L_AES_GCM_decrypt_vaes_calc_aad_16_loop: add ecx, 16 cmp ecx, edx jl L_AES_GCM_decrypt_vaes_calc_aad_16_loop +L_AES_GCM_decrypt_vaes_calc_aad_partial: mov edx, r11d cmp ecx, edx je L_AES_GCM_decrypt_vaes_calc_aad_done @@ -19769,17 +21157,17 @@ L_AES_GCM_decrypt_vaes_cmp_tag_16: L_AES_GCM_decrypt_vaes_cmp_tag_done: mov DWORD PTR [rbp], ebx vzeroupper - vmovdqu xmm6, OWORD PTR [rsp+544] - vmovdqu xmm7, OWORD PTR [rsp+560] - vmovdqu xmm8, OWORD PTR [rsp+576] - vmovdqu xmm9, OWORD PTR [rsp+592] - vmovdqu xmm10, OWORD PTR [rsp+608] - vmovdqu xmm11, OWORD PTR [rsp+624] - vmovdqu xmm12, OWORD PTR [rsp+640] - vmovdqu xmm13, OWORD PTR [rsp+656] - vmovdqu xmm14, OWORD PTR [rsp+672] - vmovdqu xmm15, OWORD PTR [rsp+688] - add rsp, 704 + vmovdqu xmm6, OWORD PTR [rsp+592] + vmovdqu xmm7, OWORD PTR [rsp+608] + vmovdqu xmm8, OWORD PTR [rsp+624] + vmovdqu xmm9, OWORD PTR [rsp+640] + vmovdqu xmm10, OWORD PTR [rsp+656] + vmovdqu xmm11, OWORD PTR [rsp+672] + vmovdqu xmm12, OWORD PTR [rsp+688] + vmovdqu xmm13, OWORD PTR [rsp+704] + vmovdqu xmm14, OWORD PTR [rsp+720] + vmovdqu xmm15, OWORD PTR [rsp+736] + add rsp, 752 pop rbp pop r15 pop r14 @@ -20115,67 +21503,545 @@ AES_GCM_init_vaes ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA AES_GCM_aad_update_vaes PROC - mov rax, rcx - sub rsp, 32 + mov r10, r8 + mov r8, rcx + mov r11, r9 + mov r9d, edx + sub rsp, 672 + vmovdqu OWORD PTR [rsp+512], xmm6 + vmovdqu OWORD PTR [rsp+528], xmm7 + vmovdqu OWORD PTR [rsp+544], xmm8 + vmovdqu OWORD PTR [rsp+560], xmm9 + vmovdqu OWORD PTR [rsp+576], xmm10 + vmovdqu OWORD PTR [rsp+592], xmm11 + vmovdqu OWORD PTR [rsp+608], xmm12 + vmovdqu OWORD PTR [rsp+624], xmm13 + vmovdqu OWORD PTR [rsp+640], xmm14 + vmovdqu OWORD PTR [rsp+656], xmm15 + vmovdqa xmm15, OWORD PTR [r10] + vmovdqa xmm6, OWORD PTR [r11] + xor edx, edx + vpsrlq xmm8, xmm6, 63 + vpsllq xmm7, xmm6, 1 + vpslldq xmm8, xmm8, 8 + vpor xmm7, xmm7, xmm8 + vpshufd xmm6, xmm6, 255 + vpsrad xmm6, xmm6, 31 + vpand xmm6, xmm6, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpxor xmm6, xmm6, xmm7 + vmovdqa xmm2, xmm15 + ; H ^ 1 vmovdqu OWORD PTR [rsp], xmm6 - vmovdqu OWORD PTR [rsp+16], xmm7 - vmovdqa xmm5, OWORD PTR [r8] - vmovdqa xmm6, OWORD PTR [r9] - xor ecx, ecx -L_AES_GCM_aad_update_vaes_16_loop: - vmovdqu xmm7, OWORD PTR [rax+rcx] - vpshufb xmm7, xmm7, OWORD PTR L_vaes_aes_gcm_bswap_mask - vpxor xmm5, xmm5, xmm7 - ; ghash_gfmul_avx - vpshufd xmm1, xmm5, 78 - vpshufd xmm2, xmm6, 78 - vpclmulqdq xmm3, xmm6, xmm5, 17 - vpclmulqdq xmm0, xmm6, xmm5, 0 - vpxor xmm1, xmm1, xmm5 - vpxor xmm2, xmm2, xmm6 - vpclmulqdq xmm1, xmm1, xmm2, 0 - vpxor xmm1, xmm1, xmm0 - vpxor xmm1, xmm1, xmm3 - vmovdqa xmm4, xmm0 - vmovdqa xmm5, xmm3 - vpslldq xmm2, xmm1, 8 - vpsrldq xmm1, xmm1, 8 - vpxor xmm4, xmm4, xmm2 - vpxor xmm5, xmm5, xmm1 - vpsrld xmm0, xmm4, 31 - vpsrld xmm1, xmm5, 31 - vpslld xmm4, xmm4, 1 - vpslld xmm5, xmm5, 1 - vpsrldq xmm2, xmm0, 12 - vpslldq xmm0, xmm0, 4 - vpslldq xmm1, xmm1, 4 - vpor xmm5, xmm5, xmm2 - vpor xmm4, xmm4, xmm0 - vpor xmm5, xmm5, xmm1 - vpslld xmm0, xmm4, 31 - vpslld xmm1, xmm4, 30 - vpslld xmm2, xmm4, 25 - vpxor xmm0, xmm0, xmm1 - vpxor xmm0, xmm0, xmm2 - vmovdqa xmm1, xmm0 - vpsrldq xmm1, xmm1, 4 - vpslldq xmm0, xmm0, 12 - vpxor xmm4, xmm4, xmm0 - vpsrld xmm2, xmm4, 1 - vpsrld xmm3, xmm4, 2 - vpsrld xmm0, xmm4, 7 - vpxor xmm2, xmm2, xmm3 - vpxor xmm2, xmm2, xmm0 - vpxor xmm2, xmm2, xmm1 - vpxor xmm2, xmm2, xmm4 - vpxor xmm5, xmm5, xmm2 - add ecx, 16 - cmp ecx, edx - jl L_AES_GCM_aad_update_vaes_16_loop - vmovdqa OWORD PTR [r8], xmm5 + ; H ^ 2 + vpclmulqdq xmm7, xmm6, xmm6, 0 + vpclmulqdq xmm10, xmm6, xmm6, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm0, xmm10 + vmovdqu OWORD PTR [rsp+16], xmm0 + ; H ^ 3 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm0, xmm6, 0 + vpclmulqdq xmm8, xmm0, xmm6, 1 + vpclmulqdq xmm9, xmm0, xmm6, 16 + vpclmulqdq xmm10, xmm0, xmm6, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm1, xmm10 + vmovdqu OWORD PTR [rsp+32], xmm1 + ; H ^ 4 + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm3, xmm10 + vmovdqu OWORD PTR [rsp+48], xmm3 + ; H ^ 5 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+64], xmm4 + ; H ^ 6 + vpclmulqdq xmm7, xmm1, xmm1, 0 + vpclmulqdq xmm10, xmm1, xmm1, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+80], xmm4 + ; H ^ 7 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm3, xmm1, 0 + vpclmulqdq xmm8, xmm3, xmm1, 1 + vpclmulqdq xmm9, xmm3, xmm1, 16 + vpclmulqdq xmm10, xmm3, xmm1, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+96], xmm4 + ; H ^ 8 + vpclmulqdq xmm7, xmm3, xmm3, 0 + vpclmulqdq xmm10, xmm3, xmm3, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+112], xmm4 + cmp r9d, 256 + jl L_AES_GCM_aad_update_vaes_no_ext + ; H ^ 9 + vmovdqu xmm0, OWORD PTR [rsp+48] + vmovdqu xmm1, OWORD PTR [rsp+64] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+128], xmm4 + ; H ^ 10 + vmovdqu xmm0, OWORD PTR [rsp+64] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+144], xmm4 + ; H ^ 11 + vmovdqu xmm0, OWORD PTR [rsp+64] + vmovdqu xmm1, OWORD PTR [rsp+80] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+160], xmm4 + ; H ^ 12 + vmovdqu xmm0, OWORD PTR [rsp+80] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+176], xmm4 + ; H ^ 13 + vmovdqu xmm0, OWORD PTR [rsp+80] + vmovdqu xmm1, OWORD PTR [rsp+96] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+192], xmm4 + ; H ^ 14 + vmovdqu xmm0, OWORD PTR [rsp+96] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+208], xmm4 + ; H ^ 15 + vmovdqu xmm0, OWORD PTR [rsp+96] + vmovdqu xmm1, OWORD PTR [rsp+112] + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm1, xmm0, 0 + vpclmulqdq xmm8, xmm1, xmm0, 1 + vpclmulqdq xmm9, xmm1, xmm0, 16 + vpclmulqdq xmm10, xmm1, xmm0, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+224], xmm4 + ; H ^ 16 + vmovdqu xmm0, OWORD PTR [rsp+112] + vpclmulqdq xmm7, xmm0, xmm0, 0 + vpclmulqdq xmm10, xmm0, xmm0, 17 + vpxor xmm8, xmm8, xmm8 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm4, xmm10 + vmovdqu OWORD PTR [rsp+240], xmm4 + vmovdqu ymm7, YMMWORD PTR [rsp+224] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+192] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+160] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp+128] + vpermq ymm10, ymm10, 78 + vmovdqu YMMWORD PTR [rsp+256], ymm7 + vmovdqu YMMWORD PTR [rsp+288], ymm8 + vmovdqu YMMWORD PTR [rsp+320], ymm9 + vmovdqu YMMWORD PTR [rsp+352], ymm10 + vmovdqu ymm7, YMMWORD PTR [rsp+96] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+64] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+32] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp] + vpermq ymm10, ymm10, 78 + vmovdqu YMMWORD PTR [rsp+384], ymm7 + vmovdqu YMMWORD PTR [rsp+416], ymm8 + vmovdqu YMMWORD PTR [rsp+448], ymm9 + vmovdqu YMMWORD PTR [rsp+480], ymm10 +L_AES_GCM_aad_update_vaes_no_ext: + vbroadcasti128 ymm14, ptr_L_vaes_aes_gcm_mod2_128 + mov eax, r9d + and eax, 4294967040 + cmp edx, eax + jge L_AES_GCM_aad_update_vaes_after_256 +L_AES_GCM_aad_update_vaes_loop_256: + ; 256 bytes of input + lea rcx, QWORD PTR [r8+rdx] + vbroadcasti128 ymm6, ptr_L_vaes_aes_gcm_bswap_mask + vpxor ymm4, ymm4, ymm4 + vinserti128 ymm4, ymm4, xmm15, 0 + vmovdqu ymm7, YMMWORD PTR [rsp+256] + vmovdqu ymm8, YMMWORD PTR [rsp+288] + vmovdqu ymm9, YMMWORD PTR [rsp+320] + vmovdqu ymm10, YMMWORD PTR [rsp+352] + vmovdqu ymm5, YMMWORD PTR [rcx] + vpshufb ymm5, ymm5, ymm6 + vpxor ymm5, ymm5, ymm4 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vmovdqa ymm11, ymm0 + vpxor ymm12, ymm2, ymm1 + vmovdqa ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+32] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+64] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+96] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm7, YMMWORD PTR [rsp+384] + vmovdqu ymm8, YMMWORD PTR [rsp+416] + vmovdqu ymm9, YMMWORD PTR [rsp+448] + vmovdqu ymm10, YMMWORD PTR [rsp+480] + vmovdqu ymm5, YMMWORD PTR [rcx+128] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+160] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+192] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+224] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vpclmulqdq ymm5, ymm14, ymm11, 1 + vpshufd ymm11, ymm11, 78 + vpxor ymm12, ymm12, ymm5 + vpxor ymm12, ymm12, ymm11 + vpclmulqdq ymm5, ymm14, ymm12, 1 + vpshufd ymm12, ymm12, 78 + vpxor ymm13, ymm13, ymm5 + vpxor ymm13, ymm13, ymm12 + vextracti128 xmm0, ymm13, 1 + vpxor xmm15, xmm13, xmm0 + add edx, 256 + cmp edx, eax + jl L_AES_GCM_aad_update_vaes_loop_256 +L_AES_GCM_aad_update_vaes_after_256: + mov eax, r9d + and eax, 4294967168 + cmp edx, eax + jge L_AES_GCM_aad_update_vaes_after_128 + ; 128 bytes of input + lea rcx, QWORD PTR [r8+rdx] + vmovdqu ymm7, YMMWORD PTR [rsp+96] + vpermq ymm7, ymm7, 78 + vmovdqu ymm8, YMMWORD PTR [rsp+64] + vpermq ymm8, ymm8, 78 + vmovdqu ymm9, YMMWORD PTR [rsp+32] + vpermq ymm9, ymm9, 78 + vmovdqu ymm10, YMMWORD PTR [rsp] + vpermq ymm10, ymm10, 78 + vbroadcasti128 ymm6, ptr_L_vaes_aes_gcm_bswap_mask + vpxor ymm4, ymm4, ymm4 + vinserti128 ymm4, ymm4, xmm15, 0 + vmovdqu ymm5, YMMWORD PTR [rcx] + vpshufb ymm5, ymm5, ymm6 + vpxor ymm5, ymm5, ymm4 + vpclmulqdq ymm0, ymm5, ymm7, 0 + vpclmulqdq ymm1, ymm5, ymm7, 1 + vpclmulqdq ymm2, ymm5, ymm7, 16 + vpclmulqdq ymm3, ymm5, ymm7, 17 + vmovdqa ymm11, ymm0 + vpxor ymm12, ymm2, ymm1 + vmovdqa ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+32] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm8, 0 + vpclmulqdq ymm1, ymm5, ymm8, 1 + vpclmulqdq ymm2, ymm5, ymm8, 16 + vpclmulqdq ymm3, ymm5, ymm8, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+64] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm9, 0 + vpclmulqdq ymm1, ymm5, ymm9, 1 + vpclmulqdq ymm2, ymm5, ymm9, 16 + vpclmulqdq ymm3, ymm5, ymm9, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vmovdqu ymm5, YMMWORD PTR [rcx+96] + vpshufb ymm5, ymm5, ymm6 + vpclmulqdq ymm0, ymm5, ymm10, 0 + vpclmulqdq ymm1, ymm5, ymm10, 1 + vpclmulqdq ymm2, ymm5, ymm10, 16 + vpclmulqdq ymm3, ymm5, ymm10, 17 + vpxor ymm11, ymm11, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm12, ymm12, ymm2 + vpxor ymm13, ymm13, ymm3 + vpclmulqdq ymm5, ymm14, ymm11, 1 + vpshufd ymm11, ymm11, 78 + vpxor ymm12, ymm12, ymm5 + vpxor ymm12, ymm12, ymm11 + vpclmulqdq ymm5, ymm14, ymm12, 1 + vpshufd ymm12, ymm12, 78 + vpxor ymm13, ymm13, ymm5 + vpxor ymm13, ymm13, ymm12 + vextracti128 xmm0, ymm13, 1 + vpxor xmm15, xmm13, xmm0 + add edx, 128 +L_AES_GCM_aad_update_vaes_after_128: vmovdqu xmm6, OWORD PTR [rsp] - vmovdqu xmm7, OWORD PTR [rsp+16] - add rsp, 32 + mov eax, r9d + and eax, 4294967280 + cmp edx, eax + jge L_AES_GCM_aad_update_vaes_done +L_AES_GCM_aad_update_vaes_tail: + ; 16 bytes of input + vmovdqu xmm7, OWORD PTR [r8+rdx] + vpshufb xmm7, xmm7, OWORD PTR L_vaes_aes_gcm_bswap_mask + vpxor xmm15, xmm15, xmm7 + ; ghash_gfmul_red_avx + vpclmulqdq xmm7, xmm15, xmm6, 0 + vpclmulqdq xmm8, xmm15, xmm6, 1 + vpclmulqdq xmm9, xmm15, xmm6, 16 + vpclmulqdq xmm10, xmm15, xmm6, 17 + vpxor xmm8, xmm8, xmm9 + vmovdqa xmm9, OWORD PTR L_vaes_aes_gcm_mod2_128 + vpclmulqdq xmm11, xmm9, xmm7, 1 + vpshufd xmm7, xmm7, 78 + vpxor xmm8, xmm8, xmm11 + vpxor xmm8, xmm8, xmm7 + vpclmulqdq xmm11, xmm9, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpxor xmm10, xmm10, xmm11 + vpxor xmm10, xmm10, xmm8 + vmovdqa xmm15, xmm10 + add edx, 16 + cmp edx, eax + jl L_AES_GCM_aad_update_vaes_tail +L_AES_GCM_aad_update_vaes_done: + vmovdqa OWORD PTR [r10], xmm15 + vzeroupper + vmovdqu xmm6, OWORD PTR [rsp+512] + vmovdqu xmm7, OWORD PTR [rsp+528] + vmovdqu xmm8, OWORD PTR [rsp+544] + vmovdqu xmm9, OWORD PTR [rsp+560] + vmovdqu xmm10, OWORD PTR [rsp+576] + vmovdqu xmm11, OWORD PTR [rsp+592] + vmovdqu xmm12, OWORD PTR [rsp+608] + vmovdqu xmm13, OWORD PTR [rsp+624] + vmovdqu xmm14, OWORD PTR [rsp+640] + vmovdqu xmm15, OWORD PTR [rsp+656] + add rsp, 672 ret AES_GCM_aad_update_vaes ENDP _TEXT ENDS @@ -22719,17 +24585,17 @@ AES_GCM_encrypt_avx512 PROC mov r14d, DWORD PTR [rsp+128] mov r15, QWORD PTR [rsp+136] mov r10d, DWORD PTR [rsp+144] - sub rsp, 1248 - vmovdqu OWORD PTR [rsp+1088], xmm6 - vmovdqu OWORD PTR [rsp+1104], xmm7 - vmovdqu OWORD PTR [rsp+1120], xmm8 - vmovdqu OWORD PTR [rsp+1136], xmm9 - vmovdqu OWORD PTR [rsp+1152], xmm10 - vmovdqu OWORD PTR [rsp+1168], xmm11 - vmovdqu OWORD PTR [rsp+1184], xmm12 - vmovdqu OWORD PTR [rsp+1200], xmm13 - vmovdqu OWORD PTR [rsp+1216], xmm14 - vmovdqu OWORD PTR [rsp+1232], xmm15 + sub rsp, 1280 + vmovdqu OWORD PTR [rsp+1120], xmm6 + vmovdqu OWORD PTR [rsp+1136], xmm7 + vmovdqu OWORD PTR [rsp+1152], xmm8 + vmovdqu OWORD PTR [rsp+1168], xmm9 + vmovdqu OWORD PTR [rsp+1184], xmm10 + vmovdqu OWORD PTR [rsp+1200], xmm11 + vmovdqu OWORD PTR [rsp+1216], xmm12 + vmovdqu OWORD PTR [rsp+1232], xmm13 + vmovdqu OWORD PTR [rsp+1248], xmm14 + vmovdqu OWORD PTR [rsp+1264], xmm15 vpxor xmm4, xmm4, xmm4 vpxor xmm6, xmm6, xmm6 mov edx, ebx @@ -23023,9 +24889,426 @@ L_AES_GCM_encrypt_avx512_iv_done: cmp edx, 0 je L_AES_GCM_encrypt_avx512_calc_aad_done xor ecx, ecx - cmp edx, 16 - jl L_AES_GCM_encrypt_avx512_calc_aad_lt16 + cmp edx, 128 + jl L_AES_GCM_encrypt_avx512_calc_aad_ser + vmovdqu OWORD PTR [rsp+1088], xmm5 + vmovdqu OWORD PTR [rsp+1104], xmm4 + vpsrlq xmm9, xmm5, 63 + vpsllq xmm8, xmm5, 1 + vpslldq xmm9, xmm9, 8 + vpor xmm8, xmm8, xmm9 + vpshufd xmm5, xmm5, 255 + vpsrad xmm5, xmm5, 31 + vpand xmm5, xmm5, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpxor xmm5, xmm5, xmm8 + vmovdqa xmm2, xmm6 + ; H ^ 1 + vmovdqu OWORD PTR [rsp], xmm5 + ; H ^ 2 + vpclmulqdq xmm8, xmm5, xmm5, 0 + vpclmulqdq xmm11, xmm5, xmm5, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm0, xmm11 + vmovdqu OWORD PTR [rsp+16], xmm0 + ; H ^ 3 + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm5, 78 + vpxor xmm9, xmm9, xmm5 + vpshufd xmm10, xmm0, 78 + vpxor xmm10, xmm10, xmm0 + vpclmulqdq xmm8, xmm0, xmm5, 0 + vpclmulqdq xmm11, xmm0, xmm5, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm1, xmm11 + vmovdqu OWORD PTR [rsp+32], xmm1 + ; H ^ 4 + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm3, xmm11 + vmovdqu OWORD PTR [rsp+48], xmm3 + ; H ^ 5 + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+64], xmm7 + ; H ^ 6 + vpclmulqdq xmm8, xmm1, xmm1, 0 + vpclmulqdq xmm11, xmm1, xmm1, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+80], xmm7 + ; H ^ 7 + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm1, 78 + vpxor xmm9, xmm9, xmm1 + vpshufd xmm10, xmm3, 78 + vpxor xmm10, xmm10, xmm3 + vpclmulqdq xmm8, xmm3, xmm1, 0 + vpclmulqdq xmm11, xmm3, xmm1, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+96], xmm7 + ; H ^ 8 + vpclmulqdq xmm8, xmm3, xmm3, 0 + vpclmulqdq xmm11, xmm3, xmm3, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+112], xmm7 + ; H ^ 9 + vmovdqu xmm0, OWORD PTR [rsp+48] + vmovdqu xmm1, OWORD PTR [rsp+64] + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+128], xmm7 + ; H ^ 10 + vmovdqu xmm0, OWORD PTR [rsp+64] + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+144], xmm7 + ; H ^ 11 + vmovdqu xmm0, OWORD PTR [rsp+64] + vmovdqu xmm1, OWORD PTR [rsp+80] + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+160], xmm7 + ; H ^ 12 + vmovdqu xmm0, OWORD PTR [rsp+80] + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+176], xmm7 + ; H ^ 13 + vmovdqu xmm0, OWORD PTR [rsp+80] + vmovdqu xmm1, OWORD PTR [rsp+96] + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+192], xmm7 + ; H ^ 14 + vmovdqu xmm0, OWORD PTR [rsp+96] + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+208], xmm7 + ; H ^ 15 + vmovdqu xmm0, OWORD PTR [rsp+96] + vmovdqu xmm1, OWORD PTR [rsp+112] + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+224], xmm7 + ; H ^ 16 + vmovdqu xmm0, OWORD PTR [rsp+112] + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+240], xmm7 + vbroadcasti32x4 zmm30, ptr_L_avx512_aes_gcm_bswap_mask + vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 + vmovdqu64 zmm23, [rsp+192] + vshufi64x2 zmm23, zmm23, zmm23, 27 + vmovdqu64 zmm24, [rsp+128] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vmovdqu64 zmm25, [rsp+64] + vshufi64x2 zmm25, zmm25, zmm25, 27 + vmovdqu64 zmm26, [rsp] + vshufi64x2 zmm26, zmm26, zmm26, 27 + mov r13d, r11d + and r13d, 4294967040 + cmp ecx, r13d + jge L_AES_GCM_encrypt_avx512_calc_aad_a256 +L_AES_GCM_encrypt_avx512_calc_aad_l256: + ; 256 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rbx] + vpshufb zmm21, zmm21, zmm30 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm23, 0 + vpclmulqdq zmm17, zmm21, zmm23, 1 + vpclmulqdq zmm18, zmm21, zmm23, 16 + vpclmulqdq zmm19, zmm21, zmm23, 17 + vmovdqa64 zmm27, zmm16 + vpxorq zmm28, zmm18, zmm17 + vmovdqa64 zmm29, zmm19 + vmovdqu64 zmm21, [rbx+64] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vmovdqu64 zmm21, [rbx+128] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm25, 0 + vpclmulqdq zmm17, zmm21, zmm25, 1 + vpclmulqdq zmm18, zmm21, zmm25, 16 + vpclmulqdq zmm19, zmm21, zmm25, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vmovdqu64 zmm21, [rbx+192] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm26, 0 + vpclmulqdq zmm17, zmm21, zmm26, 1 + vpclmulqdq zmm18, zmm21, zmm26, 16 + vpclmulqdq zmm19, zmm21, zmm26, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vpclmulqdq zmm21, zmm31, zmm27, 1 + vpshufd zmm27, zmm27, 78 + vpternlogq zmm28, zmm27, zmm21, 150 + vpclmulqdq zmm21, zmm31, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vextracti32x4 xmm0, zmm29, 1 + vextracti32x4 xmm4, zmm29, 2 + vextracti32x4 xmm5, zmm29, 3 + vpxorq xmm6, xmm29, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + add ecx, 256 + cmp ecx, r13d + jl L_AES_GCM_encrypt_avx512_calc_aad_l256 +L_AES_GCM_encrypt_avx512_calc_aad_a256: + mov r13d, r11d + and r13d, 4294967168 + cmp ecx, r13d + jge L_AES_GCM_encrypt_avx512_calc_aad_a128 + ; 128 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu64 zmm23, [rsp+64] + vshufi64x2 zmm23, zmm23, zmm23, 27 + vmovdqu64 zmm24, [rsp] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rbx] + vpshufb zmm21, zmm21, zmm30 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm23, 0 + vpclmulqdq zmm17, zmm21, zmm23, 1 + vpclmulqdq zmm18, zmm21, zmm23, 16 + vpclmulqdq zmm19, zmm21, zmm23, 17 + vmovdqa64 zmm27, zmm16 + vpxorq zmm28, zmm18, zmm17 + vmovdqa64 zmm29, zmm19 + vmovdqu64 zmm21, [rbx+64] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 + vpclmulqdq zmm21, zmm31, zmm27, 1 + vpshufd zmm27, zmm27, 78 + vpternlogq zmm28, zmm27, zmm21, 150 + vpclmulqdq zmm21, zmm31, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vextracti32x4 xmm0, zmm29, 1 + vextracti32x4 xmm4, zmm29, 2 + vextracti32x4 xmm5, zmm29, 3 + vpxorq xmm6, xmm29, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + add ecx, 128 +L_AES_GCM_encrypt_avx512_calc_aad_a128: + mov r13d, r11d + and r13d, 4294967280 + cmp ecx, r13d + jge L_AES_GCM_encrypt_avx512_calc_aad_arem + vmovdqu xmm5, OWORD PTR [rsp] +L_AES_GCM_encrypt_avx512_calc_aad_rem: + ; 16 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu xmm8, OWORD PTR [rbx] + vpshufb xmm8, xmm8, OWORD PTR L_avx512_aes_gcm_bswap_mask + vpxor xmm6, xmm6, xmm8 + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm5, 78 + vpxor xmm9, xmm9, xmm5 + vpshufd xmm10, xmm6, 78 + vpxor xmm10, xmm10, xmm6 + vpclmulqdq xmm8, xmm6, xmm5, 0 + vpclmulqdq xmm11, xmm6, xmm5, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm6, xmm11 + add ecx, 16 + cmp ecx, r13d + jl L_AES_GCM_encrypt_avx512_calc_aad_rem +L_AES_GCM_encrypt_avx512_calc_aad_arem: + vmovdqu xmm5, OWORD PTR [rsp+1088] + vmovdqu xmm4, OWORD PTR [rsp+1104] + jmp L_AES_GCM_encrypt_avx512_calc_aad_partial +L_AES_GCM_encrypt_avx512_calc_aad_ser: + mov edx, r11d and edx, 4294967280 + cmp ecx, edx + je L_AES_GCM_encrypt_avx512_calc_aad_partial L_AES_GCM_encrypt_avx512_calc_aad_16_loop: vmovdqu xmm8, OWORD PTR [r12+rcx] vpshufb xmm8, xmm8, OWORD PTR L_avx512_aes_gcm_bswap_mask @@ -23076,6 +25359,7 @@ L_AES_GCM_encrypt_avx512_calc_aad_16_loop: add ecx, 16 cmp ecx, edx jl L_AES_GCM_encrypt_avx512_calc_aad_16_loop +L_AES_GCM_encrypt_avx512_calc_aad_partial: mov edx, r11d cmp ecx, edx je L_AES_GCM_encrypt_avx512_calc_aad_done @@ -23152,7 +25436,7 @@ L_AES_GCM_encrypt_avx512_calc_aad_done: vpxor xmm5, xmm5, xmm8 vmovdqu OWORD PTR [rsp+1024], xmm4 xor ebx, ebx - cmp r9d, 256 + cmp r9d, 128 jl L_AES_GCM_encrypt_avx512_done_128 vmovdqa xmm2, xmm6 ; H ^ 1 @@ -23266,6 +25550,8 @@ L_AES_GCM_encrypt_avx512_calc_aad_done: vpternlogq xmm11, xmm9, xmm12, 150 vmovdqa xmm7, xmm11 vmovdqu OWORD PTR [rsp+112], xmm7 + cmp r9d, 256 + jl L_AES_GCM_encrypt_avx512_no_ext2 ; H ^ 9 vmovdqu xmm0, OWORD PTR [rsp+48] vmovdqu xmm1, OWORD PTR [rsp+64] @@ -23689,6 +25975,7 @@ L_AES_GCM_encrypt_avx512_calc_aad_done: vmovdqa xmm7, xmm11 vmovdqu OWORD PTR [rsp+496], xmm7 L_AES_GCM_encrypt_avx512_no_ext: +L_AES_GCM_encrypt_avx512_no_ext2: vbroadcasti32x4 zmm22, ptr_L_avx512_aes_gcm_bswap_epi64 vbroadcasti32x4 zmm30, ptr_L_avx512_aes_gcm_bswap_mask vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 @@ -24633,6 +26920,116 @@ L_AES_GCM_encrypt_avx512_last_ghash: vpxorq xmm6, xmm29, xmm0 vpternlogq xmm6, xmm5, xmm4, 150 L_AES_GCM_encrypt_avx512_after_256: + mov r13d, r9d + and r13d, 4294967168 + cmp ebx, r13d + jge L_AES_GCM_encrypt_avx512_after_128 + ; 128 bytes of input + vbroadcasti32x4 zmm9, [r15] + vbroadcasti32x4 zmm10, [r15+16] + vbroadcasti32x4 zmm11, [r15+32] + vbroadcasti32x4 zmm12, [r15+48] + vbroadcasti32x4 zmm13, [r15+64] + vbroadcasti32x4 zmm14, [r15+80] + vbroadcasti32x4 zmm15, [r15+96] + vbroadcasti32x4 zmm1, [r15+112] + vbroadcasti32x4 zmm2, [r15+128] + vbroadcasti32x4 zmm3, [r15+144] + vbroadcasti32x4 zmm20, [rsp+1024] + vpaddd zmm16, zmm20, ptr_L_avx512_aes_gcm_inc_z0 + vpshufb zmm16, zmm16, zmm22 + vpaddd zmm17, zmm20, ptr_L_avx512_aes_gcm_inc_z1 + vpshufb zmm17, zmm17, zmm22 + vmovdqu xmm8, OWORD PTR [rsp+1024] + vpaddd xmm8, xmm8, OWORD PTR L_avx512_aes_gcm_eight + vmovdqu OWORD PTR [rsp+1024], xmm8 + vpxorq zmm16, zmm16, zmm9 + vpxorq zmm17, zmm17, zmm9 + vaesenc zmm16, zmm16, zmm10 + vaesenc zmm17, zmm17, zmm10 + vaesenc zmm16, zmm16, zmm11 + vaesenc zmm17, zmm17, zmm11 + vaesenc zmm16, zmm16, zmm12 + vaesenc zmm17, zmm17, zmm12 + vaesenc zmm16, zmm16, zmm13 + vaesenc zmm17, zmm17, zmm13 + vaesenc zmm16, zmm16, zmm14 + vaesenc zmm17, zmm17, zmm14 + vaesenc zmm16, zmm16, zmm15 + vaesenc zmm17, zmm17, zmm15 + vaesenc zmm16, zmm16, zmm1 + vaesenc zmm17, zmm17, zmm1 + vaesenc zmm16, zmm16, zmm2 + vaesenc zmm17, zmm17, zmm2 + vaesenc zmm16, zmm16, zmm3 + vaesenc zmm17, zmm17, zmm3 + cmp r10d, 11 + vbroadcasti32x4 zmm20, [r15+160] + jl L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [r15+176] + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + cmp r10d, 13 + vbroadcasti32x4 zmm20, [r15+192] + jl L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [r15+208] + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [r15+224] +L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last: + vaesenclast zmm16, zmm16, zmm20 + vaesenclast zmm17, zmm17, zmm20 + lea rcx, QWORD PTR [rdi+rbx] + lea rdx, QWORD PTR [rsi+rbx] + vmovdqu64 zmm21, [rcx] + vpxorq zmm16, zmm16, zmm21 + vmovdqu64 [rdx], zmm16 + vmovdqu64 zmm21, [rcx+64] + vpxorq zmm17, zmm17, zmm21 + vmovdqu64 [rdx+64], zmm17 + vmovdqu64 zmm23, [rsp+64] + vshufi64x2 zmm23, zmm23, zmm23, 27 + vmovdqu64 zmm24, [rsp] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rdx] + vpshufb zmm21, zmm21, zmm30 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm23, 0 + vpclmulqdq zmm17, zmm21, zmm23, 1 + vpclmulqdq zmm18, zmm21, zmm23, 16 + vpclmulqdq zmm19, zmm21, zmm23, 17 + vmovdqa64 zmm27, zmm16 + vpxorq zmm28, zmm18, zmm17 + vmovdqa64 zmm29, zmm19 + vmovdqu64 zmm21, [rdx+64] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 + vpclmulqdq zmm21, zmm31, zmm27, 1 + vpshufd zmm27, zmm27, 78 + vpternlogq zmm28, zmm27, zmm21, 150 + vpclmulqdq zmm21, zmm31, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vextracti32x4 xmm0, zmm29, 1 + vextracti32x4 xmm4, zmm29, 2 + vextracti32x4 xmm5, zmm29, 3 + vpxorq xmm6, xmm29, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + add ebx, 128 +L_AES_GCM_encrypt_avx512_after_128: vmovdqu xmm5, OWORD PTR [rsp] L_AES_GCM_encrypt_avx512_done_128: mov edx, r9d @@ -24866,17 +27263,17 @@ L_AES_GCM_encrypt_avx512_store_tag_16: vmovdqu OWORD PTR [r8], xmm0 L_AES_GCM_encrypt_avx512_store_tag_done: vzeroupper - vmovdqu xmm6, OWORD PTR [rsp+1088] - vmovdqu xmm7, OWORD PTR [rsp+1104] - vmovdqu xmm8, OWORD PTR [rsp+1120] - vmovdqu xmm9, OWORD PTR [rsp+1136] - vmovdqu xmm10, OWORD PTR [rsp+1152] - vmovdqu xmm11, OWORD PTR [rsp+1168] - vmovdqu xmm12, OWORD PTR [rsp+1184] - vmovdqu xmm13, OWORD PTR [rsp+1200] - vmovdqu xmm14, OWORD PTR [rsp+1216] - vmovdqu xmm15, OWORD PTR [rsp+1232] - add rsp, 1248 + vmovdqu xmm6, OWORD PTR [rsp+1120] + vmovdqu xmm7, OWORD PTR [rsp+1136] + vmovdqu xmm8, OWORD PTR [rsp+1152] + vmovdqu xmm9, OWORD PTR [rsp+1168] + vmovdqu xmm10, OWORD PTR [rsp+1184] + vmovdqu xmm11, OWORD PTR [rsp+1200] + vmovdqu xmm12, OWORD PTR [rsp+1216] + vmovdqu xmm13, OWORD PTR [rsp+1232] + vmovdqu xmm14, OWORD PTR [rsp+1248] + vmovdqu xmm15, OWORD PTR [rsp+1264] + add rsp, 1280 pop r15 pop r14 pop rbx @@ -24909,17 +27306,17 @@ AES_GCM_decrypt_avx512 PROC mov r15, QWORD PTR [rsp+144] mov r10d, DWORD PTR [rsp+152] mov rbp, QWORD PTR [rsp+160] - sub rsp, 1216 - vmovdqu OWORD PTR [rsp+1056], xmm6 - vmovdqu OWORD PTR [rsp+1072], xmm7 - vmovdqu OWORD PTR [rsp+1088], xmm8 - vmovdqu OWORD PTR [rsp+1104], xmm9 - vmovdqu OWORD PTR [rsp+1120], xmm10 - vmovdqu OWORD PTR [rsp+1136], xmm11 - vmovdqu OWORD PTR [rsp+1152], xmm12 - vmovdqu OWORD PTR [rsp+1168], xmm13 - vmovdqu OWORD PTR [rsp+1184], xmm14 - vmovdqu OWORD PTR [rsp+1200], xmm15 + sub rsp, 1280 + vmovdqu OWORD PTR [rsp+1120], xmm6 + vmovdqu OWORD PTR [rsp+1136], xmm7 + vmovdqu OWORD PTR [rsp+1152], xmm8 + vmovdqu OWORD PTR [rsp+1168], xmm9 + vmovdqu OWORD PTR [rsp+1184], xmm10 + vmovdqu OWORD PTR [rsp+1200], xmm11 + vmovdqu OWORD PTR [rsp+1216], xmm12 + vmovdqu OWORD PTR [rsp+1232], xmm13 + vmovdqu OWORD PTR [rsp+1248], xmm14 + vmovdqu OWORD PTR [rsp+1264], xmm15 vpxor xmm4, xmm4, xmm4 vpxor xmm6, xmm6, xmm6 cmp ebx, 12 @@ -25213,9 +27610,426 @@ L_AES_GCM_decrypt_avx512_iv_done: cmp edx, 0 je L_AES_GCM_decrypt_avx512_calc_aad_done xor ecx, ecx - cmp edx, 16 - jl L_AES_GCM_decrypt_avx512_calc_aad_lt16 + cmp edx, 128 + jl L_AES_GCM_decrypt_avx512_calc_aad_ser + vmovdqu OWORD PTR [rsp+1088], xmm5 + vmovdqu OWORD PTR [rsp+1104], xmm4 + vpsrlq xmm9, xmm5, 63 + vpsllq xmm8, xmm5, 1 + vpslldq xmm9, xmm9, 8 + vpor xmm8, xmm8, xmm9 + vpshufd xmm5, xmm5, 255 + vpsrad xmm5, xmm5, 31 + vpand xmm5, xmm5, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpxor xmm5, xmm5, xmm8 + vmovdqa xmm2, xmm6 + ; H ^ 1 + vmovdqu OWORD PTR [rsp], xmm5 + ; H ^ 2 + vpclmulqdq xmm8, xmm5, xmm5, 0 + vpclmulqdq xmm11, xmm5, xmm5, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm0, xmm11 + vmovdqu OWORD PTR [rsp+16], xmm0 + ; H ^ 3 + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm5, 78 + vpxor xmm9, xmm9, xmm5 + vpshufd xmm10, xmm0, 78 + vpxor xmm10, xmm10, xmm0 + vpclmulqdq xmm8, xmm0, xmm5, 0 + vpclmulqdq xmm11, xmm0, xmm5, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm1, xmm11 + vmovdqu OWORD PTR [rsp+32], xmm1 + ; H ^ 4 + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm3, xmm11 + vmovdqu OWORD PTR [rsp+48], xmm3 + ; H ^ 5 + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+64], xmm7 + ; H ^ 6 + vpclmulqdq xmm8, xmm1, xmm1, 0 + vpclmulqdq xmm11, xmm1, xmm1, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+80], xmm7 + ; H ^ 7 + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm1, 78 + vpxor xmm9, xmm9, xmm1 + vpshufd xmm10, xmm3, 78 + vpxor xmm10, xmm10, xmm3 + vpclmulqdq xmm8, xmm3, xmm1, 0 + vpclmulqdq xmm11, xmm3, xmm1, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+96], xmm7 + ; H ^ 8 + vpclmulqdq xmm8, xmm3, xmm3, 0 + vpclmulqdq xmm11, xmm3, xmm3, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+112], xmm7 + ; H ^ 9 + vmovdqu xmm0, OWORD PTR [rsp+48] + vmovdqu xmm1, OWORD PTR [rsp+64] + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+128], xmm7 + ; H ^ 10 + vmovdqu xmm0, OWORD PTR [rsp+64] + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+144], xmm7 + ; H ^ 11 + vmovdqu xmm0, OWORD PTR [rsp+64] + vmovdqu xmm1, OWORD PTR [rsp+80] + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+160], xmm7 + ; H ^ 12 + vmovdqu xmm0, OWORD PTR [rsp+80] + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+176], xmm7 + ; H ^ 13 + vmovdqu xmm0, OWORD PTR [rsp+80] + vmovdqu xmm1, OWORD PTR [rsp+96] + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+192], xmm7 + ; H ^ 14 + vmovdqu xmm0, OWORD PTR [rsp+96] + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+208], xmm7 + ; H ^ 15 + vmovdqu xmm0, OWORD PTR [rsp+96] + vmovdqu xmm1, OWORD PTR [rsp+112] + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm0, 78 + vpxor xmm9, xmm9, xmm0 + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpclmulqdq xmm8, xmm1, xmm0, 0 + vpclmulqdq xmm11, xmm1, xmm0, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+224], xmm7 + ; H ^ 16 + vmovdqu xmm0, OWORD PTR [rsp+112] + vpclmulqdq xmm8, xmm0, xmm0, 0 + vpclmulqdq xmm11, xmm0, xmm0, 17 + vpxor xmm9, xmm9, xmm9 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm7, xmm11 + vmovdqu OWORD PTR [rsp+240], xmm7 + vbroadcasti32x4 zmm30, ptr_L_avx512_aes_gcm_bswap_mask + vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 + vmovdqu64 zmm23, [rsp+192] + vshufi64x2 zmm23, zmm23, zmm23, 27 + vmovdqu64 zmm24, [rsp+128] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vmovdqu64 zmm25, [rsp+64] + vshufi64x2 zmm25, zmm25, zmm25, 27 + vmovdqu64 zmm26, [rsp] + vshufi64x2 zmm26, zmm26, zmm26, 27 + mov r13d, r11d + and r13d, 4294967040 + cmp ecx, r13d + jge L_AES_GCM_decrypt_avx512_calc_aad_a256 +L_AES_GCM_decrypt_avx512_calc_aad_l256: + ; 256 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rbx] + vpshufb zmm21, zmm21, zmm30 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm23, 0 + vpclmulqdq zmm17, zmm21, zmm23, 1 + vpclmulqdq zmm18, zmm21, zmm23, 16 + vpclmulqdq zmm19, zmm21, zmm23, 17 + vmovdqa64 zmm27, zmm16 + vpxorq zmm28, zmm18, zmm17 + vmovdqa64 zmm29, zmm19 + vmovdqu64 zmm21, [rbx+64] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vmovdqu64 zmm21, [rbx+128] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm25, 0 + vpclmulqdq zmm17, zmm21, zmm25, 1 + vpclmulqdq zmm18, zmm21, zmm25, 16 + vpclmulqdq zmm19, zmm21, zmm25, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vmovdqu64 zmm21, [rbx+192] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm26, 0 + vpclmulqdq zmm17, zmm21, zmm26, 1 + vpclmulqdq zmm18, zmm21, zmm26, 16 + vpclmulqdq zmm19, zmm21, zmm26, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vpclmulqdq zmm21, zmm31, zmm27, 1 + vpshufd zmm27, zmm27, 78 + vpternlogq zmm28, zmm27, zmm21, 150 + vpclmulqdq zmm21, zmm31, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vextracti32x4 xmm0, zmm29, 1 + vextracti32x4 xmm4, zmm29, 2 + vextracti32x4 xmm5, zmm29, 3 + vpxorq xmm6, xmm29, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + add ecx, 256 + cmp ecx, r13d + jl L_AES_GCM_decrypt_avx512_calc_aad_l256 +L_AES_GCM_decrypt_avx512_calc_aad_a256: + mov r13d, r11d + and r13d, 4294967168 + cmp ecx, r13d + jge L_AES_GCM_decrypt_avx512_calc_aad_a128 + ; 128 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu64 zmm23, [rsp+64] + vshufi64x2 zmm23, zmm23, zmm23, 27 + vmovdqu64 zmm24, [rsp] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rbx] + vpshufb zmm21, zmm21, zmm30 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm23, 0 + vpclmulqdq zmm17, zmm21, zmm23, 1 + vpclmulqdq zmm18, zmm21, zmm23, 16 + vpclmulqdq zmm19, zmm21, zmm23, 17 + vmovdqa64 zmm27, zmm16 + vpxorq zmm28, zmm18, zmm17 + vmovdqa64 zmm29, zmm19 + vmovdqu64 zmm21, [rbx+64] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 + vpclmulqdq zmm21, zmm31, zmm27, 1 + vpshufd zmm27, zmm27, 78 + vpternlogq zmm28, zmm27, zmm21, 150 + vpclmulqdq zmm21, zmm31, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vextracti32x4 xmm0, zmm29, 1 + vextracti32x4 xmm4, zmm29, 2 + vextracti32x4 xmm5, zmm29, 3 + vpxorq xmm6, xmm29, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + add ecx, 128 +L_AES_GCM_decrypt_avx512_calc_aad_a128: + mov r13d, r11d + and r13d, 4294967280 + cmp ecx, r13d + jge L_AES_GCM_decrypt_avx512_calc_aad_arem + vmovdqu xmm5, OWORD PTR [rsp] +L_AES_GCM_decrypt_avx512_calc_aad_rem: + ; 16 bytes of AAD + lea rbx, QWORD PTR [r12+rcx] + vmovdqu xmm8, OWORD PTR [rbx] + vpshufb xmm8, xmm8, OWORD PTR L_avx512_aes_gcm_bswap_mask + vpxor xmm6, xmm6, xmm8 + ; ghash_gfmul_red_avx + vpshufd xmm9, xmm5, 78 + vpxor xmm9, xmm9, xmm5 + vpshufd xmm10, xmm6, 78 + vpxor xmm10, xmm10, xmm6 + vpclmulqdq xmm8, xmm6, xmm5, 0 + vpclmulqdq xmm11, xmm6, xmm5, 17 + vpclmulqdq xmm9, xmm9, xmm10, 0 + vpternlogq xmm9, xmm11, xmm8, 150 + vmovdqa xmm10, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm12, xmm10, xmm8, 1 + vpshufd xmm8, xmm8, 78 + vpternlogq xmm9, xmm8, xmm12, 150 + vpclmulqdq xmm12, xmm10, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm11, xmm9, xmm12, 150 + vmovdqa xmm6, xmm11 + add ecx, 16 + cmp ecx, r13d + jl L_AES_GCM_decrypt_avx512_calc_aad_rem +L_AES_GCM_decrypt_avx512_calc_aad_arem: + vmovdqu xmm5, OWORD PTR [rsp+1088] + vmovdqu xmm4, OWORD PTR [rsp+1104] + jmp L_AES_GCM_decrypt_avx512_calc_aad_partial +L_AES_GCM_decrypt_avx512_calc_aad_ser: + mov edx, r11d and edx, 4294967280 + cmp ecx, edx + je L_AES_GCM_decrypt_avx512_calc_aad_partial L_AES_GCM_decrypt_avx512_calc_aad_16_loop: vmovdqu xmm8, OWORD PTR [r12+rcx] vpshufb xmm8, xmm8, OWORD PTR L_avx512_aes_gcm_bswap_mask @@ -25266,6 +28080,7 @@ L_AES_GCM_decrypt_avx512_calc_aad_16_loop: add ecx, 16 cmp ecx, edx jl L_AES_GCM_decrypt_avx512_calc_aad_16_loop +L_AES_GCM_decrypt_avx512_calc_aad_partial: mov edx, r11d cmp ecx, edx je L_AES_GCM_decrypt_avx512_calc_aad_done @@ -25342,7 +28157,7 @@ L_AES_GCM_decrypt_avx512_calc_aad_done: vpxor xmm5, xmm5, xmm8 vmovdqu OWORD PTR [rsp+1024], xmm4 xor ebx, ebx - cmp r9d, 256 + cmp r9d, 128 jl L_AES_GCM_decrypt_avx512_done_128 vmovdqa xmm2, xmm6 ; H ^ 1 @@ -25456,6 +28271,8 @@ L_AES_GCM_decrypt_avx512_calc_aad_done: vpternlogq xmm11, xmm9, xmm12, 150 vmovdqa xmm7, xmm11 vmovdqu OWORD PTR [rsp+112], xmm7 + cmp r9d, 256 + jl L_AES_GCM_decrypt_avx512_no_ext2 ; H ^ 9 vmovdqu xmm0, OWORD PTR [rsp+48] vmovdqu xmm1, OWORD PTR [rsp+64] @@ -25879,6 +28696,7 @@ L_AES_GCM_decrypt_avx512_calc_aad_done: vmovdqa xmm7, xmm11 vmovdqu OWORD PTR [rsp+496], xmm7 L_AES_GCM_decrypt_avx512_no_ext: +L_AES_GCM_decrypt_avx512_no_ext2: vbroadcasti32x4 zmm22, ptr_L_avx512_aes_gcm_bswap_epi64 vbroadcasti32x4 zmm30, ptr_L_avx512_aes_gcm_bswap_mask vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 @@ -26664,6 +29482,117 @@ L_AES_GCM_decrypt_avx512_t_avx512_ctr16_last: vmovdqu64 [rdx+192], zmm19 add ebx, 256 L_AES_GCM_decrypt_avx512_after_256: + mov r13d, r9d + and r13d, 4294967168 + cmp ebx, r13d + jge L_AES_GCM_decrypt_avx512_after_128 + ; 128 bytes of input + lea rax, QWORD PTR [rdi+rbx] + vmovdqu64 zmm23, [rsp+64] + vshufi64x2 zmm23, zmm23, zmm23, 27 + vmovdqu64 zmm24, [rsp] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rax] + vpshufb zmm21, zmm21, zmm30 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm23, 0 + vpclmulqdq zmm17, zmm21, zmm23, 1 + vpclmulqdq zmm18, zmm21, zmm23, 16 + vpclmulqdq zmm19, zmm21, zmm23, 17 + vmovdqa64 zmm27, zmm16 + vpxorq zmm28, zmm18, zmm17 + vmovdqa64 zmm29, zmm19 + vmovdqu64 zmm21, [rax+64] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 + vpclmulqdq zmm21, zmm31, zmm27, 1 + vpshufd zmm27, zmm27, 78 + vpternlogq zmm28, zmm27, zmm21, 150 + vpclmulqdq zmm21, zmm31, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vextracti32x4 xmm0, zmm29, 1 + vextracti32x4 xmm4, zmm29, 2 + vextracti32x4 xmm5, zmm29, 3 + vpxorq xmm6, xmm29, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + vbroadcasti32x4 zmm9, [r15] + vbroadcasti32x4 zmm10, [r15+16] + vbroadcasti32x4 zmm11, [r15+32] + vbroadcasti32x4 zmm12, [r15+48] + vbroadcasti32x4 zmm13, [r15+64] + vbroadcasti32x4 zmm14, [r15+80] + vbroadcasti32x4 zmm15, [r15+96] + vbroadcasti32x4 zmm1, [r15+112] + vbroadcasti32x4 zmm2, [r15+128] + vbroadcasti32x4 zmm3, [r15+144] + vbroadcasti32x4 zmm20, [rsp+1024] + vpaddd zmm16, zmm20, ptr_L_avx512_aes_gcm_inc_z0 + vpshufb zmm16, zmm16, zmm22 + vpaddd zmm17, zmm20, ptr_L_avx512_aes_gcm_inc_z1 + vpshufb zmm17, zmm17, zmm22 + vmovdqu xmm8, OWORD PTR [rsp+1024] + vpaddd xmm8, xmm8, OWORD PTR L_avx512_aes_gcm_eight + vmovdqu OWORD PTR [rsp+1024], xmm8 + vpxorq zmm16, zmm16, zmm9 + vpxorq zmm17, zmm17, zmm9 + vaesenc zmm16, zmm16, zmm10 + vaesenc zmm17, zmm17, zmm10 + vaesenc zmm16, zmm16, zmm11 + vaesenc zmm17, zmm17, zmm11 + vaesenc zmm16, zmm16, zmm12 + vaesenc zmm17, zmm17, zmm12 + vaesenc zmm16, zmm16, zmm13 + vaesenc zmm17, zmm17, zmm13 + vaesenc zmm16, zmm16, zmm14 + vaesenc zmm17, zmm17, zmm14 + vaesenc zmm16, zmm16, zmm15 + vaesenc zmm17, zmm17, zmm15 + vaesenc zmm16, zmm16, zmm1 + vaesenc zmm17, zmm17, zmm1 + vaesenc zmm16, zmm16, zmm2 + vaesenc zmm17, zmm17, zmm2 + vaesenc zmm16, zmm16, zmm3 + vaesenc zmm17, zmm17, zmm3 + cmp r10d, 11 + vbroadcasti32x4 zmm20, [r15+160] + jl L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [r15+176] + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + cmp r10d, 13 + vbroadcasti32x4 zmm20, [r15+192] + jl L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [r15+208] + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [r15+224] +L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last: + vaesenclast zmm16, zmm16, zmm20 + vaesenclast zmm17, zmm17, zmm20 + lea rcx, QWORD PTR [rdi+rbx] + lea rdx, QWORD PTR [rsi+rbx] + vmovdqu64 zmm21, [rcx] + vpxorq zmm16, zmm16, zmm21 + vmovdqu64 [rdx], zmm16 + vmovdqu64 zmm21, [rcx+64] + vpxorq zmm17, zmm17, zmm21 + vmovdqu64 [rdx+64], zmm17 + add ebx, 128 +L_AES_GCM_decrypt_avx512_after_128: vmovdqu xmm5, OWORD PTR [rsp] L_AES_GCM_decrypt_avx512_done_128: mov edx, r9d @@ -26852,17 +29781,17 @@ L_AES_GCM_decrypt_avx512_cmp_tag_16: L_AES_GCM_decrypt_avx512_cmp_tag_done: mov DWORD PTR [rbp], ebx vzeroupper - vmovdqu xmm6, OWORD PTR [rsp+1056] - vmovdqu xmm7, OWORD PTR [rsp+1072] - vmovdqu xmm8, OWORD PTR [rsp+1088] - vmovdqu xmm9, OWORD PTR [rsp+1104] - vmovdqu xmm10, OWORD PTR [rsp+1120] - vmovdqu xmm11, OWORD PTR [rsp+1136] - vmovdqu xmm12, OWORD PTR [rsp+1152] - vmovdqu xmm13, OWORD PTR [rsp+1168] - vmovdqu xmm14, OWORD PTR [rsp+1184] - vmovdqu xmm15, OWORD PTR [rsp+1200] - add rsp, 1216 + vmovdqu xmm6, OWORD PTR [rsp+1120] + vmovdqu xmm7, OWORD PTR [rsp+1136] + vmovdqu xmm8, OWORD PTR [rsp+1152] + vmovdqu xmm9, OWORD PTR [rsp+1168] + vmovdqu xmm10, OWORD PTR [rsp+1184] + vmovdqu xmm11, OWORD PTR [rsp+1200] + vmovdqu xmm12, OWORD PTR [rsp+1216] + vmovdqu xmm13, OWORD PTR [rsp+1232] + vmovdqu xmm14, OWORD PTR [rsp+1248] + vmovdqu xmm15, OWORD PTR [rsp+1264] + add rsp, 1280 pop rbp pop r15 pop r14 @@ -27198,67 +30127,444 @@ AES_GCM_init_avx512 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA AES_GCM_aad_update_avx512 PROC - mov rax, rcx - sub rsp, 32 - vmovdqu OWORD PTR [rsp], xmm6 - vmovdqu OWORD PTR [rsp+16], xmm7 - vmovdqa xmm5, OWORD PTR [r8] - vmovdqa xmm6, OWORD PTR [r9] - xor ecx, ecx -L_AES_GCM_aad_update_avx512_16_loop: - vmovdqu xmm7, OWORD PTR [rax+rcx] - vpshufb xmm7, xmm7, OWORD PTR L_avx512_aes_gcm_bswap_mask - vpxor xmm5, xmm5, xmm7 - ; ghash_gfmul_avx - vpshufd xmm1, xmm5, 78 - vpshufd xmm2, xmm6, 78 - vpclmulqdq xmm3, xmm6, xmm5, 17 - vpclmulqdq xmm0, xmm6, xmm5, 0 - vpxor xmm1, xmm1, xmm5 - vpxor xmm2, xmm2, xmm6 - vpclmulqdq xmm1, xmm1, xmm2, 0 - vpxor xmm1, xmm1, xmm0 - vpxor xmm1, xmm1, xmm3 - vmovdqa xmm4, xmm0 - vmovdqa xmm5, xmm3 - vpslldq xmm2, xmm1, 8 - vpsrldq xmm1, xmm1, 8 - vpxor xmm4, xmm4, xmm2 - vpxor xmm5, xmm5, xmm1 - vpsrld xmm0, xmm4, 31 - vpsrld xmm1, xmm5, 31 - vpslld xmm4, xmm4, 1 - vpslld xmm5, xmm5, 1 - vpsrldq xmm2, xmm0, 12 - vpslldq xmm0, xmm0, 4 - vpslldq xmm1, xmm1, 4 - vpor xmm5, xmm5, xmm2 - vpor xmm4, xmm4, xmm0 - vpor xmm5, xmm5, xmm1 - vpslld xmm0, xmm4, 31 - vpslld xmm1, xmm4, 30 - vpslld xmm2, xmm4, 25 - vpxor xmm0, xmm0, xmm1 - vpxor xmm0, xmm0, xmm2 - vmovdqa xmm1, xmm0 - vpsrldq xmm1, xmm1, 4 - vpslldq xmm0, xmm0, 12 - vpxor xmm4, xmm4, xmm0 - vpsrld xmm2, xmm4, 1 - vpsrld xmm3, xmm4, 2 - vpsrld xmm0, xmm4, 7 - vpxor xmm2, xmm2, xmm3 - vpxor xmm2, xmm2, xmm0 - vpxor xmm2, xmm2, xmm1 - vpxor xmm2, xmm2, xmm4 - vpxor xmm5, xmm5, xmm2 - add ecx, 16 - cmp ecx, edx - jl L_AES_GCM_aad_update_avx512_16_loop - vmovdqa OWORD PTR [r8], xmm5 - vmovdqu xmm6, OWORD PTR [rsp] - vmovdqu xmm7, OWORD PTR [rsp+16] - add rsp, 32 + mov r10, r8 + mov r8, rcx + mov r11, r9 + mov r9d, edx + sub rsp, 416 + vmovdqu OWORD PTR [rsp+256], xmm6 + vmovdqu OWORD PTR [rsp+272], xmm7 + vmovdqu OWORD PTR [rsp+288], xmm8 + vmovdqu OWORD PTR [rsp+304], xmm9 + vmovdqu OWORD PTR [rsp+320], xmm10 + vmovdqu OWORD PTR [rsp+336], xmm11 + vmovdqu OWORD PTR [rsp+352], xmm12 + vmovdqu OWORD PTR [rsp+368], xmm13 + vmovdqu OWORD PTR [rsp+384], xmm14 + vmovdqu OWORD PTR [rsp+400], xmm15 + vmovdqa xmm6, OWORD PTR [r10] + vmovdqa xmm7, OWORD PTR [r11] + xor edx, edx + vpsrlq xmm10, xmm7, 63 + vpsllq xmm9, xmm7, 1 + vpslldq xmm10, xmm10, 8 + vpor xmm9, xmm9, xmm10 + vpshufd xmm7, xmm7, 255 + vpsrad xmm7, xmm7, 31 + vpand xmm7, xmm7, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpxor xmm7, xmm7, xmm9 + vmovdqa xmm2, xmm6 + ; H ^ 1 + vmovdqu OWORD PTR [rsp], xmm7 + ; H ^ 2 + vpclmulqdq xmm9, xmm7, xmm7, 0 + vpclmulqdq xmm12, xmm7, xmm7, 17 + vpxor xmm10, xmm10, xmm10 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm0, xmm12 + vmovdqu OWORD PTR [rsp+16], xmm0 + ; H ^ 3 + ; ghash_gfmul_red_avx + vpshufd xmm10, xmm7, 78 + vpxor xmm10, xmm10, xmm7 + vpshufd xmm11, xmm0, 78 + vpxor xmm11, xmm11, xmm0 + vpclmulqdq xmm9, xmm0, xmm7, 0 + vpclmulqdq xmm12, xmm0, xmm7, 17 + vpclmulqdq xmm10, xmm10, xmm11, 0 + vpternlogq xmm10, xmm12, xmm9, 150 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm1, xmm12 + vmovdqu OWORD PTR [rsp+32], xmm1 + ; H ^ 4 + vpclmulqdq xmm9, xmm0, xmm0, 0 + vpclmulqdq xmm12, xmm0, xmm0, 17 + vpxor xmm10, xmm10, xmm10 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm3, xmm12 + vmovdqu OWORD PTR [rsp+48], xmm3 + ; H ^ 5 + ; ghash_gfmul_red_avx + vpshufd xmm10, xmm0, 78 + vpxor xmm10, xmm10, xmm0 + vpshufd xmm11, xmm1, 78 + vpxor xmm11, xmm11, xmm1 + vpclmulqdq xmm9, xmm1, xmm0, 0 + vpclmulqdq xmm12, xmm1, xmm0, 17 + vpclmulqdq xmm10, xmm10, xmm11, 0 + vpternlogq xmm10, xmm12, xmm9, 150 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+64], xmm8 + ; H ^ 6 + vpclmulqdq xmm9, xmm1, xmm1, 0 + vpclmulqdq xmm12, xmm1, xmm1, 17 + vpxor xmm10, xmm10, xmm10 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+80], xmm8 + ; H ^ 7 + ; ghash_gfmul_red_avx + vpshufd xmm10, xmm1, 78 + vpxor xmm10, xmm10, xmm1 + vpshufd xmm11, xmm3, 78 + vpxor xmm11, xmm11, xmm3 + vpclmulqdq xmm9, xmm3, xmm1, 0 + vpclmulqdq xmm12, xmm3, xmm1, 17 + vpclmulqdq xmm10, xmm10, xmm11, 0 + vpternlogq xmm10, xmm12, xmm9, 150 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+96], xmm8 + ; H ^ 8 + vpclmulqdq xmm9, xmm3, xmm3, 0 + vpclmulqdq xmm12, xmm3, xmm3, 17 + vpxor xmm10, xmm10, xmm10 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+112], xmm8 + ; H ^ 9 + vmovdqu xmm0, OWORD PTR [rsp+48] + vmovdqu xmm1, OWORD PTR [rsp+64] + ; ghash_gfmul_red_avx + vpshufd xmm10, xmm0, 78 + vpxor xmm10, xmm10, xmm0 + vpshufd xmm11, xmm1, 78 + vpxor xmm11, xmm11, xmm1 + vpclmulqdq xmm9, xmm1, xmm0, 0 + vpclmulqdq xmm12, xmm1, xmm0, 17 + vpclmulqdq xmm10, xmm10, xmm11, 0 + vpternlogq xmm10, xmm12, xmm9, 150 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+128], xmm8 + ; H ^ 10 + vmovdqu xmm0, OWORD PTR [rsp+64] + vpclmulqdq xmm9, xmm0, xmm0, 0 + vpclmulqdq xmm12, xmm0, xmm0, 17 + vpxor xmm10, xmm10, xmm10 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+144], xmm8 + ; H ^ 11 + vmovdqu xmm0, OWORD PTR [rsp+64] + vmovdqu xmm1, OWORD PTR [rsp+80] + ; ghash_gfmul_red_avx + vpshufd xmm10, xmm0, 78 + vpxor xmm10, xmm10, xmm0 + vpshufd xmm11, xmm1, 78 + vpxor xmm11, xmm11, xmm1 + vpclmulqdq xmm9, xmm1, xmm0, 0 + vpclmulqdq xmm12, xmm1, xmm0, 17 + vpclmulqdq xmm10, xmm10, xmm11, 0 + vpternlogq xmm10, xmm12, xmm9, 150 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+160], xmm8 + ; H ^ 12 + vmovdqu xmm0, OWORD PTR [rsp+80] + vpclmulqdq xmm9, xmm0, xmm0, 0 + vpclmulqdq xmm12, xmm0, xmm0, 17 + vpxor xmm10, xmm10, xmm10 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+176], xmm8 + ; H ^ 13 + vmovdqu xmm0, OWORD PTR [rsp+80] + vmovdqu xmm1, OWORD PTR [rsp+96] + ; ghash_gfmul_red_avx + vpshufd xmm10, xmm0, 78 + vpxor xmm10, xmm10, xmm0 + vpshufd xmm11, xmm1, 78 + vpxor xmm11, xmm11, xmm1 + vpclmulqdq xmm9, xmm1, xmm0, 0 + vpclmulqdq xmm12, xmm1, xmm0, 17 + vpclmulqdq xmm10, xmm10, xmm11, 0 + vpternlogq xmm10, xmm12, xmm9, 150 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+192], xmm8 + ; H ^ 14 + vmovdqu xmm0, OWORD PTR [rsp+96] + vpclmulqdq xmm9, xmm0, xmm0, 0 + vpclmulqdq xmm12, xmm0, xmm0, 17 + vpxor xmm10, xmm10, xmm10 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+208], xmm8 + ; H ^ 15 + vmovdqu xmm0, OWORD PTR [rsp+96] + vmovdqu xmm1, OWORD PTR [rsp+112] + ; ghash_gfmul_red_avx + vpshufd xmm10, xmm0, 78 + vpxor xmm10, xmm10, xmm0 + vpshufd xmm11, xmm1, 78 + vpxor xmm11, xmm11, xmm1 + vpclmulqdq xmm9, xmm1, xmm0, 0 + vpclmulqdq xmm12, xmm1, xmm0, 17 + vpclmulqdq xmm10, xmm10, xmm11, 0 + vpternlogq xmm10, xmm12, xmm9, 150 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+224], xmm8 + ; H ^ 16 + vmovdqu xmm0, OWORD PTR [rsp+112] + vpclmulqdq xmm9, xmm0, xmm0, 0 + vpclmulqdq xmm12, xmm0, xmm0, 17 + vpxor xmm10, xmm10, xmm10 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm8, xmm12 + vmovdqu OWORD PTR [rsp+240], xmm8 + vbroadcasti32x4 zmm22, ptr_L_avx512_aes_gcm_bswap_mask + vbroadcasti32x4 zmm23, ptr_L_avx512_aes_gcm_mod2_128 + vmovdqu64 zmm24, [rsp+192] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vmovdqu64 zmm25, [rsp+128] + vshufi64x2 zmm25, zmm25, zmm25, 27 + vmovdqu64 zmm26, [rsp+64] + vshufi64x2 zmm26, zmm26, zmm26, 27 + vmovdqu64 zmm27, [rsp] + vshufi64x2 zmm27, zmm27, zmm27, 27 + mov eax, r9d + and eax, 4294967040 + cmp edx, eax + jge L_AES_GCM_aad_update_avx512_after_256 +L_AES_GCM_aad_update_avx512_loop_256: + ; 256 bytes of input + lea rcx, QWORD PTR [r8+rdx] + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rcx] + vpshufb zmm21, zmm21, zmm22 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vmovdqa64 zmm28, zmm16 + vpxorq zmm29, zmm18, zmm17 + vmovdqa64 zmm30, zmm19 + vmovdqu64 zmm21, [rcx+64] + vpshufb zmm21, zmm21, zmm22 + vpclmulqdq zmm16, zmm21, zmm25, 0 + vpclmulqdq zmm17, zmm21, zmm25, 1 + vpclmulqdq zmm18, zmm21, zmm25, 16 + vpclmulqdq zmm19, zmm21, zmm25, 17 + vpxorq zmm28, zmm28, zmm16 + vpternlogq zmm29, zmm18, zmm17, 150 + vpxorq zmm30, zmm30, zmm19 + vmovdqu64 zmm21, [rcx+128] + vpshufb zmm21, zmm21, zmm22 + vpclmulqdq zmm16, zmm21, zmm26, 0 + vpclmulqdq zmm17, zmm21, zmm26, 1 + vpclmulqdq zmm18, zmm21, zmm26, 16 + vpclmulqdq zmm19, zmm21, zmm26, 17 + vpxorq zmm28, zmm28, zmm16 + vpternlogq zmm29, zmm18, zmm17, 150 + vpxorq zmm30, zmm30, zmm19 + vmovdqu64 zmm21, [rcx+192] + vpshufb zmm21, zmm21, zmm22 + vpclmulqdq zmm16, zmm21, zmm27, 0 + vpclmulqdq zmm17, zmm21, zmm27, 1 + vpclmulqdq zmm18, zmm21, zmm27, 16 + vpclmulqdq zmm19, zmm21, zmm27, 17 + vpxorq zmm28, zmm28, zmm16 + vpternlogq zmm29, zmm18, zmm17, 150 + vpxorq zmm30, zmm30, zmm19 + vpclmulqdq zmm21, zmm23, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vpclmulqdq zmm21, zmm23, zmm29, 1 + vpshufd zmm29, zmm29, 78 + vpternlogq zmm30, zmm29, zmm21, 150 + vextracti32x4 xmm0, zmm30, 1 + vextracti32x4 xmm4, zmm30, 2 + vextracti32x4 xmm5, zmm30, 3 + vpxorq xmm6, xmm30, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + add edx, 256 + cmp edx, eax + jl L_AES_GCM_aad_update_avx512_loop_256 +L_AES_GCM_aad_update_avx512_after_256: + mov eax, r9d + and eax, 4294967168 + cmp edx, eax + jge L_AES_GCM_aad_update_avx512_after_128 + ; 128 bytes of input + lea rcx, QWORD PTR [r8+rdx] + vmovdqu64 zmm24, [rsp+64] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vmovdqu64 zmm25, [rsp] + vshufi64x2 zmm25, zmm25, zmm25, 27 + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rcx] + vpshufb zmm21, zmm21, zmm22 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vmovdqa64 zmm28, zmm16 + vpxorq zmm29, zmm18, zmm17 + vmovdqa64 zmm30, zmm19 + vmovdqu64 zmm21, [rcx+64] + vpshufb zmm21, zmm21, zmm22 + vpclmulqdq zmm16, zmm21, zmm25, 0 + vpclmulqdq zmm17, zmm21, zmm25, 1 + vpclmulqdq zmm18, zmm21, zmm25, 16 + vpclmulqdq zmm19, zmm21, zmm25, 17 + vpxorq zmm28, zmm28, zmm16 + vpternlogq zmm29, zmm18, zmm17, 150 + vpxorq zmm30, zmm30, zmm19 + vbroadcasti32x4 zmm23, ptr_L_avx512_aes_gcm_mod2_128 + vpclmulqdq zmm21, zmm23, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vpclmulqdq zmm21, zmm23, zmm29, 1 + vpshufd zmm29, zmm29, 78 + vpternlogq zmm30, zmm29, zmm21, 150 + vextracti32x4 xmm0, zmm30, 1 + vextracti32x4 xmm4, zmm30, 2 + vextracti32x4 xmm5, zmm30, 3 + vpxorq xmm6, xmm30, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + add edx, 128 +L_AES_GCM_aad_update_avx512_after_128: + mov eax, r9d + and eax, 4294967280 + cmp edx, eax + jge L_AES_GCM_aad_update_avx512_done + vmovdqu xmm7, OWORD PTR [rsp] +L_AES_GCM_aad_update_avx512_tail: + ; 16 bytes of input + vmovdqu xmm9, OWORD PTR [r8+rdx] + vpshufb xmm9, xmm9, OWORD PTR L_avx512_aes_gcm_bswap_mask + vpxor xmm6, xmm6, xmm9 + ; ghash_gfmul_red_avx + vpshufd xmm10, xmm7, 78 + vpxor xmm10, xmm10, xmm7 + vpshufd xmm11, xmm6, 78 + vpxor xmm11, xmm11, xmm6 + vpclmulqdq xmm9, xmm6, xmm7, 0 + vpclmulqdq xmm12, xmm6, xmm7, 17 + vpclmulqdq xmm10, xmm10, xmm11, 0 + vpternlogq xmm10, xmm12, xmm9, 150 + vmovdqa xmm11, OWORD PTR L_avx512_aes_gcm_mod2_128 + vpclmulqdq xmm13, xmm11, xmm9, 1 + vpshufd xmm9, xmm9, 78 + vpternlogq xmm10, xmm9, xmm13, 150 + vpclmulqdq xmm13, xmm11, xmm10, 1 + vpshufd xmm10, xmm10, 78 + vpternlogq xmm12, xmm10, xmm13, 150 + vmovdqa xmm6, xmm12 + add edx, 16 + cmp edx, eax + jl L_AES_GCM_aad_update_avx512_tail +L_AES_GCM_aad_update_avx512_done: + vmovdqa OWORD PTR [r10], xmm6 + vzeroupper + vmovdqu xmm6, OWORD PTR [rsp+256] + vmovdqu xmm7, OWORD PTR [rsp+272] + vmovdqu xmm8, OWORD PTR [rsp+288] + vmovdqu xmm9, OWORD PTR [rsp+304] + vmovdqu xmm10, OWORD PTR [rsp+320] + vmovdqu xmm11, OWORD PTR [rsp+336] + vmovdqu xmm12, OWORD PTR [rsp+352] + vmovdqu xmm13, OWORD PTR [rsp+368] + vmovdqu xmm14, OWORD PTR [rsp+384] + vmovdqu xmm15, OWORD PTR [rsp+400] + add rsp, 416 ret AES_GCM_aad_update_avx512 ENDP _TEXT ENDS @@ -27403,7 +30709,7 @@ AES_GCM_encrypt_update_avx512 PROC vpand xmm5, xmm5, OWORD PTR L_avx512_aes_gcm_mod2_128 vpxor xmm5, xmm5, xmm8 xor edi, edi - cmp r9d, 256 + cmp r9d, 128 jl L_AES_GCM_encrypt_update_avx512_done_128 vmovdqa xmm2, xmm6 ; H ^ 1 @@ -27517,6 +30823,8 @@ AES_GCM_encrypt_update_avx512 PROC vpternlogq xmm11, xmm9, xmm12, 150 vmovdqa xmm7, xmm11 vmovdqu OWORD PTR [rsp+112], xmm7 + cmp r9d, 256 + jl L_AES_GCM_encrypt_update_avx512_no_ext2 ; H ^ 9 vmovdqu xmm0, OWORD PTR [rsp+48] vmovdqu xmm1, OWORD PTR [rsp+64] @@ -27940,6 +31248,7 @@ AES_GCM_encrypt_update_avx512 PROC vmovdqa xmm7, xmm11 vmovdqu OWORD PTR [rsp+496], xmm7 L_AES_GCM_encrypt_update_avx512_no_ext: +L_AES_GCM_encrypt_update_avx512_no_ext2: vbroadcasti32x4 zmm22, ptr_L_avx512_aes_gcm_bswap_epi64 vbroadcasti32x4 zmm30, ptr_L_avx512_aes_gcm_bswap_mask vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 @@ -28877,6 +32186,117 @@ L_AES_GCM_encrypt_update_avx512_last_ghash: vpxorq xmm6, xmm29, xmm0 vpternlogq xmm6, xmm5, xmm4, 150 L_AES_GCM_encrypt_update_avx512_after_256: + mov r13d, r9d + and r13d, 4294967168 + cmp edi, r13d + jge L_AES_GCM_encrypt_update_avx512_after_128 + ; 128 bytes of input + vbroadcasti32x4 zmm9, [rax] + vbroadcasti32x4 zmm10, [rax+16] + vbroadcasti32x4 zmm11, [rax+32] + vbroadcasti32x4 zmm12, [rax+48] + vbroadcasti32x4 zmm13, [rax+64] + vbroadcasti32x4 zmm14, [rax+80] + vbroadcasti32x4 zmm15, [rax+96] + vbroadcasti32x4 zmm1, [rax+112] + vbroadcasti32x4 zmm2, [rax+128] + vbroadcasti32x4 zmm3, [rax+144] + lea rsi, QWORD PTR [r10+rdi] + vbroadcasti32x4 zmm20, [r15] + vpaddd zmm16, zmm20, ptr_L_avx512_aes_gcm_inc_z0 + vpshufb zmm16, zmm16, zmm22 + vpaddd zmm17, zmm20, ptr_L_avx512_aes_gcm_inc_z1 + vpshufb zmm17, zmm17, zmm22 + vmovdqu xmm8, OWORD PTR [r15] + vpaddd xmm8, xmm8, OWORD PTR L_avx512_aes_gcm_eight + vmovdqu OWORD PTR [r15], xmm8 + vpxorq zmm16, zmm16, zmm9 + vpxorq zmm17, zmm17, zmm9 + vaesenc zmm16, zmm16, zmm10 + vaesenc zmm17, zmm17, zmm10 + vaesenc zmm16, zmm16, zmm11 + vaesenc zmm17, zmm17, zmm11 + vaesenc zmm16, zmm16, zmm12 + vaesenc zmm17, zmm17, zmm12 + vaesenc zmm16, zmm16, zmm13 + vaesenc zmm17, zmm17, zmm13 + vaesenc zmm16, zmm16, zmm14 + vaesenc zmm17, zmm17, zmm14 + vaesenc zmm16, zmm16, zmm15 + vaesenc zmm17, zmm17, zmm15 + vaesenc zmm16, zmm16, zmm1 + vaesenc zmm17, zmm17, zmm1 + vaesenc zmm16, zmm16, zmm2 + vaesenc zmm17, zmm17, zmm2 + vaesenc zmm16, zmm16, zmm3 + vaesenc zmm17, zmm17, zmm3 + cmp r8d, 11 + vbroadcasti32x4 zmm20, [rax+160] + jl L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [rax+176] + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + cmp r8d, 13 + vbroadcasti32x4 zmm20, [rax+192] + jl L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [rax+208] + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [rax+224] +L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last: + vaesenclast zmm16, zmm16, zmm20 + vaesenclast zmm17, zmm17, zmm20 + lea rcx, QWORD PTR [r11+rdi] + lea rdx, QWORD PTR [r10+rdi] + vmovdqu64 zmm21, [rcx] + vpxorq zmm16, zmm16, zmm21 + vmovdqu64 [rdx], zmm16 + vmovdqu64 zmm21, [rcx+64] + vpxorq zmm17, zmm17, zmm21 + vmovdqu64 [rdx+64], zmm17 + add edi, 128 + vmovdqu64 zmm23, [rsp+64] + vshufi64x2 zmm23, zmm23, zmm23, 27 + vmovdqu64 zmm24, [rsp] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rsi] + vpshufb zmm21, zmm21, zmm30 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm23, 0 + vpclmulqdq zmm17, zmm21, zmm23, 1 + vpclmulqdq zmm18, zmm21, zmm23, 16 + vpclmulqdq zmm19, zmm21, zmm23, 17 + vmovdqa64 zmm27, zmm16 + vpxorq zmm28, zmm18, zmm17 + vmovdqa64 zmm29, zmm19 + vmovdqu64 zmm21, [rsi+64] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 + vpclmulqdq zmm21, zmm31, zmm27, 1 + vpshufd zmm27, zmm27, 78 + vpternlogq zmm28, zmm27, zmm21, 150 + vpclmulqdq zmm21, zmm31, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vextracti32x4 xmm0, zmm29, 1 + vextracti32x4 xmm4, zmm29, 2 + vextracti32x4 xmm5, zmm29, 3 + vpxorq xmm6, xmm29, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 +L_AES_GCM_encrypt_update_avx512_after_128: vmovdqu xmm5, OWORD PTR [rsp] L_AES_GCM_encrypt_update_avx512_done_128: mov edx, r9d @@ -29146,7 +32566,7 @@ AES_GCM_decrypt_update_avx512 PROC vpand xmm5, xmm5, OWORD PTR L_avx512_aes_gcm_mod2_128 vpxor xmm5, xmm5, xmm8 xor edi, edi - cmp r9d, 256 + cmp r9d, 128 jl L_AES_GCM_decrypt_update_avx512_done_128 vmovdqa xmm2, xmm6 ; H ^ 1 @@ -29260,6 +32680,8 @@ AES_GCM_decrypt_update_avx512 PROC vpternlogq xmm11, xmm9, xmm12, 150 vmovdqa xmm7, xmm11 vmovdqu OWORD PTR [rsp+112], xmm7 + cmp r9d, 256 + jl L_AES_GCM_decrypt_update_avx512_no_ext2 ; H ^ 9 vmovdqu xmm0, OWORD PTR [rsp+48] vmovdqu xmm1, OWORD PTR [rsp+64] @@ -29683,6 +33105,7 @@ AES_GCM_decrypt_update_avx512 PROC vmovdqa xmm7, xmm11 vmovdqu OWORD PTR [rsp+496], xmm7 L_AES_GCM_decrypt_update_avx512_no_ext: +L_AES_GCM_decrypt_update_avx512_no_ext2: vbroadcasti32x4 zmm22, ptr_L_avx512_aes_gcm_bswap_epi64 vbroadcasti32x4 zmm30, ptr_L_avx512_aes_gcm_bswap_mask vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 @@ -30468,6 +33891,117 @@ L_AES_GCM_decrypt_update_avx512_t_avx512_ctr16_last: vmovdqu64 [rdx+192], zmm19 add edi, 256 L_AES_GCM_decrypt_update_avx512_after_256: + mov r13d, r9d + and r13d, 4294967168 + cmp edi, r13d + jge L_AES_GCM_decrypt_update_avx512_after_128 + ; 128 bytes of input + lea rbx, QWORD PTR [r11+rdi] + vmovdqu64 zmm23, [rsp+64] + vshufi64x2 zmm23, zmm23, zmm23, 27 + vmovdqu64 zmm24, [rsp] + vshufi64x2 zmm24, zmm24, zmm24, 27 + vpxorq zmm20, zmm20, zmm20 + vinserti32x4 zmm20, zmm20, xmm6, 0 + vmovdqu64 zmm21, [rbx] + vpshufb zmm21, zmm21, zmm30 + vpxorq zmm21, zmm21, zmm20 + vpclmulqdq zmm16, zmm21, zmm23, 0 + vpclmulqdq zmm17, zmm21, zmm23, 1 + vpclmulqdq zmm18, zmm21, zmm23, 16 + vpclmulqdq zmm19, zmm21, zmm23, 17 + vmovdqa64 zmm27, zmm16 + vpxorq zmm28, zmm18, zmm17 + vmovdqa64 zmm29, zmm19 + vmovdqu64 zmm21, [rbx+64] + vpshufb zmm21, zmm21, zmm30 + vpclmulqdq zmm16, zmm21, zmm24, 0 + vpclmulqdq zmm17, zmm21, zmm24, 1 + vpclmulqdq zmm18, zmm21, zmm24, 16 + vpclmulqdq zmm19, zmm21, zmm24, 17 + vpxorq zmm27, zmm27, zmm16 + vpternlogq zmm28, zmm18, zmm17, 150 + vpxorq zmm29, zmm29, zmm19 + vbroadcasti32x4 zmm31, ptr_L_avx512_aes_gcm_mod2_128 + vpclmulqdq zmm21, zmm31, zmm27, 1 + vpshufd zmm27, zmm27, 78 + vpternlogq zmm28, zmm27, zmm21, 150 + vpclmulqdq zmm21, zmm31, zmm28, 1 + vpshufd zmm28, zmm28, 78 + vpternlogq zmm29, zmm28, zmm21, 150 + vextracti32x4 xmm0, zmm29, 1 + vextracti32x4 xmm4, zmm29, 2 + vextracti32x4 xmm5, zmm29, 3 + vpxorq xmm6, xmm29, xmm0 + vpternlogq xmm6, xmm5, xmm4, 150 + vbroadcasti32x4 zmm9, [rax] + vbroadcasti32x4 zmm10, [rax+16] + vbroadcasti32x4 zmm11, [rax+32] + vbroadcasti32x4 zmm12, [rax+48] + vbroadcasti32x4 zmm13, [rax+64] + vbroadcasti32x4 zmm14, [rax+80] + vbroadcasti32x4 zmm15, [rax+96] + vbroadcasti32x4 zmm1, [rax+112] + vbroadcasti32x4 zmm2, [rax+128] + vbroadcasti32x4 zmm3, [rax+144] + vbroadcasti32x4 zmm20, [r15] + vpaddd zmm16, zmm20, ptr_L_avx512_aes_gcm_inc_z0 + vpshufb zmm16, zmm16, zmm22 + vpaddd zmm17, zmm20, ptr_L_avx512_aes_gcm_inc_z1 + vpshufb zmm17, zmm17, zmm22 + vmovdqu xmm8, OWORD PTR [r15] + vpaddd xmm8, xmm8, OWORD PTR L_avx512_aes_gcm_eight + vmovdqu OWORD PTR [r15], xmm8 + vpxorq zmm16, zmm16, zmm9 + vpxorq zmm17, zmm17, zmm9 + vaesenc zmm16, zmm16, zmm10 + vaesenc zmm17, zmm17, zmm10 + vaesenc zmm16, zmm16, zmm11 + vaesenc zmm17, zmm17, zmm11 + vaesenc zmm16, zmm16, zmm12 + vaesenc zmm17, zmm17, zmm12 + vaesenc zmm16, zmm16, zmm13 + vaesenc zmm17, zmm17, zmm13 + vaesenc zmm16, zmm16, zmm14 + vaesenc zmm17, zmm17, zmm14 + vaesenc zmm16, zmm16, zmm15 + vaesenc zmm17, zmm17, zmm15 + vaesenc zmm16, zmm16, zmm1 + vaesenc zmm17, zmm17, zmm1 + vaesenc zmm16, zmm16, zmm2 + vaesenc zmm17, zmm17, zmm2 + vaesenc zmm16, zmm16, zmm3 + vaesenc zmm17, zmm17, zmm3 + cmp r8d, 11 + vbroadcasti32x4 zmm20, [rax+160] + jl L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [rax+176] + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + cmp r8d, 13 + vbroadcasti32x4 zmm20, [rax+192] + jl L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [rax+208] + vaesenc zmm16, zmm16, zmm20 + vaesenc zmm17, zmm17, zmm20 + vbroadcasti32x4 zmm20, [rax+224] +L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last: + vaesenclast zmm16, zmm16, zmm20 + vaesenclast zmm17, zmm17, zmm20 + lea rcx, QWORD PTR [r11+rdi] + lea rdx, QWORD PTR [r10+rdi] + vmovdqu64 zmm21, [rcx] + vpxorq zmm16, zmm16, zmm21 + vmovdqu64 [rdx], zmm16 + vmovdqu64 zmm21, [rcx+64] + vpxorq zmm17, zmm17, zmm21 + vmovdqu64 [rdx+64], zmm17 + add edi, 128 +L_AES_GCM_decrypt_update_avx512_after_128: vmovdqu xmm5, OWORD PTR [rsp] L_AES_GCM_decrypt_update_avx512_done_128: mov edx, r9d diff --git a/wolfcrypt/src/aes_x86_64_asm.S b/wolfcrypt/src/aes_x86_64_asm.S index 9eb85b49c7..fba512db26 100644 --- a/wolfcrypt/src/aes_x86_64_asm.S +++ b/wolfcrypt/src/aes_x86_64_asm.S @@ -2284,6 +2284,71 @@ L_AES_ECB_encrypt_vaes_128_aes_enc_block_last: cmpl %r9d, %eax jl L_AES_ECB_encrypt_vaes_enc_128 L_AES_ECB_encrypt_vaes_done_128: + movl %edx, %r9d + subl %eax, %r9d + cmpl $0x40, %r9d + jl L_AES_ECB_encrypt_vaes_done_64 + # 64 bytes of input + # aes_ecb_enc_64 + leaq (%rdi,%rax,1), %r10 + leaq (%rsi,%rax,1), %r11 + vmovdqu (%r10), %ymm0 + vmovdqu 32(%r10), %ymm1 + # aes_enc_block + vbroadcasti128 (%rcx), %ymm7 + vpxor %ymm7, %ymm0, %ymm0 + vpxor %ymm7, %ymm1, %ymm1 + vbroadcasti128 16(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 32(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 48(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 64(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 80(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 96(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 112(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 128(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 144(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + cmpl $11, %r8d + vbroadcasti128 160(%rcx), %ymm7 + jl L_AES_ECB_encrypt_vaes_64_aes_enc_block_last + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 176(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + cmpl $13, %r8d + vbroadcasti128 192(%rcx), %ymm7 + jl L_AES_ECB_encrypt_vaes_64_aes_enc_block_last + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 208(%rcx), %ymm7 + vaesenc %ymm7, %ymm0, %ymm0 + vaesenc %ymm7, %ymm1, %ymm1 + vbroadcasti128 224(%rcx), %ymm7 +L_AES_ECB_encrypt_vaes_64_aes_enc_block_last: + vaesenclast %ymm7, %ymm0, %ymm0 + vaesenclast %ymm7, %ymm1, %ymm1 + vmovdqu %ymm0, (%r11) + vmovdqu %ymm1, 32(%r11) + addl $0x40, %eax +L_AES_ECB_encrypt_vaes_done_64: movl %edx, %r9d andl $0xffffffe0, %r9d cmpl %r9d, %eax @@ -2503,6 +2568,71 @@ L_AES_ECB_decrypt_vaes_128_aes_dec_block_last: cmpl %r9d, %eax jl L_AES_ECB_decrypt_vaes_dec_128 L_AES_ECB_decrypt_vaes_done_128: + movl %edx, %r9d + subl %eax, %r9d + cmpl $0x40, %r9d + jl L_AES_ECB_decrypt_vaes_done_64 + # 64 bytes of input + # aes_ecb_dec_64 + leaq (%rdi,%rax,1), %r10 + leaq (%rsi,%rax,1), %r11 + vmovdqu (%r10), %ymm0 + vmovdqu 32(%r10), %ymm1 + # aes_dec_block + vbroadcasti128 (%rcx), %ymm7 + vpxor %ymm7, %ymm0, %ymm0 + vpxor %ymm7, %ymm1, %ymm1 + vbroadcasti128 16(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 32(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 48(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 64(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 80(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 96(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 112(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 128(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 144(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + cmpl $11, %r8d + vbroadcasti128 160(%rcx), %ymm7 + jl L_AES_ECB_decrypt_vaes_64_aes_dec_block_last + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 176(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + cmpl $13, %r8d + vbroadcasti128 192(%rcx), %ymm7 + jl L_AES_ECB_decrypt_vaes_64_aes_dec_block_last + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 208(%rcx), %ymm7 + vaesdec %ymm7, %ymm0, %ymm0 + vaesdec %ymm7, %ymm1, %ymm1 + vbroadcasti128 224(%rcx), %ymm7 +L_AES_ECB_decrypt_vaes_64_aes_dec_block_last: + vaesdeclast %ymm7, %ymm0, %ymm0 + vaesdeclast %ymm7, %ymm1, %ymm1 + vmovdqu %ymm0, (%r11) + vmovdqu %ymm1, 32(%r11) + addl $0x40, %eax +L_AES_ECB_decrypt_vaes_done_64: movl %edx, %r9d andl $0xffffffe0, %r9d cmpl %r9d, %eax @@ -2801,6 +2931,76 @@ L_AES_CBC_decrypt_vaes_128_aes_dec_block_last: cmpl %r10d, %eax jl L_AES_CBC_decrypt_vaes_dec_128 L_AES_CBC_decrypt_vaes_done_128: + movl %ecx, %r10d + subl %eax, %r10d + cmpl $0x40, %r10d + jl L_AES_CBC_decrypt_vaes_done_64 + # 64 bytes of input + # aes_cbc_dec_64 + leaq (%rdi,%rax,1), %r11 + leaq (%rsi,%rax,1), %r12 + vmovdqu (%r11), %ymm0 + vmovdqu 32(%r11), %ymm1 + vinserti128 $0x01, %xmm0, %ymm8, %ymm10 + vmovdqu 16(%r11), %ymm11 + vextracti128 $0x01, %ymm1, %xmm8 + # aes_dec_block + vbroadcasti128 (%r8), %ymm9 + vpxor %ymm9, %ymm0, %ymm0 + vpxor %ymm9, %ymm1, %ymm1 + vbroadcasti128 16(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 32(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 48(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 64(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 80(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 96(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 112(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 128(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 144(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + cmpl $11, %r9d + vbroadcasti128 160(%r8), %ymm9 + jl L_AES_CBC_decrypt_vaes_64_aes_dec_block_last + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 176(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + cmpl $13, %r9d + vbroadcasti128 192(%r8), %ymm9 + jl L_AES_CBC_decrypt_vaes_64_aes_dec_block_last + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 208(%r8), %ymm9 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm9, %ymm1, %ymm1 + vbroadcasti128 224(%r8), %ymm9 +L_AES_CBC_decrypt_vaes_64_aes_dec_block_last: + vaesdeclast %ymm9, %ymm0, %ymm0 + vaesdeclast %ymm9, %ymm1, %ymm1 + vpxor %ymm10, %ymm0, %ymm0 + vpxor %ymm11, %ymm1, %ymm1 + vmovdqu %ymm0, (%r12) + vmovdqu %ymm1, 32(%r12) + addl $0x40, %eax +L_AES_CBC_decrypt_vaes_done_64: movl %ecx, %r10d andl $0xffffffe0, %r10d cmpl %r10d, %eax @@ -2969,47 +3169,47 @@ _AES_CTR_encrypt_vaes: vbroadcasti128 L_aes_ctr_bswap_vaes(%rip), %ymm8 vbroadcasti128 (%r9), %ymm7 vpshufb %ymm8, %ymm7, %ymm7 - vbroadcasti128 128+L_aes_ctr_inc_vaes(%rip), %ymm10 - vbroadcasti128 32+L_aes_ctr_inc_vaes(%rip), %ymm11 - vbroadcasti128 16+L_aes_ctr_inc_vaes(%rip), %ymm12 + vbroadcasti128 32+L_aes_ctr_inc_vaes(%rip), %ymm12 + vbroadcasti128 16+L_aes_ctr_inc_vaes(%rip), %ymm13 xorl %eax, %eax cmpl $0x80, %edx movl %edx, %r10d jl L_AES_CTR_encrypt_vaes_done_128 andl $0xffffff80, %r10d + vbroadcasti128 128+L_aes_ctr_inc_vaes(%rip), %ymm10 vmovdqa %ymm7, %ymm9 vpaddq 0+L_aes_ctr_inc_vaes(%rip), %ymm7, %ymm4 - vpand 0+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm14 + vpand 0+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm15 vpor 0+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm9 vpandn %ymm9, %ymm4, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm4, %ymm4 vmovdqa %ymm7, %ymm9 vpaddq 32+L_aes_ctr_inc_vaes(%rip), %ymm7, %ymm5 - vpand 32+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm14 + vpand 32+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm15 vpor 32+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm9 vpandn %ymm9, %ymm5, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm5, %ymm5 vmovdqa %ymm7, %ymm9 vpaddq 64+L_aes_ctr_inc_vaes(%rip), %ymm7, %ymm6 - vpand 64+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm14 + vpand 64+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm15 vpor 64+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm9 vpandn %ymm9, %ymm6, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm6, %ymm6 vmovdqa %ymm7, %ymm9 vpaddq 96+L_aes_ctr_inc_vaes(%rip), %ymm7, %ymm7 - vpand 96+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm14 + vpand 96+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm15 vpor 96+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm9 vpandn %ymm9, %ymm7, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm7, %ymm7 @@ -3023,121 +3223,121 @@ L_AES_CTR_encrypt_vaes_enc_128: vpshufb %ymm8, %ymm7, %ymm3 vmovdqa %ymm4, %ymm9 vpaddq %ymm10, %ymm4, %ymm4 - vpand %ymm10, %ymm9, %ymm14 + vpand %ymm10, %ymm9, %ymm15 vpor %ymm10, %ymm9, %ymm9 vpandn %ymm9, %ymm4, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm4, %ymm4 vmovdqa %ymm5, %ymm9 vpaddq %ymm10, %ymm5, %ymm5 - vpand %ymm10, %ymm9, %ymm14 + vpand %ymm10, %ymm9, %ymm15 vpor %ymm10, %ymm9, %ymm9 vpandn %ymm9, %ymm5, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm5, %ymm5 vmovdqa %ymm6, %ymm9 vpaddq %ymm10, %ymm6, %ymm6 - vpand %ymm10, %ymm9, %ymm14 + vpand %ymm10, %ymm9, %ymm15 vpor %ymm10, %ymm9, %ymm9 vpandn %ymm9, %ymm6, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm6, %ymm6 vmovdqa %ymm7, %ymm9 vpaddq %ymm10, %ymm7, %ymm7 - vpand %ymm10, %ymm9, %ymm14 + vpand %ymm10, %ymm9, %ymm15 vpor %ymm10, %ymm9, %ymm9 vpandn %ymm9, %ymm7, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm7, %ymm7 # aes_enc_block - vbroadcasti128 (%rcx), %ymm13 - vpxor %ymm13, %ymm0, %ymm0 - vpxor %ymm13, %ymm1, %ymm1 - vpxor %ymm13, %ymm2, %ymm2 - vpxor %ymm13, %ymm3, %ymm3 - vbroadcasti128 16(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 32(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 48(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 64(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 80(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 96(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 112(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 128(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 144(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 + vbroadcasti128 (%rcx), %ymm14 + vpxor %ymm14, %ymm0, %ymm0 + vpxor %ymm14, %ymm1, %ymm1 + vpxor %ymm14, %ymm2, %ymm2 + vpxor %ymm14, %ymm3, %ymm3 + vbroadcasti128 16(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 32(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 48(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 64(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 80(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 96(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 112(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 128(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 144(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 cmpl $11, %r8d - vbroadcasti128 160(%rcx), %ymm13 + vbroadcasti128 160(%rcx), %ymm14 jl L_AES_CTR_encrypt_vaes_128_aes_enc_block_last - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 176(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 176(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 cmpl $13, %r8d - vbroadcasti128 192(%rcx), %ymm13 + vbroadcasti128 192(%rcx), %ymm14 jl L_AES_CTR_encrypt_vaes_128_aes_enc_block_last - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 208(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vaesenc %ymm13, %ymm1, %ymm1 - vaesenc %ymm13, %ymm2, %ymm2 - vaesenc %ymm13, %ymm3, %ymm3 - vbroadcasti128 224(%rcx), %ymm13 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 208(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vaesenc %ymm14, %ymm2, %ymm2 + vaesenc %ymm14, %ymm3, %ymm3 + vbroadcasti128 224(%rcx), %ymm14 L_AES_CTR_encrypt_vaes_128_aes_enc_block_last: - vaesenclast %ymm13, %ymm0, %ymm0 - vaesenclast %ymm13, %ymm1, %ymm1 - vaesenclast %ymm13, %ymm2, %ymm2 - vaesenclast %ymm13, %ymm3, %ymm3 + vaesenclast %ymm14, %ymm0, %ymm0 + vaesenclast %ymm14, %ymm1, %ymm1 + vaesenclast %ymm14, %ymm2, %ymm2 + vaesenclast %ymm14, %ymm3, %ymm3 vpxor (%r11), %ymm0, %ymm0 vpxor 32(%r11), %ymm1, %ymm1 vpxor 64(%r11), %ymm2, %ymm2 @@ -3151,6 +3351,101 @@ L_AES_CTR_encrypt_vaes_128_aes_enc_block_last: jl L_AES_CTR_encrypt_vaes_enc_128 vperm2i128 $0x00, %ymm4, %ymm4, %ymm7 L_AES_CTR_encrypt_vaes_done_128: + movl %edx, %r10d + subl %eax, %r10d + cmpl $0x40, %r10d + jl L_AES_CTR_encrypt_vaes_done_64 + # 64 bytes of input + vbroadcasti128 64+L_aes_ctr_inc_vaes(%rip), %ymm11 + # aes_ctr_enc_64 + leaq (%rdi,%rax,1), %r11 + leaq (%rsi,%rax,1), %rbx + vpaddq 0+L_aes_ctr_inc_vaes(%rip), %ymm7, %ymm0 + vmovdqa %ymm7, %ymm9 + vpand 0+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm15 + vpor 0+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm9 + vpandn %ymm9, %ymm0, %ymm9 + vpor %ymm15, %ymm9, %ymm9 + vpsrlq $63, %ymm9, %ymm9 + vpslldq $8, %ymm9, %ymm9 + vpaddq %ymm9, %ymm0, %ymm0 + vpshufb %ymm8, %ymm0, %ymm0 + vpaddq 32+L_aes_ctr_inc_vaes(%rip), %ymm7, %ymm1 + vmovdqa %ymm7, %ymm9 + vpand 32+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm15 + vpor 32+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm9 + vpandn %ymm9, %ymm1, %ymm9 + vpor %ymm15, %ymm9, %ymm9 + vpsrlq $63, %ymm9, %ymm9 + vpslldq $8, %ymm9, %ymm9 + vpaddq %ymm9, %ymm1, %ymm1 + vpshufb %ymm8, %ymm1, %ymm1 + vmovdqa %ymm7, %ymm9 + vpaddq %ymm11, %ymm7, %ymm7 + vpand %ymm11, %ymm9, %ymm15 + vpor %ymm11, %ymm9, %ymm9 + vpandn %ymm9, %ymm7, %ymm9 + vpor %ymm15, %ymm9, %ymm9 + vpsrlq $63, %ymm9, %ymm9 + vpslldq $8, %ymm9, %ymm9 + vpaddq %ymm9, %ymm7, %ymm7 + # aes_enc_block + vbroadcasti128 (%rcx), %ymm14 + vpxor %ymm14, %ymm0, %ymm0 + vpxor %ymm14, %ymm1, %ymm1 + vbroadcasti128 16(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 32(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 48(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 64(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 80(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 96(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 112(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 128(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 144(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + cmpl $11, %r8d + vbroadcasti128 160(%rcx), %ymm14 + jl L_AES_CTR_encrypt_vaes_64_aes_enc_block_last + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 176(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + cmpl $13, %r8d + vbroadcasti128 192(%rcx), %ymm14 + jl L_AES_CTR_encrypt_vaes_64_aes_enc_block_last + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 208(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm14, %ymm1, %ymm1 + vbroadcasti128 224(%rcx), %ymm14 +L_AES_CTR_encrypt_vaes_64_aes_enc_block_last: + vaesenclast %ymm14, %ymm0, %ymm0 + vaesenclast %ymm14, %ymm1, %ymm1 + vpxor (%r11), %ymm0, %ymm0 + vpxor 32(%r11), %ymm1, %ymm1 + vmovdqu %ymm0, (%rbx) + vmovdqu %ymm1, 32(%rbx) + addl $0x40, %eax +L_AES_CTR_encrypt_vaes_done_64: movl %edx, %r10d andl $0xffffffe0, %r10d cmpl %r10d, %eax @@ -3162,59 +3457,59 @@ L_AES_CTR_encrypt_vaes_enc_32: leaq (%rsi,%rax,1), %rbx vpaddq 0+L_aes_ctr_inc_vaes(%rip), %ymm7, %ymm0 vmovdqa %ymm7, %ymm9 - vpand 0+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm14 + vpand 0+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm15 vpor 0+L_aes_ctr_inc_vaes(%rip), %ymm9, %ymm9 vpandn %ymm9, %ymm0, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm0, %ymm0 vpshufb %ymm8, %ymm0, %ymm0 vmovdqa %ymm7, %ymm9 - vpaddq %ymm11, %ymm7, %ymm7 - vpand %ymm11, %ymm9, %ymm14 - vpor %ymm11, %ymm9, %ymm9 + vpaddq %ymm12, %ymm7, %ymm7 + vpand %ymm12, %ymm9, %ymm15 + vpor %ymm12, %ymm9, %ymm9 vpandn %ymm9, %ymm7, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm7, %ymm7 # aes_enc_block - vbroadcasti128 (%rcx), %ymm13 - vpxor %ymm13, %ymm0, %ymm0 - vbroadcasti128 16(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 32(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 48(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 64(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 80(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 96(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 112(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 128(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 144(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 + vbroadcasti128 (%rcx), %ymm14 + vpxor %ymm14, %ymm0, %ymm0 + vbroadcasti128 16(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 32(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 48(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 64(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 80(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 96(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 112(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 128(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 144(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 cmpl $11, %r8d - vbroadcasti128 160(%rcx), %ymm13 + vbroadcasti128 160(%rcx), %ymm14 jl L_AES_CTR_encrypt_vaes_32_aes_enc_block_last - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 176(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 176(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 cmpl $13, %r8d - vbroadcasti128 192(%rcx), %ymm13 + vbroadcasti128 192(%rcx), %ymm14 jl L_AES_CTR_encrypt_vaes_32_aes_enc_block_last - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 208(%rcx), %ymm13 - vaesenc %ymm13, %ymm0, %ymm0 - vbroadcasti128 224(%rcx), %ymm13 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 208(%rcx), %ymm14 + vaesenc %ymm14, %ymm0, %ymm0 + vbroadcasti128 224(%rcx), %ymm14 L_AES_CTR_encrypt_vaes_32_aes_enc_block_last: - vaesenclast %ymm13, %ymm0, %ymm0 + vaesenclast %ymm14, %ymm0, %ymm0 vpxor (%r11), %ymm0, %ymm0 vmovdqu %ymm0, (%rbx) addl $32, %eax @@ -3229,11 +3524,11 @@ L_AES_CTR_encrypt_vaes_enc_16: # 16 bytes of input vpshufb %xmm8, %xmm7, %xmm0 vmovdqa %ymm7, %ymm9 - vpaddq %ymm12, %ymm7, %ymm7 - vpand %ymm12, %ymm9, %ymm14 - vpor %ymm12, %ymm9, %ymm9 + vpaddq %ymm13, %ymm7, %ymm7 + vpand %ymm13, %ymm9, %ymm15 + vpor %ymm13, %ymm9, %ymm9 vpandn %ymm9, %ymm7, %ymm9 - vpor %ymm14, %ymm9, %ymm9 + vpor %ymm15, %ymm9, %ymm9 vpsrlq $63, %ymm9, %ymm9 vpslldq $8, %ymm9, %ymm9 vpaddq %ymm9, %ymm7, %ymm7 @@ -3302,8 +3597,8 @@ AES_ECB_encrypt_avx512: _AES_ECB_encrypt_avx512: #endif /* __APPLE__ */ xorl %eax, %eax - cmpl $0x40, %edx - jl L_AES_ECB_encrypt_avx512_done_64 + cmpl $32, %edx + jl L_AES_ECB_encrypt_avx512_done_32 vbroadcasti32x4 (%rcx), %zmm8 vbroadcasti32x4 16(%rcx), %zmm9 vbroadcasti32x4 32(%rcx), %zmm10 @@ -3414,6 +3709,59 @@ L_AES_ECB_encrypt_avx512_256_aes_enc_block_last: cmpl %r9d, %eax jl L_AES_ECB_encrypt_avx512_enc_256 L_AES_ECB_encrypt_avx512_done_256: + movl %edx, %r9d + subl %eax, %r9d + cmpl $0x80, %r9d + jl L_AES_ECB_encrypt_avx512_done_128 + # 128 bytes of input + # aes_ecb_enc_128 + leaq (%rdi,%rax,1), %r10 + leaq (%rsi,%rax,1), %r11 + vmovdqu64 (%r10), %zmm0 + vmovdqu64 64(%r10), %zmm1 + # aes_enc_block + vpxorq %zmm8, %zmm0, %zmm0 + vpxorq %zmm8, %zmm1, %zmm1 + vaesenc %zmm9, %zmm0, %zmm0 + vaesenc %zmm9, %zmm1, %zmm1 + vaesenc %zmm10, %zmm0, %zmm0 + vaesenc %zmm10, %zmm1, %zmm1 + vaesenc %zmm11, %zmm0, %zmm0 + vaesenc %zmm11, %zmm1, %zmm1 + vaesenc %zmm12, %zmm0, %zmm0 + vaesenc %zmm12, %zmm1, %zmm1 + vaesenc %zmm13, %zmm0, %zmm0 + vaesenc %zmm13, %zmm1, %zmm1 + vaesenc %zmm14, %zmm0, %zmm0 + vaesenc %zmm14, %zmm1, %zmm1 + vaesenc %zmm15, %zmm0, %zmm0 + vaesenc %zmm15, %zmm1, %zmm1 + vaesenc %zmm16, %zmm0, %zmm0 + vaesenc %zmm16, %zmm1, %zmm1 + vaesenc %zmm17, %zmm0, %zmm0 + vaesenc %zmm17, %zmm1, %zmm1 + cmpl $11, %r8d + vmovdqa64 %zmm18, %zmm7 + jl L_AES_ECB_encrypt_avx512_128_aes_enc_block_last + vaesenc %zmm18, %zmm0, %zmm0 + vaesenc %zmm18, %zmm1, %zmm1 + vaesenc %zmm19, %zmm0, %zmm0 + vaesenc %zmm19, %zmm1, %zmm1 + cmpl $13, %r8d + vmovdqa64 %zmm20, %zmm7 + jl L_AES_ECB_encrypt_avx512_128_aes_enc_block_last + vaesenc %zmm20, %zmm0, %zmm0 + vaesenc %zmm20, %zmm1, %zmm1 + vaesenc %zmm21, %zmm0, %zmm0 + vaesenc %zmm21, %zmm1, %zmm1 + vmovdqa64 %zmm22, %zmm7 +L_AES_ECB_encrypt_avx512_128_aes_enc_block_last: + vaesenclast %zmm7, %zmm0, %zmm0 + vaesenclast %zmm7, %zmm1, %zmm1 + vmovdqu64 %zmm0, (%r11) + vmovdqu64 %zmm1, 64(%r11) + addl $0x80, %eax +L_AES_ECB_encrypt_avx512_done_128: movl %edx, %r9d andl $0xffffffc0, %r9d cmpl %r9d, %eax @@ -3453,6 +3801,41 @@ L_AES_ECB_encrypt_avx512_64_aes_enc_block_last: cmpl %r9d, %eax jl L_AES_ECB_encrypt_avx512_enc_64 L_AES_ECB_encrypt_avx512_done_64: + movl %edx, %r9d + subl %eax, %r9d + cmpl $32, %r9d + jl L_AES_ECB_encrypt_avx512_done_32 + # 32 bytes of input + leaq (%rdi,%rax,1), %r10 + vmovdqu (%r10), %ymm0 + # aes_enc_block + vpxorq %ymm8, %ymm0, %ymm0 + vaesenc %ymm9, %ymm0, %ymm0 + vaesenc %ymm10, %ymm0, %ymm0 + vaesenc %ymm11, %ymm0, %ymm0 + vaesenc %ymm12, %ymm0, %ymm0 + vaesenc %ymm13, %ymm0, %ymm0 + vaesenc %ymm14, %ymm0, %ymm0 + vaesenc %ymm15, %ymm0, %ymm0 + vaesenc %ymm16, %ymm0, %ymm0 + vaesenc %ymm17, %ymm0, %ymm0 + cmpl $11, %r8d + vmovdqa64 %ymm18, %ymm7 + jl L_AES_ECB_encrypt_avx512_32_aes_enc_block_last + vaesenc %ymm18, %ymm0, %ymm0 + vaesenc %ymm19, %ymm0, %ymm0 + cmpl $13, %r8d + vmovdqa64 %ymm20, %ymm7 + jl L_AES_ECB_encrypt_avx512_32_aes_enc_block_last + vaesenc %ymm20, %ymm0, %ymm0 + vaesenc %ymm21, %ymm0, %ymm0 + vmovdqa64 %ymm22, %ymm7 +L_AES_ECB_encrypt_avx512_32_aes_enc_block_last: + vaesenclast %ymm7, %ymm0, %ymm0 + leaq (%rsi,%rax,1), %r10 + vmovdqu %ymm0, (%r10) + addl $32, %eax +L_AES_ECB_encrypt_avx512_done_32: cmpl %edx, %eax movl %edx, %r9d je L_AES_ECB_encrypt_avx512_done_enc @@ -3519,8 +3902,8 @@ AES_ECB_decrypt_avx512: _AES_ECB_decrypt_avx512: #endif /* __APPLE__ */ xorl %eax, %eax - cmpl $0x40, %edx - jl L_AES_ECB_decrypt_avx512_done_64 + cmpl $32, %edx + jl L_AES_ECB_decrypt_avx512_done_32 vbroadcasti32x4 (%rcx), %zmm8 vbroadcasti32x4 16(%rcx), %zmm9 vbroadcasti32x4 32(%rcx), %zmm10 @@ -3631,6 +4014,59 @@ L_AES_ECB_decrypt_avx512_256_aes_dec_block_last: cmpl %r9d, %eax jl L_AES_ECB_decrypt_avx512_dec_256 L_AES_ECB_decrypt_avx512_done_256: + movl %edx, %r9d + subl %eax, %r9d + cmpl $0x80, %r9d + jl L_AES_ECB_decrypt_avx512_done_128 + # 128 bytes of input + # aes_ecb_dec_128 + leaq (%rdi,%rax,1), %r10 + leaq (%rsi,%rax,1), %r11 + vmovdqu64 (%r10), %zmm0 + vmovdqu64 64(%r10), %zmm1 + # aes_dec_block + vpxorq %zmm8, %zmm0, %zmm0 + vpxorq %zmm8, %zmm1, %zmm1 + vaesdec %zmm9, %zmm0, %zmm0 + vaesdec %zmm9, %zmm1, %zmm1 + vaesdec %zmm10, %zmm0, %zmm0 + vaesdec %zmm10, %zmm1, %zmm1 + vaesdec %zmm11, %zmm0, %zmm0 + vaesdec %zmm11, %zmm1, %zmm1 + vaesdec %zmm12, %zmm0, %zmm0 + vaesdec %zmm12, %zmm1, %zmm1 + vaesdec %zmm13, %zmm0, %zmm0 + vaesdec %zmm13, %zmm1, %zmm1 + vaesdec %zmm14, %zmm0, %zmm0 + vaesdec %zmm14, %zmm1, %zmm1 + vaesdec %zmm15, %zmm0, %zmm0 + vaesdec %zmm15, %zmm1, %zmm1 + vaesdec %zmm16, %zmm0, %zmm0 + vaesdec %zmm16, %zmm1, %zmm1 + vaesdec %zmm17, %zmm0, %zmm0 + vaesdec %zmm17, %zmm1, %zmm1 + cmpl $11, %r8d + vmovdqa64 %zmm18, %zmm7 + jl L_AES_ECB_decrypt_avx512_128_aes_dec_block_last + vaesdec %zmm18, %zmm0, %zmm0 + vaesdec %zmm18, %zmm1, %zmm1 + vaesdec %zmm19, %zmm0, %zmm0 + vaesdec %zmm19, %zmm1, %zmm1 + cmpl $13, %r8d + vmovdqa64 %zmm20, %zmm7 + jl L_AES_ECB_decrypt_avx512_128_aes_dec_block_last + vaesdec %zmm20, %zmm0, %zmm0 + vaesdec %zmm20, %zmm1, %zmm1 + vaesdec %zmm21, %zmm0, %zmm0 + vaesdec %zmm21, %zmm1, %zmm1 + vmovdqa64 %zmm22, %zmm7 +L_AES_ECB_decrypt_avx512_128_aes_dec_block_last: + vaesdeclast %zmm7, %zmm0, %zmm0 + vaesdeclast %zmm7, %zmm1, %zmm1 + vmovdqu64 %zmm0, (%r11) + vmovdqu64 %zmm1, 64(%r11) + addl $0x80, %eax +L_AES_ECB_decrypt_avx512_done_128: movl %edx, %r9d andl $0xffffffc0, %r9d cmpl %r9d, %eax @@ -3670,6 +4106,41 @@ L_AES_ECB_decrypt_avx512_64_aes_dec_block_last: cmpl %r9d, %eax jl L_AES_ECB_decrypt_avx512_dec_64 L_AES_ECB_decrypt_avx512_done_64: + movl %edx, %r9d + subl %eax, %r9d + cmpl $32, %r9d + jl L_AES_ECB_decrypt_avx512_done_32 + # 32 bytes of input + leaq (%rdi,%rax,1), %r10 + vmovdqu (%r10), %ymm0 + # aes_dec_block + vpxorq %ymm8, %ymm0, %ymm0 + vaesdec %ymm9, %ymm0, %ymm0 + vaesdec %ymm10, %ymm0, %ymm0 + vaesdec %ymm11, %ymm0, %ymm0 + vaesdec %ymm12, %ymm0, %ymm0 + vaesdec %ymm13, %ymm0, %ymm0 + vaesdec %ymm14, %ymm0, %ymm0 + vaesdec %ymm15, %ymm0, %ymm0 + vaesdec %ymm16, %ymm0, %ymm0 + vaesdec %ymm17, %ymm0, %ymm0 + cmpl $11, %r8d + vmovdqa64 %ymm18, %ymm7 + jl L_AES_ECB_decrypt_avx512_32_aes_dec_block_last + vaesdec %ymm18, %ymm0, %ymm0 + vaesdec %ymm19, %ymm0, %ymm0 + cmpl $13, %r8d + vmovdqa64 %ymm20, %ymm7 + jl L_AES_ECB_decrypt_avx512_32_aes_dec_block_last + vaesdec %ymm20, %ymm0, %ymm0 + vaesdec %ymm21, %ymm0, %ymm0 + vmovdqa64 %ymm22, %ymm7 +L_AES_ECB_decrypt_avx512_32_aes_dec_block_last: + vaesdeclast %ymm7, %ymm0, %ymm0 + leaq (%rsi,%rax,1), %r10 + vmovdqu %ymm0, (%r10) + addl $32, %eax +L_AES_ECB_decrypt_avx512_done_32: cmpl %edx, %eax movl %edx, %r9d je L_AES_ECB_decrypt_avx512_done_dec @@ -3805,8 +4276,8 @@ _AES_CBC_decrypt_avx512: pushq %r12 vmovdqu (%rdx), %xmm8 xorl %eax, %eax - cmpl $0x40, %ecx - jl L_AES_CBC_decrypt_avx512_done_64 + cmpl $32, %ecx + jl L_AES_CBC_decrypt_avx512_done_32 vbroadcasti32x4 (%r8), %zmm14 vbroadcasti32x4 16(%r8), %zmm15 vbroadcasti32x4 32(%r8), %zmm16 @@ -3927,6 +4398,65 @@ L_AES_CBC_decrypt_avx512_256_aes_dec_block_last: cmpl %r10d, %eax jl L_AES_CBC_decrypt_avx512_dec_256 L_AES_CBC_decrypt_avx512_done_256: + movl %ecx, %r10d + subl %eax, %r10d + cmpl $0x80, %r10d + jl L_AES_CBC_decrypt_avx512_done_128 + # 128 bytes of input + # aes_cbc_dec_128 + leaq (%rdi,%rax,1), %r11 + leaq (%rsi,%rax,1), %r12 + vmovdqu64 (%r11), %zmm0 + vmovdqu64 64(%r11), %zmm1 + vshufi64x2 $0x90, %zmm0, %zmm0, %zmm10 + vinserti32x4 $0x00, %xmm8, %zmm10, %zmm10 + vmovdqu64 48(%r11), %zmm11 + vextracti32x4 $3, %zmm1, %xmm8 + # aes_dec_block + vpxorq %zmm14, %zmm0, %zmm0 + vpxorq %zmm14, %zmm1, %zmm1 + vaesdec %zmm15, %zmm0, %zmm0 + vaesdec %zmm15, %zmm1, %zmm1 + vaesdec %zmm16, %zmm0, %zmm0 + vaesdec %zmm16, %zmm1, %zmm1 + vaesdec %zmm17, %zmm0, %zmm0 + vaesdec %zmm17, %zmm1, %zmm1 + vaesdec %zmm18, %zmm0, %zmm0 + vaesdec %zmm18, %zmm1, %zmm1 + vaesdec %zmm19, %zmm0, %zmm0 + vaesdec %zmm19, %zmm1, %zmm1 + vaesdec %zmm20, %zmm0, %zmm0 + vaesdec %zmm20, %zmm1, %zmm1 + vaesdec %zmm21, %zmm0, %zmm0 + vaesdec %zmm21, %zmm1, %zmm1 + vaesdec %zmm22, %zmm0, %zmm0 + vaesdec %zmm22, %zmm1, %zmm1 + vaesdec %zmm23, %zmm0, %zmm0 + vaesdec %zmm23, %zmm1, %zmm1 + cmpl $11, %r9d + vmovdqa64 %zmm24, %zmm9 + jl L_AES_CBC_decrypt_avx512_128_aes_dec_block_last + vaesdec %zmm24, %zmm0, %zmm0 + vaesdec %zmm24, %zmm1, %zmm1 + vaesdec %zmm25, %zmm0, %zmm0 + vaesdec %zmm25, %zmm1, %zmm1 + cmpl $13, %r9d + vmovdqa64 %zmm26, %zmm9 + jl L_AES_CBC_decrypt_avx512_128_aes_dec_block_last + vaesdec %zmm26, %zmm0, %zmm0 + vaesdec %zmm26, %zmm1, %zmm1 + vaesdec %zmm27, %zmm0, %zmm0 + vaesdec %zmm27, %zmm1, %zmm1 + vmovdqa64 %zmm28, %zmm9 +L_AES_CBC_decrypt_avx512_128_aes_dec_block_last: + vaesdeclast %zmm9, %zmm0, %zmm0 + vaesdeclast %zmm9, %zmm1, %zmm1 + vpxorq %zmm10, %zmm0, %zmm0 + vpxorq %zmm11, %zmm1, %zmm1 + vmovdqu64 %zmm0, (%r12) + vmovdqu64 %zmm1, 64(%r12) + addl $0x80, %eax +L_AES_CBC_decrypt_avx512_done_128: movl %ecx, %r10d andl $0xffffffc0, %r10d cmpl %r10d, %eax @@ -3970,6 +4500,44 @@ L_AES_CBC_decrypt_avx512_64_aes_dec_block_last: cmpl %r10d, %eax jl L_AES_CBC_decrypt_avx512_dec_64 L_AES_CBC_decrypt_avx512_done_64: + movl %ecx, %r10d + subl %eax, %r10d + cmpl $32, %r10d + jl L_AES_CBC_decrypt_avx512_done_32 + # 32 bytes of input + leaq (%rdi,%rax,1), %r11 + vmovdqu (%r11), %ymm0 + vinserti128 $0x01, %xmm0, %ymm8, %ymm10 + vextracti128 $0x01, %ymm0, %xmm8 + # aes_dec_block + vpxorq %ymm14, %ymm0, %ymm0 + vaesdec %ymm15, %ymm0, %ymm0 + vaesdec %ymm16, %ymm0, %ymm0 + vaesdec %ymm17, %ymm0, %ymm0 + vaesdec %ymm18, %ymm0, %ymm0 + vaesdec %ymm19, %ymm0, %ymm0 + vaesdec %ymm20, %ymm0, %ymm0 + vaesdec %ymm21, %ymm0, %ymm0 + vaesdec %ymm22, %ymm0, %ymm0 + vaesdec %ymm23, %ymm0, %ymm0 + cmpl $11, %r9d + vmovdqa64 %ymm24, %ymm9 + jl L_AES_CBC_decrypt_avx512_32_aes_dec_block_last + vaesdec %ymm24, %ymm0, %ymm0 + vaesdec %ymm25, %ymm0, %ymm0 + cmpl $13, %r9d + vmovdqa64 %ymm26, %ymm9 + jl L_AES_CBC_decrypt_avx512_32_aes_dec_block_last + vaesdec %ymm26, %ymm0, %ymm0 + vaesdec %ymm27, %ymm0, %ymm0 + vmovdqa64 %ymm28, %ymm9 +L_AES_CBC_decrypt_avx512_32_aes_dec_block_last: + vaesdeclast %ymm9, %ymm0, %ymm0 + vpxorq %ymm10, %ymm0, %ymm0 + leaq (%rsi,%rax,1), %r11 + vmovdqu %ymm0, (%r11) + addl $32, %eax +L_AES_CBC_decrypt_avx512_done_32: cmpl %ecx, %eax movl %ecx, %r10d je L_AES_CBC_decrypt_avx512_done_dec @@ -4084,36 +4652,36 @@ _AES_CTR_encrypt_avx512: vbroadcasti32x4 L_aes_ctr_bswap_avx512(%rip), %zmm8 vbroadcasti32x4 (%r9), %zmm7 vpshufb %zmm8, %zmm7, %zmm7 - vbroadcasti32x4 256+L_aes_ctr_inc_avx512(%rip), %zmm10 - vbroadcasti32x4 64+L_aes_ctr_inc_avx512(%rip), %zmm11 - vbroadcasti32x4 16+L_aes_ctr_inc_avx512(%rip), %zmm12 + vbroadcasti32x4 64+L_aes_ctr_inc_avx512(%rip), %zmm12 + vbroadcasti32x4 16+L_aes_ctr_inc_avx512(%rip), %zmm13 xorl %eax, %eax - cmpl $0x40, %edx - jl L_AES_CTR_encrypt_avx512_done_64 - vbroadcasti32x4 (%rcx), %zmm14 - vbroadcasti32x4 16(%rcx), %zmm15 - vbroadcasti32x4 32(%rcx), %zmm16 - vbroadcasti32x4 48(%rcx), %zmm17 - vbroadcasti32x4 64(%rcx), %zmm18 - vbroadcasti32x4 80(%rcx), %zmm19 - vbroadcasti32x4 96(%rcx), %zmm20 - vbroadcasti32x4 112(%rcx), %zmm21 - vbroadcasti32x4 128(%rcx), %zmm22 - vbroadcasti32x4 144(%rcx), %zmm23 - vbroadcasti32x4 160(%rcx), %zmm24 + cmpl $32, %edx + jl L_AES_CTR_encrypt_avx512_done_32 + vbroadcasti32x4 (%rcx), %zmm15 + vbroadcasti32x4 16(%rcx), %zmm16 + vbroadcasti32x4 32(%rcx), %zmm17 + vbroadcasti32x4 48(%rcx), %zmm18 + vbroadcasti32x4 64(%rcx), %zmm19 + vbroadcasti32x4 80(%rcx), %zmm20 + vbroadcasti32x4 96(%rcx), %zmm21 + vbroadcasti32x4 112(%rcx), %zmm22 + vbroadcasti32x4 128(%rcx), %zmm23 + vbroadcasti32x4 144(%rcx), %zmm24 + vbroadcasti32x4 160(%rcx), %zmm25 cmpl $11, %r8d jl L_AES_CTR_encrypt_avx512_key_cached - vbroadcasti32x4 176(%rcx), %zmm25 - vbroadcasti32x4 192(%rcx), %zmm26 + vbroadcasti32x4 176(%rcx), %zmm26 + vbroadcasti32x4 192(%rcx), %zmm27 cmpl $13, %r8d jl L_AES_CTR_encrypt_avx512_key_cached - vbroadcasti32x4 208(%rcx), %zmm27 - vbroadcasti32x4 224(%rcx), %zmm28 + vbroadcasti32x4 208(%rcx), %zmm28 + vbroadcasti32x4 224(%rcx), %zmm29 L_AES_CTR_encrypt_avx512_key_cached: cmpl $0x100, %edx movl %edx, %r10d jl L_AES_CTR_encrypt_avx512_done_256 andl $0xffffff00, %r10d + vbroadcasti32x4 256+L_aes_ctr_inc_avx512(%rip), %zmm10 vmovdqa64 %zmm7, %zmm9 vpaddq 0+L_aes_ctr_inc_avx512(%rip), %zmm7, %zmm4 vpternlogq $0xb2, 0+L_aes_ctr_inc_avx512(%rip), %zmm4, %zmm9 @@ -4171,14 +4739,10 @@ L_AES_CTR_encrypt_avx512_enc_256: vpslldq $8, %zmm9, %zmm9 vpaddq %zmm9, %zmm7, %zmm7 # aes_enc_block - vpxorq %zmm14, %zmm0, %zmm0 - vpxorq %zmm14, %zmm1, %zmm1 - vpxorq %zmm14, %zmm2, %zmm2 - vpxorq %zmm14, %zmm3, %zmm3 - vaesenc %zmm15, %zmm0, %zmm0 - vaesenc %zmm15, %zmm1, %zmm1 - vaesenc %zmm15, %zmm2, %zmm2 - vaesenc %zmm15, %zmm3, %zmm3 + vpxorq %zmm15, %zmm0, %zmm0 + vpxorq %zmm15, %zmm1, %zmm1 + vpxorq %zmm15, %zmm2, %zmm2 + vpxorq %zmm15, %zmm3, %zmm3 vaesenc %zmm16, %zmm0, %zmm0 vaesenc %zmm16, %zmm1, %zmm1 vaesenc %zmm16, %zmm2, %zmm2 @@ -4211,34 +4775,38 @@ L_AES_CTR_encrypt_avx512_enc_256: vaesenc %zmm23, %zmm1, %zmm1 vaesenc %zmm23, %zmm2, %zmm2 vaesenc %zmm23, %zmm3, %zmm3 - cmpl $11, %r8d - vmovdqa64 %zmm24, %zmm13 - jl L_AES_CTR_encrypt_avx512_256_aes_enc_block_last vaesenc %zmm24, %zmm0, %zmm0 vaesenc %zmm24, %zmm1, %zmm1 vaesenc %zmm24, %zmm2, %zmm2 vaesenc %zmm24, %zmm3, %zmm3 + cmpl $11, %r8d + vmovdqa64 %zmm25, %zmm14 + jl L_AES_CTR_encrypt_avx512_256_aes_enc_block_last vaesenc %zmm25, %zmm0, %zmm0 vaesenc %zmm25, %zmm1, %zmm1 vaesenc %zmm25, %zmm2, %zmm2 vaesenc %zmm25, %zmm3, %zmm3 - cmpl $13, %r8d - vmovdqa64 %zmm26, %zmm13 - jl L_AES_CTR_encrypt_avx512_256_aes_enc_block_last vaesenc %zmm26, %zmm0, %zmm0 vaesenc %zmm26, %zmm1, %zmm1 vaesenc %zmm26, %zmm2, %zmm2 vaesenc %zmm26, %zmm3, %zmm3 + cmpl $13, %r8d + vmovdqa64 %zmm27, %zmm14 + jl L_AES_CTR_encrypt_avx512_256_aes_enc_block_last vaesenc %zmm27, %zmm0, %zmm0 vaesenc %zmm27, %zmm1, %zmm1 vaesenc %zmm27, %zmm2, %zmm2 vaesenc %zmm27, %zmm3, %zmm3 - vmovdqa64 %zmm28, %zmm13 + vaesenc %zmm28, %zmm0, %zmm0 + vaesenc %zmm28, %zmm1, %zmm1 + vaesenc %zmm28, %zmm2, %zmm2 + vaesenc %zmm28, %zmm3, %zmm3 + vmovdqa64 %zmm29, %zmm14 L_AES_CTR_encrypt_avx512_256_aes_enc_block_last: - vaesenclast %zmm13, %zmm0, %zmm0 - vaesenclast %zmm13, %zmm1, %zmm1 - vaesenclast %zmm13, %zmm2, %zmm2 - vaesenclast %zmm13, %zmm3, %zmm3 + vaesenclast %zmm14, %zmm0, %zmm0 + vaesenclast %zmm14, %zmm1, %zmm1 + vaesenclast %zmm14, %zmm2, %zmm2 + vaesenclast %zmm14, %zmm3, %zmm3 vpxorq (%r11), %zmm0, %zmm0 vpxorq 64(%r11), %zmm1, %zmm1 vpxorq 128(%r11), %zmm2, %zmm2 @@ -4252,6 +4820,80 @@ L_AES_CTR_encrypt_avx512_256_aes_enc_block_last: jl L_AES_CTR_encrypt_avx512_enc_256 vshufi64x2 $0x00, %zmm4, %zmm4, %zmm7 L_AES_CTR_encrypt_avx512_done_256: + movl %edx, %r10d + subl %eax, %r10d + cmpl $0x80, %r10d + jl L_AES_CTR_encrypt_avx512_done_128 + # 128 bytes of input + vbroadcasti32x4 128+L_aes_ctr_inc_avx512(%rip), %zmm11 + # aes_ctr_enc_128 + leaq (%rdi,%rax,1), %r11 + leaq (%rsi,%rax,1), %rbx + vpaddq 0+L_aes_ctr_inc_avx512(%rip), %zmm7, %zmm0 + vmovdqa64 %zmm7, %zmm9 + vpternlogq $0xb2, 0+L_aes_ctr_inc_avx512(%rip), %zmm0, %zmm9 + vpsrlq $63, %zmm9, %zmm9 + vpslldq $8, %zmm9, %zmm9 + vpaddq %zmm9, %zmm0, %zmm0 + vpshufb %zmm8, %zmm0, %zmm0 + vpaddq 64+L_aes_ctr_inc_avx512(%rip), %zmm7, %zmm1 + vmovdqa64 %zmm7, %zmm9 + vpternlogq $0xb2, 64+L_aes_ctr_inc_avx512(%rip), %zmm1, %zmm9 + vpsrlq $63, %zmm9, %zmm9 + vpslldq $8, %zmm9, %zmm9 + vpaddq %zmm9, %zmm1, %zmm1 + vpshufb %zmm8, %zmm1, %zmm1 + vmovdqa64 %zmm7, %zmm9 + vpaddq %zmm11, %zmm7, %zmm7 + vpternlogq $0xb2, %zmm11, %zmm7, %zmm9 + vpsrlq $63, %zmm9, %zmm9 + vpslldq $8, %zmm9, %zmm9 + vpaddq %zmm9, %zmm7, %zmm7 + # aes_enc_block + vpxorq %zmm15, %zmm0, %zmm0 + vpxorq %zmm15, %zmm1, %zmm1 + vaesenc %zmm16, %zmm0, %zmm0 + vaesenc %zmm16, %zmm1, %zmm1 + vaesenc %zmm17, %zmm0, %zmm0 + vaesenc %zmm17, %zmm1, %zmm1 + vaesenc %zmm18, %zmm0, %zmm0 + vaesenc %zmm18, %zmm1, %zmm1 + vaesenc %zmm19, %zmm0, %zmm0 + vaesenc %zmm19, %zmm1, %zmm1 + vaesenc %zmm20, %zmm0, %zmm0 + vaesenc %zmm20, %zmm1, %zmm1 + vaesenc %zmm21, %zmm0, %zmm0 + vaesenc %zmm21, %zmm1, %zmm1 + vaesenc %zmm22, %zmm0, %zmm0 + vaesenc %zmm22, %zmm1, %zmm1 + vaesenc %zmm23, %zmm0, %zmm0 + vaesenc %zmm23, %zmm1, %zmm1 + vaesenc %zmm24, %zmm0, %zmm0 + vaesenc %zmm24, %zmm1, %zmm1 + cmpl $11, %r8d + vmovdqa64 %zmm25, %zmm14 + jl L_AES_CTR_encrypt_avx512_128_aes_enc_block_last + vaesenc %zmm25, %zmm0, %zmm0 + vaesenc %zmm25, %zmm1, %zmm1 + vaesenc %zmm26, %zmm0, %zmm0 + vaesenc %zmm26, %zmm1, %zmm1 + cmpl $13, %r8d + vmovdqa64 %zmm27, %zmm14 + jl L_AES_CTR_encrypt_avx512_128_aes_enc_block_last + vaesenc %zmm27, %zmm0, %zmm0 + vaesenc %zmm27, %zmm1, %zmm1 + vaesenc %zmm28, %zmm0, %zmm0 + vaesenc %zmm28, %zmm1, %zmm1 + vmovdqa64 %zmm29, %zmm14 +L_AES_CTR_encrypt_avx512_128_aes_enc_block_last: + vaesenclast %zmm14, %zmm0, %zmm0 + vaesenclast %zmm14, %zmm1, %zmm1 + vpxorq (%r11), %zmm0, %zmm0 + vpxorq 64(%r11), %zmm1, %zmm1 + vmovdqu64 %zmm0, (%rbx) + vmovdqu64 %zmm1, 64(%rbx) + addl $0x80, %eax +L_AES_CTR_encrypt_avx512_done_128: movl %edx, %r10d andl $0xffffffc0, %r10d cmpl %r10d, %eax @@ -4269,14 +4911,13 @@ L_AES_CTR_encrypt_avx512_enc_64: vpaddq %zmm9, %zmm0, %zmm0 vpshufb %zmm8, %zmm0, %zmm0 vmovdqa64 %zmm7, %zmm9 - vpaddq %zmm11, %zmm7, %zmm7 - vpternlogq $0xb2, %zmm11, %zmm7, %zmm9 + vpaddq %zmm12, %zmm7, %zmm7 + vpternlogq $0xb2, %zmm12, %zmm7, %zmm9 vpsrlq $63, %zmm9, %zmm9 vpslldq $8, %zmm9, %zmm9 vpaddq %zmm9, %zmm7, %zmm7 # aes_enc_block - vpxorq %zmm14, %zmm0, %zmm0 - vaesenc %zmm15, %zmm0, %zmm0 + vpxorq %zmm15, %zmm0, %zmm0 vaesenc %zmm16, %zmm0, %zmm0 vaesenc %zmm17, %zmm0, %zmm0 vaesenc %zmm18, %zmm0, %zmm0 @@ -4285,25 +4926,75 @@ L_AES_CTR_encrypt_avx512_enc_64: vaesenc %zmm21, %zmm0, %zmm0 vaesenc %zmm22, %zmm0, %zmm0 vaesenc %zmm23, %zmm0, %zmm0 - cmpl $11, %r8d - vmovdqa64 %zmm24, %zmm13 - jl L_AES_CTR_encrypt_avx512_64_aes_enc_block_last vaesenc %zmm24, %zmm0, %zmm0 - vaesenc %zmm25, %zmm0, %zmm0 - cmpl $13, %r8d - vmovdqa64 %zmm26, %zmm13 + cmpl $11, %r8d + vmovdqa64 %zmm25, %zmm14 jl L_AES_CTR_encrypt_avx512_64_aes_enc_block_last + vaesenc %zmm25, %zmm0, %zmm0 vaesenc %zmm26, %zmm0, %zmm0 + cmpl $13, %r8d + vmovdqa64 %zmm27, %zmm14 + jl L_AES_CTR_encrypt_avx512_64_aes_enc_block_last vaesenc %zmm27, %zmm0, %zmm0 - vmovdqa64 %zmm28, %zmm13 + vaesenc %zmm28, %zmm0, %zmm0 + vmovdqa64 %zmm29, %zmm14 L_AES_CTR_encrypt_avx512_64_aes_enc_block_last: - vaesenclast %zmm13, %zmm0, %zmm0 + vaesenclast %zmm14, %zmm0, %zmm0 vpxorq (%r11), %zmm0, %zmm0 vmovdqu64 %zmm0, (%rbx) addl $0x40, %eax cmpl %r10d, %eax jl L_AES_CTR_encrypt_avx512_enc_64 L_AES_CTR_encrypt_avx512_done_64: + movl %edx, %r10d + subl %eax, %r10d + cmpl $32, %r10d + jl L_AES_CTR_encrypt_avx512_done_32 + # 32 bytes of input + vbroadcasti32x4 32+L_aes_ctr_inc_avx512(%rip), %zmm4 + vpaddq 0+L_aes_ctr_inc_avx512(%rip), %ymm7, %ymm0 + vmovdqa64 %ymm7, %ymm9 + vpternlogq $0xb2, 0+L_aes_ctr_inc_avx512(%rip), %ymm0, %ymm9 + vpsrlq $63, %ymm9, %ymm9 + vpslldq $8, %ymm9, %ymm9 + vpaddq %ymm9, %ymm0, %ymm0 + vpshufb %ymm8, %ymm0, %ymm0 + vmovdqa64 %zmm7, %zmm9 + vpaddq %zmm4, %zmm7, %zmm7 + vpternlogq $0xb2, %zmm4, %zmm7, %zmm9 + vpsrlq $63, %zmm9, %zmm9 + vpslldq $8, %zmm9, %zmm9 + vpaddq %zmm9, %zmm7, %zmm7 + # aes_enc_block + vpxorq %ymm15, %ymm0, %ymm0 + vaesenc %ymm16, %ymm0, %ymm0 + vaesenc %ymm17, %ymm0, %ymm0 + vaesenc %ymm18, %ymm0, %ymm0 + vaesenc %ymm19, %ymm0, %ymm0 + vaesenc %ymm20, %ymm0, %ymm0 + vaesenc %ymm21, %ymm0, %ymm0 + vaesenc %ymm22, %ymm0, %ymm0 + vaesenc %ymm23, %ymm0, %ymm0 + vaesenc %ymm24, %ymm0, %ymm0 + cmpl $11, %r8d + vmovdqa64 %ymm25, %ymm14 + jl L_AES_CTR_encrypt_avx512_32_aes_enc_block_last + vaesenc %ymm25, %ymm0, %ymm0 + vaesenc %ymm26, %ymm0, %ymm0 + cmpl $13, %r8d + vmovdqa64 %ymm27, %ymm14 + jl L_AES_CTR_encrypt_avx512_32_aes_enc_block_last + vaesenc %ymm27, %ymm0, %ymm0 + vaesenc %ymm28, %ymm0, %ymm0 + vmovdqa64 %ymm29, %ymm14 +L_AES_CTR_encrypt_avx512_32_aes_enc_block_last: + vaesenclast %ymm14, %ymm0, %ymm0 + leaq (%rdi,%rax,1), %r11 + vpxorq (%r11), %ymm0, %ymm0 + leaq (%rsi,%rax,1), %r11 + vmovdqu %ymm0, (%r11) + addl $32, %eax +L_AES_CTR_encrypt_avx512_done_32: cmpl %edx, %eax movl %edx, %r10d je L_AES_CTR_encrypt_avx512_done_enc @@ -4312,8 +5003,8 @@ L_AES_CTR_encrypt_avx512_enc_16: # 16 bytes of input vpshufb %xmm8, %xmm7, %xmm0 vmovdqa64 %zmm7, %zmm9 - vpaddq %zmm12, %zmm7, %zmm7 - vpternlogq $0xb2, %zmm12, %zmm7, %zmm9 + vpaddq %zmm13, %zmm7, %zmm7 + vpternlogq $0xb2, %zmm13, %zmm7, %zmm9 vpsrlq $63, %zmm9, %zmm9 vpslldq $8, %zmm9, %zmm9 vpaddq %zmm9, %zmm7, %zmm7 diff --git a/wolfcrypt/src/aes_x86_64_asm.asm b/wolfcrypt/src/aes_x86_64_asm.asm index 26ccbb5ee8..70899aadde 100644 --- a/wolfcrypt/src/aes_x86_64_asm.asm +++ b/wolfcrypt/src/aes_x86_64_asm.asm @@ -2170,6 +2170,71 @@ L_AES_ECB_encrypt_vaes_128_aes_enc_block_last: cmp eax, r9d jl L_AES_ECB_encrypt_vaes_enc_128 L_AES_ECB_encrypt_vaes_done_128: + mov r9d, r8d + sub r9d, eax + cmp r9d, 64 + jl L_AES_ECB_encrypt_vaes_done_64 + ; 64 bytes of input + ; aes_ecb_enc_64 + lea r10, QWORD PTR [rcx+rax] + lea r11, QWORD PTR [rdx+rax] + vmovdqu ymm0, YMMWORD PTR [r10] + vmovdqu ymm1, YMMWORD PTR [r10+32] + ; aes_enc_block + vbroadcasti128 ymm7, [r9] + vpxor ymm0, ymm0, ymm7 + vpxor ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+16] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+32] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+48] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+64] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+80] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+96] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+112] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+128] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+144] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + cmp eax, 11 + vbroadcasti128 ymm7, [r9+160] + jl L_AES_ECB_encrypt_vaes_64_aes_enc_block_last + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+176] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + cmp eax, 13 + vbroadcasti128 ymm7, [r9+192] + jl L_AES_ECB_encrypt_vaes_64_aes_enc_block_last + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+208] + vaesenc ymm0, ymm0, ymm7 + vaesenc ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+224] +L_AES_ECB_encrypt_vaes_64_aes_enc_block_last: + vaesenclast ymm0, ymm0, ymm7 + vaesenclast ymm1, ymm1, ymm7 + vmovdqu YMMWORD PTR [r11], ymm0 + vmovdqu YMMWORD PTR [r11+32], ymm1 + add eax, 64 +L_AES_ECB_encrypt_vaes_done_64: mov r9d, r8d and r9d, 4294967264 cmp eax, r9d @@ -2385,6 +2450,71 @@ L_AES_ECB_decrypt_vaes_128_aes_dec_block_last: cmp eax, r9d jl L_AES_ECB_decrypt_vaes_dec_128 L_AES_ECB_decrypt_vaes_done_128: + mov r9d, r8d + sub r9d, eax + cmp r9d, 64 + jl L_AES_ECB_decrypt_vaes_done_64 + ; 64 bytes of input + ; aes_ecb_dec_64 + lea r10, QWORD PTR [rcx+rax] + lea r11, QWORD PTR [rdx+rax] + vmovdqu ymm0, YMMWORD PTR [r10] + vmovdqu ymm1, YMMWORD PTR [r10+32] + ; aes_dec_block + vbroadcasti128 ymm7, [r9] + vpxor ymm0, ymm0, ymm7 + vpxor ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+16] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+32] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+48] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+64] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+80] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+96] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+112] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+128] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+144] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + cmp eax, 11 + vbroadcasti128 ymm7, [r9+160] + jl L_AES_ECB_decrypt_vaes_64_aes_dec_block_last + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+176] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + cmp eax, 13 + vbroadcasti128 ymm7, [r9+192] + jl L_AES_ECB_decrypt_vaes_64_aes_dec_block_last + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+208] + vaesdec ymm0, ymm0, ymm7 + vaesdec ymm1, ymm1, ymm7 + vbroadcasti128 ymm7, [r9+224] +L_AES_ECB_decrypt_vaes_64_aes_dec_block_last: + vaesdeclast ymm0, ymm0, ymm7 + vaesdeclast ymm1, ymm1, ymm7 + vmovdqu YMMWORD PTR [r11], ymm0 + vmovdqu YMMWORD PTR [r11+32], ymm1 + add eax, 64 +L_AES_ECB_decrypt_vaes_done_64: mov r9d, r8d and r9d, 4294967264 cmp eax, r9d @@ -2677,6 +2807,76 @@ L_AES_CBC_decrypt_vaes_128_aes_dec_block_last: cmp eax, r10d jl L_AES_CBC_decrypt_vaes_dec_128 L_AES_CBC_decrypt_vaes_done_128: + mov r10d, r9d + sub r10d, eax + cmp r10d, 64 + jl L_AES_CBC_decrypt_vaes_done_64 + ; 64 bytes of input + ; aes_cbc_dec_64 + lea r11, QWORD PTR [rcx+rax] + lea r12, QWORD PTR [rdx+rax] + vmovdqu ymm0, YMMWORD PTR [r11] + vmovdqu ymm1, YMMWORD PTR [r11+32] + vinserti128 ymm10, ymm8, xmm0, 1 + vmovdqu ymm11, YMMWORD PTR [r11+16] + vextracti128 xmm8, ymm1, 1 + ; aes_dec_block + vbroadcasti128 ymm9, [rax] + vpxor ymm0, ymm0, ymm9 + vpxor ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+16] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+32] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+48] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+64] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+80] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+96] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+112] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+128] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+144] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + cmp r10d, 11 + vbroadcasti128 ymm9, [rax+160] + jl L_AES_CBC_decrypt_vaes_64_aes_dec_block_last + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+176] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + cmp r10d, 13 + vbroadcasti128 ymm9, [rax+192] + jl L_AES_CBC_decrypt_vaes_64_aes_dec_block_last + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+208] + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm1, ymm1, ymm9 + vbroadcasti128 ymm9, [rax+224] +L_AES_CBC_decrypt_vaes_64_aes_dec_block_last: + vaesdeclast ymm0, ymm0, ymm9 + vaesdeclast ymm1, ymm1, ymm9 + vpxor ymm0, ymm0, ymm10 + vpxor ymm1, ymm1, ymm11 + vmovdqu YMMWORD PTR [r12], ymm0 + vmovdqu YMMWORD PTR [r12+32], ymm1 + add eax, 64 +L_AES_CBC_decrypt_vaes_done_64: mov r10d, r9d and r10d, 4294967264 cmp eax, r10d @@ -2830,7 +3030,7 @@ AES_CTR_encrypt_vaes PROC push rbx mov eax, DWORD PTR [rsp+48] mov r10, QWORD PTR [rsp+56] - sub rsp, 144 + sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 vmovdqu OWORD PTR [rsp+32], xmm8 @@ -2840,50 +3040,51 @@ AES_CTR_encrypt_vaes PROC vmovdqu OWORD PTR [rsp+96], xmm12 vmovdqu OWORD PTR [rsp+112], xmm13 vmovdqu OWORD PTR [rsp+128], xmm14 + vmovdqu OWORD PTR [rsp+144], xmm15 vbroadcasti128 ymm8, ptr_L_aes_ctr_bswap_vaes vbroadcasti128 ymm7, [r10] vpshufb ymm7, ymm7, ymm8 - vbroadcasti128 ymm10, [ptr_L_aes_ctr_inc_vaes+128] - vbroadcasti128 ymm11, [ptr_L_aes_ctr_inc_vaes+32] - vbroadcasti128 ymm12, [ptr_L_aes_ctr_inc_vaes+16] + vbroadcasti128 ymm12, [ptr_L_aes_ctr_inc_vaes+32] + vbroadcasti128 ymm13, [ptr_L_aes_ctr_inc_vaes+16] xor eax, eax cmp r8d, 128 mov r10d, r8d jl L_AES_CTR_encrypt_vaes_done_128 and r10d, 4294967168 + vbroadcasti128 ymm10, [ptr_L_aes_ctr_inc_vaes+128] vmovdqa ymm9, ymm7 vpaddq ymm4, ymm7, [ptr_L_aes_ctr_inc_vaes] - vpand ymm14, ymm9, [ptr_L_aes_ctr_inc_vaes] + vpand ymm15, ymm9, [ptr_L_aes_ctr_inc_vaes] vpor ymm9, ymm9, [ptr_L_aes_ctr_inc_vaes] vpandn ymm9, ymm4, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm4, ymm4, ymm9 vmovdqa ymm9, ymm7 vpaddq ymm5, ymm7, [ptr_L_aes_ctr_inc_vaes+32] - vpand ymm14, ymm9, [ptr_L_aes_ctr_inc_vaes+32] + vpand ymm15, ymm9, [ptr_L_aes_ctr_inc_vaes+32] vpor ymm9, ymm9, [ptr_L_aes_ctr_inc_vaes+32] vpandn ymm9, ymm5, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm5, ymm5, ymm9 vmovdqa ymm9, ymm7 vpaddq ymm6, ymm7, [ptr_L_aes_ctr_inc_vaes+64] - vpand ymm14, ymm9, [ptr_L_aes_ctr_inc_vaes+64] + vpand ymm15, ymm9, [ptr_L_aes_ctr_inc_vaes+64] vpor ymm9, ymm9, [ptr_L_aes_ctr_inc_vaes+64] vpandn ymm9, ymm6, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm6, ymm6, ymm9 vmovdqa ymm9, ymm7 vpaddq ymm7, ymm7, [ptr_L_aes_ctr_inc_vaes+96] - vpand ymm14, ymm9, [ptr_L_aes_ctr_inc_vaes+96] + vpand ymm15, ymm9, [ptr_L_aes_ctr_inc_vaes+96] vpor ymm9, ymm9, [ptr_L_aes_ctr_inc_vaes+96] vpandn ymm9, ymm7, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm7, ymm7, ymm9 @@ -2897,121 +3098,121 @@ L_AES_CTR_encrypt_vaes_enc_128: vpshufb ymm3, ymm7, ymm8 vmovdqa ymm9, ymm4 vpaddq ymm4, ymm4, ymm10 - vpand ymm14, ymm9, ymm10 + vpand ymm15, ymm9, ymm10 vpor ymm9, ymm9, ymm10 vpandn ymm9, ymm4, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm4, ymm4, ymm9 vmovdqa ymm9, ymm5 vpaddq ymm5, ymm5, ymm10 - vpand ymm14, ymm9, ymm10 + vpand ymm15, ymm9, ymm10 vpor ymm9, ymm9, ymm10 vpandn ymm9, ymm5, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm5, ymm5, ymm9 vmovdqa ymm9, ymm6 vpaddq ymm6, ymm6, ymm10 - vpand ymm14, ymm9, ymm10 + vpand ymm15, ymm9, ymm10 vpor ymm9, ymm9, ymm10 vpandn ymm9, ymm6, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm6, ymm6, ymm9 vmovdqa ymm9, ymm7 vpaddq ymm7, ymm7, ymm10 - vpand ymm14, ymm9, ymm10 + vpand ymm15, ymm9, ymm10 vpor ymm9, ymm9, ymm10 vpandn ymm9, ymm7, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm7, ymm7, ymm9 ; aes_enc_block - vbroadcasti128 ymm13, [r9] - vpxor ymm0, ymm0, ymm13 - vpxor ymm1, ymm1, ymm13 - vpxor ymm2, ymm2, ymm13 - vpxor ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+16] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+32] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+48] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+64] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+80] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+96] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+112] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+128] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+144] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 + vbroadcasti128 ymm14, [r9] + vpxor ymm0, ymm0, ymm14 + vpxor ymm1, ymm1, ymm14 + vpxor ymm2, ymm2, ymm14 + vpxor ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+16] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+32] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+48] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+64] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+80] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+96] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+112] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+128] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+144] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 cmp eax, 11 - vbroadcasti128 ymm13, [r9+160] + vbroadcasti128 ymm14, [r9+160] jl L_AES_CTR_encrypt_vaes_128_aes_enc_block_last - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+176] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+176] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 cmp eax, 13 - vbroadcasti128 ymm13, [r9+192] + vbroadcasti128 ymm14, [r9+192] jl L_AES_CTR_encrypt_vaes_128_aes_enc_block_last - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+208] - vaesenc ymm0, ymm0, ymm13 - vaesenc ymm1, ymm1, ymm13 - vaesenc ymm2, ymm2, ymm13 - vaesenc ymm3, ymm3, ymm13 - vbroadcasti128 ymm13, [r9+224] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+208] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vaesenc ymm2, ymm2, ymm14 + vaesenc ymm3, ymm3, ymm14 + vbroadcasti128 ymm14, [r9+224] L_AES_CTR_encrypt_vaes_128_aes_enc_block_last: - vaesenclast ymm0, ymm0, ymm13 - vaesenclast ymm1, ymm1, ymm13 - vaesenclast ymm2, ymm2, ymm13 - vaesenclast ymm3, ymm3, ymm13 + vaesenclast ymm0, ymm0, ymm14 + vaesenclast ymm1, ymm1, ymm14 + vaesenclast ymm2, ymm2, ymm14 + vaesenclast ymm3, ymm3, ymm14 vpxor ymm0, ymm0, [r11] vpxor ymm1, ymm1, [r11+32] vpxor ymm2, ymm2, [r11+64] @@ -3025,6 +3226,101 @@ L_AES_CTR_encrypt_vaes_128_aes_enc_block_last: jl L_AES_CTR_encrypt_vaes_enc_128 vperm2i128 ymm7, ymm4, ymm4, 0 L_AES_CTR_encrypt_vaes_done_128: + mov r10d, r8d + sub r10d, eax + cmp r10d, 64 + jl L_AES_CTR_encrypt_vaes_done_64 + ; 64 bytes of input + vbroadcasti128 ymm11, [ptr_L_aes_ctr_inc_vaes+64] + ; aes_ctr_enc_64 + lea r11, QWORD PTR [rcx+rax] + lea rbx, QWORD PTR [rdx+rax] + vpaddq ymm0, ymm7, [ptr_L_aes_ctr_inc_vaes] + vmovdqa ymm9, ymm7 + vpand ymm15, ymm9, [ptr_L_aes_ctr_inc_vaes] + vpor ymm9, ymm9, [ptr_L_aes_ctr_inc_vaes] + vpandn ymm9, ymm0, ymm9 + vpor ymm9, ymm9, ymm15 + vpsrlq ymm9, ymm9, 63 + vpslldq ymm9, ymm9, 8 + vpaddq ymm0, ymm0, ymm9 + vpshufb ymm0, ymm0, ymm8 + vpaddq ymm1, ymm7, [ptr_L_aes_ctr_inc_vaes+32] + vmovdqa ymm9, ymm7 + vpand ymm15, ymm9, [ptr_L_aes_ctr_inc_vaes+32] + vpor ymm9, ymm9, [ptr_L_aes_ctr_inc_vaes+32] + vpandn ymm9, ymm1, ymm9 + vpor ymm9, ymm9, ymm15 + vpsrlq ymm9, ymm9, 63 + vpslldq ymm9, ymm9, 8 + vpaddq ymm1, ymm1, ymm9 + vpshufb ymm1, ymm1, ymm8 + vmovdqa ymm9, ymm7 + vpaddq ymm7, ymm7, ymm11 + vpand ymm15, ymm9, ymm11 + vpor ymm9, ymm9, ymm11 + vpandn ymm9, ymm7, ymm9 + vpor ymm9, ymm9, ymm15 + vpsrlq ymm9, ymm9, 63 + vpslldq ymm9, ymm9, 8 + vpaddq ymm7, ymm7, ymm9 + ; aes_enc_block + vbroadcasti128 ymm14, [r9] + vpxor ymm0, ymm0, ymm14 + vpxor ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+16] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+32] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+48] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+64] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+80] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+96] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+112] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+128] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+144] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + cmp eax, 11 + vbroadcasti128 ymm14, [r9+160] + jl L_AES_CTR_encrypt_vaes_64_aes_enc_block_last + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+176] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + cmp eax, 13 + vbroadcasti128 ymm14, [r9+192] + jl L_AES_CTR_encrypt_vaes_64_aes_enc_block_last + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+208] + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm1, ymm1, ymm14 + vbroadcasti128 ymm14, [r9+224] +L_AES_CTR_encrypt_vaes_64_aes_enc_block_last: + vaesenclast ymm0, ymm0, ymm14 + vaesenclast ymm1, ymm1, ymm14 + vpxor ymm0, ymm0, [r11] + vpxor ymm1, ymm1, [r11+32] + vmovdqu YMMWORD PTR [rbx], ymm0 + vmovdqu YMMWORD PTR [rbx+32], ymm1 + add eax, 64 +L_AES_CTR_encrypt_vaes_done_64: mov r10d, r8d and r10d, 4294967264 cmp eax, r10d @@ -3036,59 +3332,59 @@ L_AES_CTR_encrypt_vaes_enc_32: lea rbx, QWORD PTR [rdx+rax] vpaddq ymm0, ymm7, [ptr_L_aes_ctr_inc_vaes] vmovdqa ymm9, ymm7 - vpand ymm14, ymm9, [ptr_L_aes_ctr_inc_vaes] + vpand ymm15, ymm9, [ptr_L_aes_ctr_inc_vaes] vpor ymm9, ymm9, [ptr_L_aes_ctr_inc_vaes] vpandn ymm9, ymm0, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm0, ymm0, ymm9 vpshufb ymm0, ymm0, ymm8 vmovdqa ymm9, ymm7 - vpaddq ymm7, ymm7, ymm11 - vpand ymm14, ymm9, ymm11 - vpor ymm9, ymm9, ymm11 + vpaddq ymm7, ymm7, ymm12 + vpand ymm15, ymm9, ymm12 + vpor ymm9, ymm9, ymm12 vpandn ymm9, ymm7, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm7, ymm7, ymm9 ; aes_enc_block - vbroadcasti128 ymm13, [r9] - vpxor ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+16] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+32] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+48] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+64] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+80] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+96] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+112] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+128] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+144] - vaesenc ymm0, ymm0, ymm13 + vbroadcasti128 ymm14, [r9] + vpxor ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+16] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+32] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+48] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+64] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+80] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+96] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+112] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+128] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+144] + vaesenc ymm0, ymm0, ymm14 cmp eax, 11 - vbroadcasti128 ymm13, [r9+160] + vbroadcasti128 ymm14, [r9+160] jl L_AES_CTR_encrypt_vaes_32_aes_enc_block_last - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+176] - vaesenc ymm0, ymm0, ymm13 + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+176] + vaesenc ymm0, ymm0, ymm14 cmp eax, 13 - vbroadcasti128 ymm13, [r9+192] + vbroadcasti128 ymm14, [r9+192] jl L_AES_CTR_encrypt_vaes_32_aes_enc_block_last - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+208] - vaesenc ymm0, ymm0, ymm13 - vbroadcasti128 ymm13, [r9+224] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+208] + vaesenc ymm0, ymm0, ymm14 + vbroadcasti128 ymm14, [r9+224] L_AES_CTR_encrypt_vaes_32_aes_enc_block_last: - vaesenclast ymm0, ymm0, ymm13 + vaesenclast ymm0, ymm0, ymm14 vpxor ymm0, ymm0, [r11] vmovdqu YMMWORD PTR [rbx], ymm0 add eax, 32 @@ -3103,11 +3399,11 @@ L_AES_CTR_encrypt_vaes_enc_16: ; 16 bytes of input vpshufb xmm0, xmm7, xmm8 vmovdqa ymm9, ymm7 - vpaddq ymm7, ymm7, ymm12 - vpand ymm14, ymm9, ymm12 - vpor ymm9, ymm9, ymm12 + vpaddq ymm7, ymm7, ymm13 + vpand ymm15, ymm9, ymm13 + vpor ymm9, ymm9, ymm13 vpandn ymm9, ymm7, ymm9 - vpor ymm9, ymm9, ymm14 + vpor ymm9, ymm9, ymm15 vpsrlq ymm9, ymm9, 63 vpslldq ymm9, ymm9, 8 vpaddq ymm7, ymm7, ymm9 @@ -3165,7 +3461,8 @@ L_AES_CTR_encrypt_vaes_done_enc: vmovdqu xmm12, OWORD PTR [rsp+96] vmovdqu xmm13, OWORD PTR [rsp+112] vmovdqu xmm14, OWORD PTR [rsp+128] - add rsp, 144 + vmovdqu xmm15, OWORD PTR [rsp+144] + add rsp, 160 pop rbx ret AES_CTR_encrypt_vaes ENDP @@ -3187,8 +3484,8 @@ AES_ECB_encrypt_avx512 PROC vmovdqu OWORD PTR [rsp+128], xmm14 vmovdqu OWORD PTR [rsp+144], xmm15 xor eax, eax - cmp r8d, 64 - jl L_AES_ECB_encrypt_avx512_done_64 + cmp r8d, 32 + jl L_AES_ECB_encrypt_avx512_done_32 vbroadcasti32x4 zmm8, [r9] vbroadcasti32x4 zmm9, [r9+16] vbroadcasti32x4 zmm10, [r9+32] @@ -3299,6 +3596,59 @@ L_AES_ECB_encrypt_avx512_256_aes_enc_block_last: cmp eax, r9d jl L_AES_ECB_encrypt_avx512_enc_256 L_AES_ECB_encrypt_avx512_done_256: + mov r9d, r8d + sub r9d, eax + cmp r9d, 128 + jl L_AES_ECB_encrypt_avx512_done_128 + ; 128 bytes of input + ; aes_ecb_enc_128 + lea r10, QWORD PTR [rcx+rax] + lea r11, QWORD PTR [rdx+rax] + vmovdqu64 zmm0, [r10] + vmovdqu64 zmm1, [r10+64] + ; aes_enc_block + vpxorq zmm0, zmm0, zmm8 + vpxorq zmm1, zmm1, zmm8 + vaesenc zmm0, zmm0, zmm9 + vaesenc zmm1, zmm1, zmm9 + vaesenc zmm0, zmm0, zmm10 + vaesenc zmm1, zmm1, zmm10 + vaesenc zmm0, zmm0, zmm11 + vaesenc zmm1, zmm1, zmm11 + vaesenc zmm0, zmm0, zmm12 + vaesenc zmm1, zmm1, zmm12 + vaesenc zmm0, zmm0, zmm13 + vaesenc zmm1, zmm1, zmm13 + vaesenc zmm0, zmm0, zmm14 + vaesenc zmm1, zmm1, zmm14 + vaesenc zmm0, zmm0, zmm15 + vaesenc zmm1, zmm1, zmm15 + vaesenc zmm0, zmm0, zmm16 + vaesenc zmm1, zmm1, zmm16 + vaesenc zmm0, zmm0, zmm17 + vaesenc zmm1, zmm1, zmm17 + cmp eax, 11 + vmovdqa64 zmm7, zmm18 + jl L_AES_ECB_encrypt_avx512_128_aes_enc_block_last + vaesenc zmm0, zmm0, zmm18 + vaesenc zmm1, zmm1, zmm18 + vaesenc zmm0, zmm0, zmm19 + vaesenc zmm1, zmm1, zmm19 + cmp eax, 13 + vmovdqa64 zmm7, zmm20 + jl L_AES_ECB_encrypt_avx512_128_aes_enc_block_last + vaesenc zmm0, zmm0, zmm20 + vaesenc zmm1, zmm1, zmm20 + vaesenc zmm0, zmm0, zmm21 + vaesenc zmm1, zmm1, zmm21 + vmovdqa64 zmm7, zmm22 +L_AES_ECB_encrypt_avx512_128_aes_enc_block_last: + vaesenclast zmm0, zmm0, zmm7 + vaesenclast zmm1, zmm1, zmm7 + vmovdqu64 [r11], zmm0 + vmovdqu64 [r11+64], zmm1 + add eax, 128 +L_AES_ECB_encrypt_avx512_done_128: mov r9d, r8d and r9d, 4294967232 cmp eax, r9d @@ -3338,6 +3688,41 @@ L_AES_ECB_encrypt_avx512_64_aes_enc_block_last: cmp eax, r9d jl L_AES_ECB_encrypt_avx512_enc_64 L_AES_ECB_encrypt_avx512_done_64: + mov r9d, r8d + sub r9d, eax + cmp r9d, 32 + jl L_AES_ECB_encrypt_avx512_done_32 + ; 32 bytes of input + lea r10, QWORD PTR [rcx+rax] + vmovdqu ymm0, YMMWORD PTR [r10] + ; aes_enc_block + vpxorq ymm0, ymm0, ymm8 + vaesenc ymm0, ymm0, ymm9 + vaesenc ymm0, ymm0, ymm10 + vaesenc ymm0, ymm0, ymm11 + vaesenc ymm0, ymm0, ymm12 + vaesenc ymm0, ymm0, ymm13 + vaesenc ymm0, ymm0, ymm14 + vaesenc ymm0, ymm0, ymm15 + vaesenc ymm0, ymm0, ymm16 + vaesenc ymm0, ymm0, ymm17 + cmp eax, 11 + vmovdqa64 ymm7, ymm18 + jl L_AES_ECB_encrypt_avx512_32_aes_enc_block_last + vaesenc ymm0, ymm0, ymm18 + vaesenc ymm0, ymm0, ymm19 + cmp eax, 13 + vmovdqa64 ymm7, ymm20 + jl L_AES_ECB_encrypt_avx512_32_aes_enc_block_last + vaesenc ymm0, ymm0, ymm20 + vaesenc ymm0, ymm0, ymm21 + vmovdqa64 ymm7, ymm22 +L_AES_ECB_encrypt_avx512_32_aes_enc_block_last: + vaesenclast ymm0, ymm0, ymm7 + lea r10, QWORD PTR [rdx+rax] + vmovdqu YMMWORD PTR [r10], ymm0 + add eax, 32 +L_AES_ECB_encrypt_avx512_done_32: cmp eax, r8d mov r9d, r8d je L_AES_ECB_encrypt_avx512_done_enc @@ -3416,8 +3801,8 @@ AES_ECB_decrypt_avx512 PROC vmovdqu OWORD PTR [rsp+128], xmm14 vmovdqu OWORD PTR [rsp+144], xmm15 xor eax, eax - cmp r8d, 64 - jl L_AES_ECB_decrypt_avx512_done_64 + cmp r8d, 32 + jl L_AES_ECB_decrypt_avx512_done_32 vbroadcasti32x4 zmm8, [r9] vbroadcasti32x4 zmm9, [r9+16] vbroadcasti32x4 zmm10, [r9+32] @@ -3528,6 +3913,59 @@ L_AES_ECB_decrypt_avx512_256_aes_dec_block_last: cmp eax, r9d jl L_AES_ECB_decrypt_avx512_dec_256 L_AES_ECB_decrypt_avx512_done_256: + mov r9d, r8d + sub r9d, eax + cmp r9d, 128 + jl L_AES_ECB_decrypt_avx512_done_128 + ; 128 bytes of input + ; aes_ecb_dec_128 + lea r10, QWORD PTR [rcx+rax] + lea r11, QWORD PTR [rdx+rax] + vmovdqu64 zmm0, [r10] + vmovdqu64 zmm1, [r10+64] + ; aes_dec_block + vpxorq zmm0, zmm0, zmm8 + vpxorq zmm1, zmm1, zmm8 + vaesdec zmm0, zmm0, zmm9 + vaesdec zmm1, zmm1, zmm9 + vaesdec zmm0, zmm0, zmm10 + vaesdec zmm1, zmm1, zmm10 + vaesdec zmm0, zmm0, zmm11 + vaesdec zmm1, zmm1, zmm11 + vaesdec zmm0, zmm0, zmm12 + vaesdec zmm1, zmm1, zmm12 + vaesdec zmm0, zmm0, zmm13 + vaesdec zmm1, zmm1, zmm13 + vaesdec zmm0, zmm0, zmm14 + vaesdec zmm1, zmm1, zmm14 + vaesdec zmm0, zmm0, zmm15 + vaesdec zmm1, zmm1, zmm15 + vaesdec zmm0, zmm0, zmm16 + vaesdec zmm1, zmm1, zmm16 + vaesdec zmm0, zmm0, zmm17 + vaesdec zmm1, zmm1, zmm17 + cmp eax, 11 + vmovdqa64 zmm7, zmm18 + jl L_AES_ECB_decrypt_avx512_128_aes_dec_block_last + vaesdec zmm0, zmm0, zmm18 + vaesdec zmm1, zmm1, zmm18 + vaesdec zmm0, zmm0, zmm19 + vaesdec zmm1, zmm1, zmm19 + cmp eax, 13 + vmovdqa64 zmm7, zmm20 + jl L_AES_ECB_decrypt_avx512_128_aes_dec_block_last + vaesdec zmm0, zmm0, zmm20 + vaesdec zmm1, zmm1, zmm20 + vaesdec zmm0, zmm0, zmm21 + vaesdec zmm1, zmm1, zmm21 + vmovdqa64 zmm7, zmm22 +L_AES_ECB_decrypt_avx512_128_aes_dec_block_last: + vaesdeclast zmm0, zmm0, zmm7 + vaesdeclast zmm1, zmm1, zmm7 + vmovdqu64 [r11], zmm0 + vmovdqu64 [r11+64], zmm1 + add eax, 128 +L_AES_ECB_decrypt_avx512_done_128: mov r9d, r8d and r9d, 4294967232 cmp eax, r9d @@ -3567,6 +4005,41 @@ L_AES_ECB_decrypt_avx512_64_aes_dec_block_last: cmp eax, r9d jl L_AES_ECB_decrypt_avx512_dec_64 L_AES_ECB_decrypt_avx512_done_64: + mov r9d, r8d + sub r9d, eax + cmp r9d, 32 + jl L_AES_ECB_decrypt_avx512_done_32 + ; 32 bytes of input + lea r10, QWORD PTR [rcx+rax] + vmovdqu ymm0, YMMWORD PTR [r10] + ; aes_dec_block + vpxorq ymm0, ymm0, ymm8 + vaesdec ymm0, ymm0, ymm9 + vaesdec ymm0, ymm0, ymm10 + vaesdec ymm0, ymm0, ymm11 + vaesdec ymm0, ymm0, ymm12 + vaesdec ymm0, ymm0, ymm13 + vaesdec ymm0, ymm0, ymm14 + vaesdec ymm0, ymm0, ymm15 + vaesdec ymm0, ymm0, ymm16 + vaesdec ymm0, ymm0, ymm17 + cmp eax, 11 + vmovdqa64 ymm7, ymm18 + jl L_AES_ECB_decrypt_avx512_32_aes_dec_block_last + vaesdec ymm0, ymm0, ymm18 + vaesdec ymm0, ymm0, ymm19 + cmp eax, 13 + vmovdqa64 ymm7, ymm20 + jl L_AES_ECB_decrypt_avx512_32_aes_dec_block_last + vaesdec ymm0, ymm0, ymm20 + vaesdec ymm0, ymm0, ymm21 + vmovdqa64 ymm7, ymm22 +L_AES_ECB_decrypt_avx512_32_aes_dec_block_last: + vaesdeclast ymm0, ymm0, ymm7 + lea r10, QWORD PTR [rdx+rax] + vmovdqu YMMWORD PTR [r10], ymm0 + add eax, 32 +L_AES_ECB_decrypt_avx512_done_32: cmp eax, r8d mov r9d, r8d je L_AES_ECB_decrypt_avx512_done_dec @@ -3706,8 +4179,8 @@ AES_CBC_decrypt_avx512 PROC vmovdqu OWORD PTR [rsp+144], xmm15 vmovdqu xmm8, OWORD PTR [r8] xor eax, eax - cmp r9d, 64 - jl L_AES_CBC_decrypt_avx512_done_64 + cmp r9d, 32 + jl L_AES_CBC_decrypt_avx512_done_32 vbroadcasti32x4 zmm14, [rax] vbroadcasti32x4 zmm15, [rax+16] vbroadcasti32x4 zmm16, [rax+32] @@ -3828,6 +4301,65 @@ L_AES_CBC_decrypt_avx512_256_aes_dec_block_last: cmp eax, r10d jl L_AES_CBC_decrypt_avx512_dec_256 L_AES_CBC_decrypt_avx512_done_256: + mov r10d, r9d + sub r10d, eax + cmp r10d, 128 + jl L_AES_CBC_decrypt_avx512_done_128 + ; 128 bytes of input + ; aes_cbc_dec_128 + lea r11, QWORD PTR [rcx+rax] + lea r12, QWORD PTR [rdx+rax] + vmovdqu64 zmm0, [r11] + vmovdqu64 zmm1, [r11+64] + vshufi64x2 zmm10, zmm0, zmm0, 144 + vinserti32x4 zmm10, zmm10, xmm8, 0 + vmovdqu64 zmm11, [r11+48] + vextracti32x4 xmm8, zmm1, 3 + ; aes_dec_block + vpxorq zmm0, zmm0, zmm14 + vpxorq zmm1, zmm1, zmm14 + vaesdec zmm0, zmm0, zmm15 + vaesdec zmm1, zmm1, zmm15 + vaesdec zmm0, zmm0, zmm16 + vaesdec zmm1, zmm1, zmm16 + vaesdec zmm0, zmm0, zmm17 + vaesdec zmm1, zmm1, zmm17 + vaesdec zmm0, zmm0, zmm18 + vaesdec zmm1, zmm1, zmm18 + vaesdec zmm0, zmm0, zmm19 + vaesdec zmm1, zmm1, zmm19 + vaesdec zmm0, zmm0, zmm20 + vaesdec zmm1, zmm1, zmm20 + vaesdec zmm0, zmm0, zmm21 + vaesdec zmm1, zmm1, zmm21 + vaesdec zmm0, zmm0, zmm22 + vaesdec zmm1, zmm1, zmm22 + vaesdec zmm0, zmm0, zmm23 + vaesdec zmm1, zmm1, zmm23 + cmp r10d, 11 + vmovdqa64 zmm9, zmm24 + jl L_AES_CBC_decrypt_avx512_128_aes_dec_block_last + vaesdec zmm0, zmm0, zmm24 + vaesdec zmm1, zmm1, zmm24 + vaesdec zmm0, zmm0, zmm25 + vaesdec zmm1, zmm1, zmm25 + cmp r10d, 13 + vmovdqa64 zmm9, zmm26 + jl L_AES_CBC_decrypt_avx512_128_aes_dec_block_last + vaesdec zmm0, zmm0, zmm26 + vaesdec zmm1, zmm1, zmm26 + vaesdec zmm0, zmm0, zmm27 + vaesdec zmm1, zmm1, zmm27 + vmovdqa64 zmm9, zmm28 +L_AES_CBC_decrypt_avx512_128_aes_dec_block_last: + vaesdeclast zmm0, zmm0, zmm9 + vaesdeclast zmm1, zmm1, zmm9 + vpxorq zmm0, zmm0, zmm10 + vpxorq zmm1, zmm1, zmm11 + vmovdqu64 [r12], zmm0 + vmovdqu64 [r12+64], zmm1 + add eax, 128 +L_AES_CBC_decrypt_avx512_done_128: mov r10d, r9d and r10d, 4294967232 cmp eax, r10d @@ -3871,6 +4403,44 @@ L_AES_CBC_decrypt_avx512_64_aes_dec_block_last: cmp eax, r10d jl L_AES_CBC_decrypt_avx512_dec_64 L_AES_CBC_decrypt_avx512_done_64: + mov r10d, r9d + sub r10d, eax + cmp r10d, 32 + jl L_AES_CBC_decrypt_avx512_done_32 + ; 32 bytes of input + lea r11, QWORD PTR [rcx+rax] + vmovdqu ymm0, YMMWORD PTR [r11] + vinserti128 ymm10, ymm8, xmm0, 1 + vextracti128 xmm8, ymm0, 1 + ; aes_dec_block + vpxorq ymm0, ymm0, ymm14 + vaesdec ymm0, ymm0, ymm15 + vaesdec ymm0, ymm0, ymm16 + vaesdec ymm0, ymm0, ymm17 + vaesdec ymm0, ymm0, ymm18 + vaesdec ymm0, ymm0, ymm19 + vaesdec ymm0, ymm0, ymm20 + vaesdec ymm0, ymm0, ymm21 + vaesdec ymm0, ymm0, ymm22 + vaesdec ymm0, ymm0, ymm23 + cmp r10d, 11 + vmovdqa64 ymm9, ymm24 + jl L_AES_CBC_decrypt_avx512_32_aes_dec_block_last + vaesdec ymm0, ymm0, ymm24 + vaesdec ymm0, ymm0, ymm25 + cmp r10d, 13 + vmovdqa64 ymm9, ymm26 + jl L_AES_CBC_decrypt_avx512_32_aes_dec_block_last + vaesdec ymm0, ymm0, ymm26 + vaesdec ymm0, ymm0, ymm27 + vmovdqa64 ymm9, ymm28 +L_AES_CBC_decrypt_avx512_32_aes_dec_block_last: + vaesdeclast ymm0, ymm0, ymm9 + vpxorq ymm0, ymm0, ymm10 + lea r11, QWORD PTR [rdx+rax] + vmovdqu YMMWORD PTR [r11], ymm0 + add eax, 32 +L_AES_CBC_decrypt_avx512_done_32: cmp eax, r9d mov r10d, r9d je L_AES_CBC_decrypt_avx512_done_dec @@ -3986,36 +4556,36 @@ AES_CTR_encrypt_avx512 PROC vbroadcasti32x4 zmm8, ptr_L_aes_ctr_bswap_avx512 vbroadcasti32x4 zmm7, [r10] vpshufb zmm7, zmm7, zmm8 - vbroadcasti32x4 zmm10, [ptr_L_aes_ctr_inc_avx512+256] - vbroadcasti32x4 zmm11, [ptr_L_aes_ctr_inc_avx512+64] - vbroadcasti32x4 zmm12, [ptr_L_aes_ctr_inc_avx512+16] + vbroadcasti32x4 zmm12, [ptr_L_aes_ctr_inc_avx512+64] + vbroadcasti32x4 zmm13, [ptr_L_aes_ctr_inc_avx512+16] xor eax, eax - cmp r8d, 64 - jl L_AES_CTR_encrypt_avx512_done_64 - vbroadcasti32x4 zmm14, [r9] - vbroadcasti32x4 zmm15, [r9+16] - vbroadcasti32x4 zmm16, [r9+32] - vbroadcasti32x4 zmm17, [r9+48] - vbroadcasti32x4 zmm18, [r9+64] - vbroadcasti32x4 zmm19, [r9+80] - vbroadcasti32x4 zmm20, [r9+96] - vbroadcasti32x4 zmm21, [r9+112] - vbroadcasti32x4 zmm22, [r9+128] - vbroadcasti32x4 zmm23, [r9+144] - vbroadcasti32x4 zmm24, [r9+160] + cmp r8d, 32 + jl L_AES_CTR_encrypt_avx512_done_32 + vbroadcasti32x4 zmm15, [r9] + vbroadcasti32x4 zmm16, [r9+16] + vbroadcasti32x4 zmm17, [r9+32] + vbroadcasti32x4 zmm18, [r9+48] + vbroadcasti32x4 zmm19, [r9+64] + vbroadcasti32x4 zmm20, [r9+80] + vbroadcasti32x4 zmm21, [r9+96] + vbroadcasti32x4 zmm22, [r9+112] + vbroadcasti32x4 zmm23, [r9+128] + vbroadcasti32x4 zmm24, [r9+144] + vbroadcasti32x4 zmm25, [r9+160] cmp eax, 11 jl L_AES_CTR_encrypt_avx512_key_cached - vbroadcasti32x4 zmm25, [r9+176] - vbroadcasti32x4 zmm26, [r9+192] + vbroadcasti32x4 zmm26, [r9+176] + vbroadcasti32x4 zmm27, [r9+192] cmp eax, 13 jl L_AES_CTR_encrypt_avx512_key_cached - vbroadcasti32x4 zmm27, [r9+208] - vbroadcasti32x4 zmm28, [r9+224] + vbroadcasti32x4 zmm28, [r9+208] + vbroadcasti32x4 zmm29, [r9+224] L_AES_CTR_encrypt_avx512_key_cached: cmp r8d, 256 mov r10d, r8d jl L_AES_CTR_encrypt_avx512_done_256 and r10d, 4294967040 + vbroadcasti32x4 zmm10, [ptr_L_aes_ctr_inc_avx512+256] vmovdqa64 zmm9, zmm7 vpaddq zmm4, zmm7, [ptr_L_aes_ctr_inc_avx512] vpternlogq zmm9, zmm4, [ptr_L_aes_ctr_inc_avx512], 178 @@ -4073,14 +4643,10 @@ L_AES_CTR_encrypt_avx512_enc_256: vpslldq zmm9, zmm9, 8 vpaddq zmm7, zmm7, zmm9 ; aes_enc_block - vpxorq zmm0, zmm0, zmm14 - vpxorq zmm1, zmm1, zmm14 - vpxorq zmm2, zmm2, zmm14 - vpxorq zmm3, zmm3, zmm14 - vaesenc zmm0, zmm0, zmm15 - vaesenc zmm1, zmm1, zmm15 - vaesenc zmm2, zmm2, zmm15 - vaesenc zmm3, zmm3, zmm15 + vpxorq zmm0, zmm0, zmm15 + vpxorq zmm1, zmm1, zmm15 + vpxorq zmm2, zmm2, zmm15 + vpxorq zmm3, zmm3, zmm15 vaesenc zmm0, zmm0, zmm16 vaesenc zmm1, zmm1, zmm16 vaesenc zmm2, zmm2, zmm16 @@ -4113,34 +4679,38 @@ L_AES_CTR_encrypt_avx512_enc_256: vaesenc zmm1, zmm1, zmm23 vaesenc zmm2, zmm2, zmm23 vaesenc zmm3, zmm3, zmm23 - cmp eax, 11 - vmovdqa64 zmm13, zmm24 - jl L_AES_CTR_encrypt_avx512_256_aes_enc_block_last vaesenc zmm0, zmm0, zmm24 vaesenc zmm1, zmm1, zmm24 vaesenc zmm2, zmm2, zmm24 vaesenc zmm3, zmm3, zmm24 + cmp eax, 11 + vmovdqa64 zmm14, zmm25 + jl L_AES_CTR_encrypt_avx512_256_aes_enc_block_last vaesenc zmm0, zmm0, zmm25 vaesenc zmm1, zmm1, zmm25 vaesenc zmm2, zmm2, zmm25 vaesenc zmm3, zmm3, zmm25 - cmp eax, 13 - vmovdqa64 zmm13, zmm26 - jl L_AES_CTR_encrypt_avx512_256_aes_enc_block_last vaesenc zmm0, zmm0, zmm26 vaesenc zmm1, zmm1, zmm26 vaesenc zmm2, zmm2, zmm26 vaesenc zmm3, zmm3, zmm26 + cmp eax, 13 + vmovdqa64 zmm14, zmm27 + jl L_AES_CTR_encrypt_avx512_256_aes_enc_block_last vaesenc zmm0, zmm0, zmm27 vaesenc zmm1, zmm1, zmm27 vaesenc zmm2, zmm2, zmm27 vaesenc zmm3, zmm3, zmm27 - vmovdqa64 zmm13, zmm28 + vaesenc zmm0, zmm0, zmm28 + vaesenc zmm1, zmm1, zmm28 + vaesenc zmm2, zmm2, zmm28 + vaesenc zmm3, zmm3, zmm28 + vmovdqa64 zmm14, zmm29 L_AES_CTR_encrypt_avx512_256_aes_enc_block_last: - vaesenclast zmm0, zmm0, zmm13 - vaesenclast zmm1, zmm1, zmm13 - vaesenclast zmm2, zmm2, zmm13 - vaesenclast zmm3, zmm3, zmm13 + vaesenclast zmm0, zmm0, zmm14 + vaesenclast zmm1, zmm1, zmm14 + vaesenclast zmm2, zmm2, zmm14 + vaesenclast zmm3, zmm3, zmm14 vpxorq zmm0, zmm0, [r11] vpxorq zmm1, zmm1, [r11+64] vpxorq zmm2, zmm2, [r11+128] @@ -4154,6 +4724,80 @@ L_AES_CTR_encrypt_avx512_256_aes_enc_block_last: jl L_AES_CTR_encrypt_avx512_enc_256 vshufi64x2 zmm7, zmm4, zmm4, 0 L_AES_CTR_encrypt_avx512_done_256: + mov r10d, r8d + sub r10d, eax + cmp r10d, 128 + jl L_AES_CTR_encrypt_avx512_done_128 + ; 128 bytes of input + vbroadcasti32x4 zmm11, [ptr_L_aes_ctr_inc_avx512+128] + ; aes_ctr_enc_128 + lea r11, QWORD PTR [rcx+rax] + lea rbx, QWORD PTR [rdx+rax] + vpaddq zmm0, zmm7, [ptr_L_aes_ctr_inc_avx512] + vmovdqa64 zmm9, zmm7 + vpternlogq zmm9, zmm0, [ptr_L_aes_ctr_inc_avx512], 178 + vpsrlq zmm9, zmm9, 63 + vpslldq zmm9, zmm9, 8 + vpaddq zmm0, zmm0, zmm9 + vpshufb zmm0, zmm0, zmm8 + vpaddq zmm1, zmm7, [ptr_L_aes_ctr_inc_avx512+64] + vmovdqa64 zmm9, zmm7 + vpternlogq zmm9, zmm1, [ptr_L_aes_ctr_inc_avx512+64], 178 + vpsrlq zmm9, zmm9, 63 + vpslldq zmm9, zmm9, 8 + vpaddq zmm1, zmm1, zmm9 + vpshufb zmm1, zmm1, zmm8 + vmovdqa64 zmm9, zmm7 + vpaddq zmm7, zmm7, zmm11 + vpternlogq zmm9, zmm7, zmm11, 178 + vpsrlq zmm9, zmm9, 63 + vpslldq zmm9, zmm9, 8 + vpaddq zmm7, zmm7, zmm9 + ; aes_enc_block + vpxorq zmm0, zmm0, zmm15 + vpxorq zmm1, zmm1, zmm15 + vaesenc zmm0, zmm0, zmm16 + vaesenc zmm1, zmm1, zmm16 + vaesenc zmm0, zmm0, zmm17 + vaesenc zmm1, zmm1, zmm17 + vaesenc zmm0, zmm0, zmm18 + vaesenc zmm1, zmm1, zmm18 + vaesenc zmm0, zmm0, zmm19 + vaesenc zmm1, zmm1, zmm19 + vaesenc zmm0, zmm0, zmm20 + vaesenc zmm1, zmm1, zmm20 + vaesenc zmm0, zmm0, zmm21 + vaesenc zmm1, zmm1, zmm21 + vaesenc zmm0, zmm0, zmm22 + vaesenc zmm1, zmm1, zmm22 + vaesenc zmm0, zmm0, zmm23 + vaesenc zmm1, zmm1, zmm23 + vaesenc zmm0, zmm0, zmm24 + vaesenc zmm1, zmm1, zmm24 + cmp eax, 11 + vmovdqa64 zmm14, zmm25 + jl L_AES_CTR_encrypt_avx512_128_aes_enc_block_last + vaesenc zmm0, zmm0, zmm25 + vaesenc zmm1, zmm1, zmm25 + vaesenc zmm0, zmm0, zmm26 + vaesenc zmm1, zmm1, zmm26 + cmp eax, 13 + vmovdqa64 zmm14, zmm27 + jl L_AES_CTR_encrypt_avx512_128_aes_enc_block_last + vaesenc zmm0, zmm0, zmm27 + vaesenc zmm1, zmm1, zmm27 + vaesenc zmm0, zmm0, zmm28 + vaesenc zmm1, zmm1, zmm28 + vmovdqa64 zmm14, zmm29 +L_AES_CTR_encrypt_avx512_128_aes_enc_block_last: + vaesenclast zmm0, zmm0, zmm14 + vaesenclast zmm1, zmm1, zmm14 + vpxorq zmm0, zmm0, [r11] + vpxorq zmm1, zmm1, [r11+64] + vmovdqu64 [rbx], zmm0 + vmovdqu64 [rbx+64], zmm1 + add eax, 128 +L_AES_CTR_encrypt_avx512_done_128: mov r10d, r8d and r10d, 4294967232 cmp eax, r10d @@ -4171,14 +4815,13 @@ L_AES_CTR_encrypt_avx512_enc_64: vpaddq zmm0, zmm0, zmm9 vpshufb zmm0, zmm0, zmm8 vmovdqa64 zmm9, zmm7 - vpaddq zmm7, zmm7, zmm11 - vpternlogq zmm9, zmm7, zmm11, 178 + vpaddq zmm7, zmm7, zmm12 + vpternlogq zmm9, zmm7, zmm12, 178 vpsrlq zmm9, zmm9, 63 vpslldq zmm9, zmm9, 8 vpaddq zmm7, zmm7, zmm9 ; aes_enc_block - vpxorq zmm0, zmm0, zmm14 - vaesenc zmm0, zmm0, zmm15 + vpxorq zmm0, zmm0, zmm15 vaesenc zmm0, zmm0, zmm16 vaesenc zmm0, zmm0, zmm17 vaesenc zmm0, zmm0, zmm18 @@ -4187,25 +4830,75 @@ L_AES_CTR_encrypt_avx512_enc_64: vaesenc zmm0, zmm0, zmm21 vaesenc zmm0, zmm0, zmm22 vaesenc zmm0, zmm0, zmm23 - cmp eax, 11 - vmovdqa64 zmm13, zmm24 - jl L_AES_CTR_encrypt_avx512_64_aes_enc_block_last vaesenc zmm0, zmm0, zmm24 - vaesenc zmm0, zmm0, zmm25 - cmp eax, 13 - vmovdqa64 zmm13, zmm26 + cmp eax, 11 + vmovdqa64 zmm14, zmm25 jl L_AES_CTR_encrypt_avx512_64_aes_enc_block_last + vaesenc zmm0, zmm0, zmm25 vaesenc zmm0, zmm0, zmm26 + cmp eax, 13 + vmovdqa64 zmm14, zmm27 + jl L_AES_CTR_encrypt_avx512_64_aes_enc_block_last vaesenc zmm0, zmm0, zmm27 - vmovdqa64 zmm13, zmm28 + vaesenc zmm0, zmm0, zmm28 + vmovdqa64 zmm14, zmm29 L_AES_CTR_encrypt_avx512_64_aes_enc_block_last: - vaesenclast zmm0, zmm0, zmm13 + vaesenclast zmm0, zmm0, zmm14 vpxorq zmm0, zmm0, [r11] vmovdqu64 [rbx], zmm0 add eax, 64 cmp eax, r10d jl L_AES_CTR_encrypt_avx512_enc_64 L_AES_CTR_encrypt_avx512_done_64: + mov r10d, r8d + sub r10d, eax + cmp r10d, 32 + jl L_AES_CTR_encrypt_avx512_done_32 + ; 32 bytes of input + vbroadcasti32x4 zmm4, [ptr_L_aes_ctr_inc_avx512+32] + vpaddq ymm0, ymm7, [ptr_L_aes_ctr_inc_avx512] + vmovdqa64 ymm9, ymm7 + vpternlogq ymm9, ymm0, [ptr_L_aes_ctr_inc_avx512], 178 + vpsrlq ymm9, ymm9, 63 + vpslldq ymm9, ymm9, 8 + vpaddq ymm0, ymm0, ymm9 + vpshufb ymm0, ymm0, ymm8 + vmovdqa64 zmm9, zmm7 + vpaddq zmm7, zmm7, zmm4 + vpternlogq zmm9, zmm7, zmm4, 178 + vpsrlq zmm9, zmm9, 63 + vpslldq zmm9, zmm9, 8 + vpaddq zmm7, zmm7, zmm9 + ; aes_enc_block + vpxorq ymm0, ymm0, ymm15 + vaesenc ymm0, ymm0, ymm16 + vaesenc ymm0, ymm0, ymm17 + vaesenc ymm0, ymm0, ymm18 + vaesenc ymm0, ymm0, ymm19 + vaesenc ymm0, ymm0, ymm20 + vaesenc ymm0, ymm0, ymm21 + vaesenc ymm0, ymm0, ymm22 + vaesenc ymm0, ymm0, ymm23 + vaesenc ymm0, ymm0, ymm24 + cmp eax, 11 + vmovdqa64 ymm14, ymm25 + jl L_AES_CTR_encrypt_avx512_32_aes_enc_block_last + vaesenc ymm0, ymm0, ymm25 + vaesenc ymm0, ymm0, ymm26 + cmp eax, 13 + vmovdqa64 ymm14, ymm27 + jl L_AES_CTR_encrypt_avx512_32_aes_enc_block_last + vaesenc ymm0, ymm0, ymm27 + vaesenc ymm0, ymm0, ymm28 + vmovdqa64 ymm14, ymm29 +L_AES_CTR_encrypt_avx512_32_aes_enc_block_last: + vaesenclast ymm0, ymm0, ymm14 + lea r11, QWORD PTR [rcx+rax] + vpxorq ymm0, ymm0, [r11] + lea r11, QWORD PTR [rdx+rax] + vmovdqu YMMWORD PTR [r11], ymm0 + add eax, 32 +L_AES_CTR_encrypt_avx512_done_32: cmp eax, r8d mov r10d, r8d je L_AES_CTR_encrypt_avx512_done_enc @@ -4214,8 +4907,8 @@ L_AES_CTR_encrypt_avx512_enc_16: ; 16 bytes of input vpshufb xmm0, xmm7, xmm8 vmovdqa64 zmm9, zmm7 - vpaddq zmm7, zmm7, zmm12 - vpternlogq zmm9, zmm7, zmm12, 178 + vpaddq zmm7, zmm7, zmm13 + vpternlogq zmm9, zmm7, zmm13, 178 vpsrlq zmm9, zmm9, 63 vpslldq zmm9, zmm9, 8 vpaddq zmm7, zmm7, zmm9